Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zooba merged 3 commits into from
Jun 1, 2026
Merged

Add URL escaping/unescaping for winhttp URL functions. #353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c34ef0
Add URL escaping/unescaping for winhttp URL functions.
zooba May 29, 2026
235e816
Rename to escape
zooba May 29, 2026
e2bbd16
Apply suggestions from code review
zooba May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 113 additions & 42 deletions src/_native/winhttp.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
parts->lpszScheme = NULL;
parts->lpszUserName = NULL;
parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
winhttp_error();
return 0;
}
if (add_nuls) {
if (parts->lpszScheme && parts->dwSchemeLength > 0) {
parts->lpszScheme[parts->dwSchemeLength] = L'\0';
}
if (parts->lpszUserName && parts->dwUserNameLength > 0) {
parts->lpszUserName[parts->dwUserNameLength] = L'\0';
}
if (parts->lpszPassword && parts->dwPasswordLength > 0) {
parts->lpszPassword[parts->dwPasswordLength] = L'\0';
}
if (parts->lpszHostName && parts->dwHostNameLength > 0) {
parts->lpszHostName[parts->dwHostNameLength] = L'\0';
}
if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
if (!url_part) {
return NULL;
}
// Need to copy the incoming string to ensure it's null terminated
cch += 1;
wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
if (!url_string) {
PyErr_NoMemory();
return NULL;
}
wcsncpy_s(url_string, cch, url_part, cch - 1);
if (!url_string[0] || cch > 32767) {
// Too long/empty for the API, just bail out
return url_string;
}
if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
// Looks like an environment variable, so we won't change it.
return url_string;
}

wchar_t *result = NULL;
HRESULT r = E_POINTER;
for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
if (!result) {
PyMem_Free(url_string);
PyErr_NoMemory();
return NULL;
}
if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
if (encode) {
// "SEGMENT_ONLY" means we want to escape the entire string
r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
} else {
r = UrlUnescapeW(url_string, result, &cch, 0);
}
}
return 1;
PyMem_Free(url_string);
if (r) {
err_SetFromWindowsErrWithMessage((DWORD)r);
return NULL;
}
return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
wchar_t *url = NULL;
wchar_t *url2 = NULL; // a copy of url for splitting
wchar_t *method = NULL;
wchar_t *headers = NULL;
wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
uint64_t content_length;
PyObject *chunks = NULL;
uint64_t content_read = 0;
size_t n = 0;

wchar_t *hostname = NULL;
wchar_t *urlpath = NULL;
wchar_t *user = NULL;
wchar_t *pass = NULL;

if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
if (!accepts_array) {
goto exit;
}
n = wcslen(url) + 1;
url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
if (!url2) {
PyErr_NoMemory();
if (!crack_url(url, &url_parts)) {
goto exit;
}
hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
if (!hostname) {
goto exit;
}
wcscpy_s(url2, n, url);
if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
if (!urlpath) {
goto exit;
}


hSession = WinHttpOpen(
NULL,
WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

hConnection = WinHttpConnect(
hSession,
url_parts.lpszHostName,
hostname,
url_parts.nPort,
0
);
CHECK_WINHTTP(hConnection);

if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
url_parts.lpszUrlPath[0] = L'/';
}
hRequest = WinHttpOpenRequest(
hConnection,
method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
NULL,
WINHTTP_NO_REFERER,
accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
));

if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
if (!user) {
goto exit;
}
pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
if (!pass) {
goto exit;
}
CHECK_WINHTTP(WinHttpSetCredentials(
hRequest,
WINHTTP_AUTH_TARGET_SERVER,
WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
NULL
));
}
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
if (hSession) {
WinHttpCloseHandle(hSession);
}
PyMem_Free(user);
PyMem_Free(pass);
PyMem_Free(hostname);
PyMem_Free(urlpath);
PyMem_Free(accepts_array);
PyMem_Free(accepts);
PyMem_Free(headers);
PyMem_Free(method);
PyMem_Free(url2);
PyMem_Free(url);
return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
return NULL;
}
URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
if (!crack_url(url, &url_parts, 0)) {
if (!crack_url(url, &url_parts)) {
PyMem_Free(url);
return NULL;
}
// Deliberately not decoding host or path. We never use a blacklist, we only
// match against values specified by the user, or pass it to. If they want
// to provide the same URL with different encoding, that's their fault.
wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
(Py_ssize_t)url_parts.nPort,
url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
);
PyMem_Free(user);
PyMem_Free(pass);
PyMem_Free(url);
return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
Py_ssize_t port = 0;
wchar_t *user = NULL;
wchar_t *pass = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
as_utf16, &url.lpszScheme,
as_utf16, &url.lpszUserName,
as_utf16, &url.lpszPassword,
as_utf16, &user,
as_utf16, &pass,
as_utf16, &url.lpszHostName,
&port,
as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
}
DWORD cch = 0;
PyObject *r = NULL;
url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
if (user && !url.lpszUserName) {
goto exit;
}
url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
if (pass && !url.lpszPassword) {
goto exit;
}
url.nPort = (INTERNET_PORT)port;
if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
// Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
} else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
if (!buf) {
PyErr_NoMemory();
} else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
} else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
winhttp_error();
PyMem_Free(buf);
} else {
r = PyUnicode_FromWideChar(buf, cch);
PyMem_Free(buf);
}
}
exit:
PyMem_Free(user);
PyMem_Free(pass);
PyMem_Free(url.lpszScheme);
PyMem_Free(url.lpszUserName);
PyMem_Free(url.lpszPassword);
Expand Down
6 changes: 4 additions & 2 deletions src/manage/urlutils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
if ex.winerror in (12005, 12006):
return url
raise
p[U_USERNAME] = None
u = p[U_USERNAME]
if u and not (u.startswith("%") and u.endswith("%")):
p[U_USERNAME] = None
pw = p[U_PASSWORD]
if pw and not (pw.startswith("%") and pw.startswith("%")):
if pw and not (pw.startswith("%") and pw.endswith("%")):
p[U_PASSWORD] = None
return winhttp_urlunsplit(*p)

Expand Down
17 changes: 17 additions & 0 deletions tests/test_urlutils.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
("https://example.com/", "https://example.com/"),
("https://user@example.com/", "https://example.com/"),
("https://user:placeholder@example.com/", "https://example.com/"),
("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
candidates = ["https://user%40example.com:place%40holder@example.com/"]
url = "https://example.com/my_path"
expect = "https://user%40example.com:place%40holder@example.com/my_path"
assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
assert "1", "" == UU.extract_url_auth("https://1@example.com")

assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

# We are intentionally blind to encoded chars.
("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Add URL escaping/unescaping for winhttp URL functions.#353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

zooba
merged 3 commits into

from

Jun 1, 2026
 Merged


Add URL escaping/unescaping for winhttp URL functions.
#353
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
4c34ef0

Add URL escaping/unescaping for winhttp URL functions.


zooba May 29, 2026
235e816

Rename to escape


zooba May 29, 2026
e2bbd16

Apply suggestions from code review


zooba May 29, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


155 changes: 113 additions & 42 deletions
src/_native/winhttp.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
#include <Python.h>
#include <windows.h>
#include <shlwapi.h>
#include <winhttp.h>
#include <netlistmgr.h>

Expand DownExpand Up@@ -108,7 +109,7 @@ static wchar_t **split_to_array(wchar_t *str, wchar_t sep) {
}


static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
static int crack_url(const wchar_t *url, URL_COMPONENTS *parts) {
 parts->lpszScheme = NULL;
 parts->lpszUserName = NULL;
 parts->lpszPassword = NULL;
Expand All@@ -125,29 +126,66 @@ static int crack_url(wchar_t *url, URL_COMPONENTS *parts, int add_nuls) {
 winhttp_error();
 return 0;
 }
 if (add_nuls) {
 if (parts->lpszScheme && parts->dwSchemeLength > 0) {
 parts->lpszScheme[parts->dwSchemeLength] = L'\0';
 }
 if (parts->lpszUserName && parts->dwUserNameLength > 0) {
 parts->lpszUserName[parts->dwUserNameLength] = L'\0';
 }
 if (parts->lpszPassword && parts->dwPasswordLength > 0) {
 parts->lpszPassword[parts->dwPasswordLength] = L'\0';
 }
 if (parts->lpszHostName && parts->dwHostNameLength > 0) {
 parts->lpszHostName[parts->dwHostNameLength] = L'\0';
 }
 if (parts->lpszUrlPath && parts->dwUrlPathLength > 0) {
 parts->lpszUrlPath[parts->dwUrlPathLength] = L'\0';
 return 1;
}

static wchar_t *_escape_url_part(bool encode, const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 if (!url_part) {
 return NULL;
 }
 // Need to copy the incoming string to ensure it's null terminated
 cch += 1;
 wchar_t *url_string = (wchar_t *)PyMem_Malloc(sizeof(wchar_t) * cch);
 if (!url_string) {
 PyErr_NoMemory();
 return NULL;
 }
 wcsncpy_s(url_string, cch, url_part, cch - 1);
 if (!url_string[0] || cch > 32767) {
 // Too long/empty for the API, just bail out
 return url_string;
 }
 if (allow_env && cch > 2 && url_string[0] == L'%' && url_string[cch - 2] == L'%') {
 // Looks like an environment variable, so we won't change it.
 return url_string;
 }

 wchar_t *result = NULL;
 HRESULT r = E_POINTER;
 for (int retries = 3; retries > 0 && r == E_POINTER; --retries) {
 result = (wchar_t *)PyMem_Realloc(result, sizeof(wchar_t) * cch);
 if (!result) {
 PyMem_Free(url_string);
 PyErr_NoMemory();
 return NULL;
 }
 if (parts->lpszExtraInfo && parts->dwExtraInfoLength > 0) {
 parts->lpszExtraInfo[parts->dwExtraInfoLength] = L'\0';
 if (encode) {
 // "SEGMENT_ONLY" means we want to escape the entire string
 r = UrlEscapeW(url_string, result, &cch, URL_ESCAPE_SEGMENT_ONLY | URL_ESCAPE_ASCII_URI_COMPONENT);
 } else {
 r = UrlUnescapeW(url_string, result, &cch, 0);
 }
 }
 return 1;
 PyMem_Free(url_string);
 if (r) {
 err_SetFromWindowsErrWithMessage((DWORD)r);
 return NULL;
 }
 return result;
}

static wchar_t *escape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(true, url_part, cch, allow_env);
}

static wchar_t *unescape_url_part(const wchar_t *url_part, DWORD cch, bool allow_env=false)
{
 return _escape_url_part(false, url_part, cch, allow_env);
}


extern "C" {

#define CHECK_WINHTTP(x) if (!x) { winhttp_error(); goto exit; }
Expand DownExpand Up@@ -226,7 +264,6 @@ static bool winhttp_apply_proxy(HINTERNET hSession, HINTERNET hRequest, const wc
PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"url", "method", "headers", "accepts", "chunksize", "on_progress", "on_cred_request", NULL};
 wchar_t *url = NULL;
 wchar_t *url2 = NULL; // a copy of url for splitting
 wchar_t *method = NULL;
 wchar_t *headers = NULL;
 wchar_t *accepts = NULL;
Expand All@@ -246,7 +283,11 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 uint64_t content_length;
 PyObject *chunks = NULL;
 uint64_t content_read = 0;
 size_t n = 0;

 wchar_t *hostname = NULL;
 wchar_t *urlpath = NULL;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;

 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&|nOO:winhttp_urlopen", keywords,
 as_utf16, &url, as_utf16, &method, as_utf16, &headers, as_utf16, &accepts, &chunksize, &on_progress, &on_cred_request)) {
Expand All@@ -264,17 +305,19 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (!accepts_array) {
 goto exit;
 }
 n = wcslen(url) + 1;
 url2 = (wchar_t *)PyMem_Malloc(n * sizeof(wchar_t));
 if (!url2) {
 PyErr_NoMemory();
 if (!crack_url(url, &url_parts)) {
 goto exit;
 }
 hostname = unescape_url_part(url_parts.lpszHostName, url_parts.dwHostNameLength);
 if (!hostname) {
 goto exit;
 }
wcscpy_s(url2, n, url);
 if (!crack_url(url2, &url_parts, 1)) {
urlpath = unescape_url_part(url_parts.lpszUrlPath, url_parts.dwUrlPathLength);
 if (!urlpath) {
 goto exit;
 }


 hSession = WinHttpOpen(
 NULL,
 WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
Expand DownExpand Up@@ -310,19 +353,16 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {

 hConnection = WinHttpConnect(
 hSession,
url_parts.lpszHostName,
hostname,
 url_parts.nPort,
 0
 );
 CHECK_WINHTTP(hConnection);

 if (url_parts.dwUrlPathLength && !url_parts.lpszUrlPath[0]) {
 url_parts.lpszUrlPath[0] = L'/';
 }
 hRequest = WinHttpOpenRequest(
 hConnection,
 method,
url_parts.lpszUrlPath,
urlpath && urlpath[0] ? urlpath : L"/",
 NULL,
 WINHTTP_NO_REFERER,
 accepts_array,
Expand All@@ -341,12 +381,20 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 ));

 if (url_parts.dwUserNameLength || url_parts.dwPasswordLength) {
 user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength);
 if (!user) {
 goto exit;
 }
 pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength);
 if (!pass) {
 goto exit;
 }
 CHECK_WINHTTP(WinHttpSetCredentials(
 hRequest,
 WINHTTP_AUTH_TARGET_SERVER,
 WINHTTP_AUTH_SCHEME_BASIC,
url_parts.lpszUserName,
url_parts.lpszPassword,
user,
pass,
 NULL
 ));
 }
Expand DownExpand Up@@ -464,11 +512,14 @@ PyObject *winhttp_urlopen(PyObject *, PyObject *args, PyObject *kwargs) {
 if (hSession) {
 WinHttpCloseHandle(hSession);
 }
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(hostname);
 PyMem_Free(urlpath);
 PyMem_Free(accepts_array);
 PyMem_Free(accepts);
 PyMem_Free(headers);
 PyMem_Free(method);
 PyMem_Free(url2);
 PyMem_Free(url);
 return result;
}
Expand DownExpand Up@@ -510,19 +561,26 @@ PyObject *winhttp_urlsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 return NULL;
 }
 URL_COMPONENTS url_parts = { sizeof(URL_COMPONENTS) };
 if (!crack_url(url, &url_parts, 0)) {
 if (!crack_url(url, &url_parts)) {
 PyMem_Free(url);
 return NULL;
 }
 // Deliberately not decoding host or path. We never use a blacklist, we only
 // match against values specified by the user, or pass it to. If they want
 // to provide the same URL with different encoding, that's their fault.
 wchar_t *user = unescape_url_part(url_parts.lpszUserName, url_parts.dwUserNameLength, true);
 wchar_t *pass = unescape_url_part(url_parts.lpszPassword, url_parts.dwPasswordLength, true);
 PyObject *r = Py_BuildValue("(u#u#u#u#nu#u#)",
 url_parts.lpszScheme, (Py_ssize_t)url_parts.dwSchemeLength,
url_parts.lpszUserName, (Py_ssize_t)url_parts.dwUserNameLength,
url_parts.lpszPassword, (Py_ssize_t)url_parts.dwPasswordLength,
user, (Py_ssize_t)(user ? wcslen(user) : 0),
pass, (Py_ssize_t)(pass ? wcslen(pass) : 0),
 url_parts.lpszHostName, (Py_ssize_t)url_parts.dwHostNameLength,
 (Py_ssize_t)url_parts.nPort,
 url_parts.lpszUrlPath, (Py_ssize_t)url_parts.dwUrlPathLength,
 url_parts.lpszExtraInfo, (Py_ssize_t)url_parts.dwExtraInfoLength
 );
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url);
 return r;
}
Expand All@@ -532,10 +590,12 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 static const char * keywords[] = {"scheme", "user", "password", "netloc", "port", "path", "extra", NULL};
 URL_COMPONENTS url = { sizeof(URL_COMPONENTS) };
 Py_ssize_t port = 0;
 wchar_t *user = NULL;
 wchar_t *pass = NULL;
 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O&O&O&nO&O&:winhttp_urlunsplit", keywords,
 as_utf16, &url.lpszScheme,
 as_utf16, &url.lpszUserName,
 as_utf16, &url.lpszPassword,
 as_utf16, &user,
 as_utf16, &pass,
 as_utf16, &url.lpszHostName,
 &port,
 as_utf16, &url.lpszUrlPath,
Expand All@@ -545,8 +605,16 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 }
 DWORD cch = 0;
 PyObject *r = NULL;
 url.lpszUserName = escape_url_part(user, user ? wcslen(user) : 0, true);
 if (user && !url.lpszUserName) {
 goto exit;
 }
 url.lpszPassword = escape_url_part(pass, pass ? wcslen(pass) : 0, true);
 if (pass && !url.lpszPassword) {
 goto exit;
 }
 url.nPort = (INTERNET_PORT)port;
 if (WinHttpCreateUrl(&url, ICU_ESCAPE, NULL, &cch)) {
 if (WinHttpCreateUrl(&url, 0, NULL, &cch)) {
 // Success path, because it should've failed with ERROR_INSUFFICIENT_BUFFER
 PyErr_SetString(PyExc_ValueError, "unable to unsplit URL");
 } else if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
Expand All@@ -556,14 +624,17 @@ PyObject *winhttp_urlunsplit(PyObject *, PyObject *args, PyObject *kwargs) {
 wchar_t *buf = (wchar_t*)PyMem_Malloc(cch * sizeof(wchar_t));
 if (!buf) {
 PyErr_NoMemory();
 } else if (!WinHttpCreateUrl(&url, ICU_ESCAPE, buf, &cch)) {
 } else if (!WinHttpCreateUrl(&url, 0, buf, &cch)) {
 winhttp_error();
 PyMem_Free(buf);
 } else {
 r = PyUnicode_FromWideChar(buf, cch);
 PyMem_Free(buf);
 }
 }
exit:
 PyMem_Free(user);
 PyMem_Free(pass);
 PyMem_Free(url.lpszScheme);
 PyMem_Free(url.lpszUserName);
 PyMem_Free(url.lpszPassword);
Expand Down

6 changes: 4 additions & 2 deletions
src/manage/urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -552,9 +552,11 @@ def sanitise_url(url):
 if ex.winerror in (12005, 12006):
 return url
 raise
 p[U_USERNAME] = None
 u = p[U_USERNAME]
 if u and not (u.startswith("%") and u.endswith("%")):
 p[U_USERNAME] = None
 pw = p[U_PASSWORD]
 if pw and not (pw.startswith("%") and pw.startswith("%")):
 if pw and not (pw.startswith("%") and pw.endswith("%")):
 p[U_PASSWORD] = None
 return winhttp_urlunsplit(*p)

Expand Down

17 changes: 17 additions & 0 deletions
tests/test_urlutils.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,6 +11,8 @@
 ("https://example.com/", "https://example.com/"),
 ("https://user@example.com/", "https://example.com/"),
 ("https://user:placeholder@example.com/", "https://example.com/"),
 ("https://%placeholder%@example.com/", "https://%placeholder%@example.com/"),
 ("https://%user%:%placeholder%@example.com/", "https://%user%:%placeholder%@example.com/"),
]])
def test_urlsanitise(url, expect):
 assert expect == UU.sanitise_url(url)
Expand All@@ -30,10 +32,20 @@ def test_urlunsanitise():
 assert None == UU.unsanitise_url(url, candidates)


def test_urlunsanitise_encoded():
 candidates = ["https://user%40example.com:place%40holder@example.com/"]
 url = "https://example.com/my_path"
 expect = "https://user%40example.com:place%40holder@example.com/my_path"
 assert expect == UU.unsanitise_url(url, candidates)


def test_extract_url_auth():
 assert "1", "2" == UU.extract_url_auth("https://1:2@example.com")
 assert "1", "" == UU.extract_url_auth("https://1@example.com")

 assert ("1", "2") == UU.extract_url_auth("https://%31:%32@example.com")
 assert ("1", "") == UU.extract_url_auth("https://%31@example.com")

 os.environ["PYMANAGER_TEST_VALUE"] = v = str(time.time())
 assert "1", v == UU.extract_url_auth("https://1:%PYMANAGER_TEST_VALUE%@example.com")

Expand All@@ -48,6 +60,11 @@ def test_extract_url_auth():
 ("https://example.com/A/B/C", "//EXAMPLE.COM/A", True, "https://EXAMPLE.COM/A"),
 ("https://example.com/A/B/C", "//EXAMPLE.COM/", None, "https://EXAMPLE.COM/"),

 # We are intentionally blind to encoded chars.
 ("https://example.com/A/B/C", "%2fD", False, "https://example.com/A/B/C/%2fD"),
 ("https://example.com/A/B/C", "%2f%2fD", False, "https://example.com/A/B/C/%2f%2fD"),
 ("https://example.com/A/B%2fC", "D", True, "https://example.com/A/D"),

 ("file:///C:/local/path", "file.json", False, "file:///C:/local/path/file.json"),
 ("file:///C:/local/path", "file.json", True, "file:///C:/local/file.json"),
 ("file:///C:/local/path", ".\\dir\\file.json", False, "file:///C:/local/path/dir/file.json"),
Expand Down
Loading