Skip to content

Commit 5630770

Browse files
sindresorhussindresorhus
committed
Preserve percent-encoded reserved query characters
Fixes #189
1 parent 2f2536e commit 5630770

2 files changed

Lines changed: 154 additions & 12 deletions

File tree

‎index.js‎

Lines changed: 82 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
22
const DATA_URL_DEFAULT_MIME_TYPE = 'text/plain';
33
const DATA_URL_DEFAULT_CHARSET = 'us-ascii';
4+
const encodedReservedCharactersPattern = '%(?:3A|2F|3F|23|5B|5D|40|21|24|26|27|28|29|2A|2B|2C|3B|3D)';
5+
const temporaryEncodedReservedTokenBase = '__normalize_url_encoded_reserved__';
6+
const temporaryEncodedReservedTokenPattern = /__normalize_url_encoded_reserved__(\d+)__/g;
7+
const hasEncodedReservedCharactersRegex = new RegExp(encodedReservedCharactersPattern, 'i');
8+
const encodedReservedCharactersRegex = new RegExp(encodedReservedCharactersPattern, 'gi');
49

510
const testParameter = (name, filters) => Array.isArray(filters) && filters.some(filter => {
611
if (filter instanceof RegExp) {
@@ -56,6 +61,63 @@ const getKeysWithoutEquals = search => {
5661
return keys;
5762
};
5863

64+
const getTemporaryEncodedReservedTokenPrefix = search => {
65+
let decodedSearch = search;
66+
67+
try {
68+
decodedSearch = decodeURIComponent(search);
69+
} catch {
70+
decodedSearch = new URLSearchParams(search).toString();
71+
}
72+
73+
const getUsedTokenIndexes = value => {
74+
const indexes = new Set();
75+
76+
for (const match of value.matchAll(temporaryEncodedReservedTokenPattern)) {
77+
indexes.add(Number.parseInt(match[1], 10));
78+
}
79+
80+
return indexes;
81+
};
82+
83+
const usedTokenIndexes = getUsedTokenIndexes(search);
84+
for (const tokenIndex of getUsedTokenIndexes(decodedSearch)) {
85+
usedTokenIndexes.add(tokenIndex);
86+
}
87+
88+
let tokenIndex = 0;
89+
while (usedTokenIndexes.has(tokenIndex)) {
90+
tokenIndex++;
91+
}
92+
93+
return `${temporaryEncodedReservedTokenBase}${tokenIndex}__`;
94+
};
95+
96+
const sortSearchParameters = (searchParameters, encodedReservedTokenRegex) => {
97+
if (!encodedReservedTokenRegex) {
98+
searchParameters.sort();
99+
return searchParameters.toString();
100+
}
101+
102+
const getSortableKey = key => key.replace(encodedReservedTokenRegex, (_, hexCode) => String.fromCodePoint(Number.parseInt(hexCode, 16)));
103+
const entries = [...searchParameters.entries()];
104+
entries.sort(([leftKey], [rightKey]) => {
105+
const left = getSortableKey(leftKey);
106+
const right = getSortableKey(rightKey);
107+
return left < right ? -1 : left > right ? 1 : 0;
108+
});
109+
110+
return new URLSearchParams(entries).toString();
111+
};
112+
113+
const decodeReservedTokens = (value, encodedReservedTokenRegex) => {
114+
if (!encodedReservedTokenRegex) {
115+
return value;
116+
}
117+
118+
return value.replace(encodedReservedTokenRegex, (_, hexCode) => String.fromCodePoint(Number.parseInt(hexCode, 16)));
119+
};
120+
59121
const normalizeEmptyQueryParameters = (search, emptyQueryValue, originalSearch) => {
60122
const isAlways = emptyQueryValue === 'always';
61123
const isNever = emptyQueryValue === 'never';
@@ -94,8 +156,8 @@ const normalizeEmptyQueryParameters = (search, emptyQueryValue, originalSearch)
94156
return formatEmptyValue(normalizeKey(key));
95157
}
96158

97-
// Normalize + to %20 in key, decode %3D to = in values (= is safe unencoded in query values)
98-
return `${normalizeKey(key)}=${value.replace(/%3D/gi, '=')}`;
159+
// Normalize + to %20 in key.
160+
return `${normalizeKey(key)}=${value}`;
99161
};
100162

101163
const params = search.slice(1).split('&').filter(Boolean);
@@ -308,14 +370,22 @@ export default function normalizeUrl(urlString, options) {
308370

309371
// Capture original query params format before any searchParams modifications
310372
const originalSearch = urlObject.search;
373+
let encodedReservedTokenRegex;
374+
375+
if (options.sortQueryParameters && hasEncodedReservedCharactersRegex.test(originalSearch)) {
376+
const encodedReservedTokenPrefix = getTemporaryEncodedReservedTokenPrefix(originalSearch);
377+
urlObject.search = originalSearch.replace(encodedReservedCharactersRegex, match => `${encodedReservedTokenPrefix}${match.slice(1).toUpperCase()}`);
378+
encodedReservedTokenRegex = new RegExp(`${encodedReservedTokenPrefix}([0-9A-F]{2})`, 'g');
379+
}
380+
311381
const hasKeepQueryParameters = Array.isArray(options.keepQueryParameters);
312382
const searchParams = urlObject.searchParams;
313383

314384
// Remove query unwanted parameters
315385
if (!hasKeepQueryParameters && Array.isArray(options.removeQueryParameters) && options.removeQueryParameters.length > 0) {
316386
// eslint-disable-next-line unicorn/no-useless-spread -- We are intentionally spreading to get a copy.
317387
for (const key of [...searchParams.keys()]) {
318-
if (testParameter(key, options.removeQueryParameters)) {
388+
if (testParameter(decodeReservedTokens(key, encodedReservedTokenRegex), options.removeQueryParameters)) {
319389
searchParams.delete(key);
320390
}
321391
}
@@ -329,7 +399,7 @@ export default function normalizeUrl(urlString, options) {
329399
if (hasKeepQueryParameters && options.keepQueryParameters.length > 0) {
330400
// eslint-disable-next-line unicorn/no-useless-spread -- We are intentionally spreading to get a copy.
331401
for (const key of [...searchParams.keys()]) {
332-
if (!testParameter(key, options.keepQueryParameters)) {
402+
if (!testParameter(decodeReservedTokens(key, encodedReservedTokenRegex), options.keepQueryParameters)) {
333403
searchParams.delete(key);
334404
}
335405
}
@@ -339,11 +409,15 @@ export default function normalizeUrl(urlString, options) {
339409

340410
// Sort query parameters
341411
if (options.sortQueryParameters) {
342-
urlObject.searchParams.sort();
412+
urlObject.search = sortSearchParameters(urlObject.searchParams, encodedReservedTokenRegex);
413+
414+
// Sorting and serializing encode the search parameters, so we need to decode them again.
415+
// Protect &%#? and %2B from decoding (would break URL structure or change meaning) by double-encoding them first.
416+
urlObject.search = decodeURIComponent(urlObject.search.replace(/%(?:26|23|3F|25|2B)/gi, match => `%25${match.slice(1)}`));
343417

344-
// Calling `.sort()` encodes the search parameters, so we need to decode them again.
345-
// Protect &=%#? %25 and %2B from decoding (would break URL structure or change meaning) by double-encoding them first.
346-
urlObject.search = decodeURIComponent(urlObject.search.replace(/%(?:26|3D|23|3F|25|2B)/gi, match => `%25${match.slice(1)}`));
418+
if (encodedReservedTokenRegex) {
419+
urlObject.search = urlObject.search.replace(encodedReservedTokenRegex, '%$1');
420+
}
347421
}
348422

349423
// Normalize empty query parameter values

‎test.js‎

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,74 @@ test('path-like query strings without equals signs are preserved', t => {
569569
t.is(normalizeUrl('https://example.com/index.php?/api&/api/v1/users'), 'https://example.com/index.php?/api&/api/v1/users');
570570
});
571571

572+
test('sortQueryParameters should preserve encoded reserved characters in query values', t => {
573+
// Issue #189 - `%2F` in query values must remain encoded
574+
t.is(
575+
normalizeUrl('https://example.com/?X-Amz-Credential=AKIA%2F20200101%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-SignedHeaders=host'),
576+
'https://example.com/?X-Amz-Credential=AKIA%2F20200101%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-SignedHeaders=host'
577+
);
578+
t.is(normalizeUrl('https://example.com/?token=a%2Fb%2Fc'), 'https://example.com/?token=a%2Fb%2Fc');
579+
t.is(normalizeUrl('https://example.com/?token=a%2fb%2fc'), 'https://example.com/?token=a%2Fb%2Fc');
580+
581+
const encodedReservedCharacters = ['%3A', '%2F', '%3F', '%23', '%5B', '%5D', '%40', '%21', '%24', '%26', '%27', '%28', '%29', '%2A', '%2B', '%2C', '%3B', '%3D'];
582+
for (const encodedReservedCharacter of encodedReservedCharacters) {
583+
t.is(normalizeUrl(`https://example.com/?value=${encodedReservedCharacter}`), `https://example.com/?value=${encodedReservedCharacter}`);
584+
}
585+
586+
t.is(normalizeUrl('https://example.com/?value=:@[];,'), 'https://example.com/?value=:@[];,');
587+
588+
// Encoded reserved characters survive sort reordering
589+
t.is(normalizeUrl('https://example.com/?z=1&token=a%2Fb'), 'https://example.com/?token=a%2Fb&z=1');
590+
591+
// Encoded reserved characters in keys with sort reordering
592+
t.is(normalizeUrl('https://example.com/?A=1&%3A=2'), 'https://example.com/?%3A=2&A=1');
593+
t.is(normalizeUrl('https://example.com/?foo%3Abar=1&a=2'), 'https://example.com/?a=2&foo%3Abar=1');
594+
t.is(normalizeUrl('https://example.com/?foo%3Dbar=1&a=2'), 'https://example.com/?a=2&foo%3Dbar=1');
595+
t.is(normalizeUrl('https://example.com/?b%26c=1&a=2'), 'https://example.com/?a=2&b%26c=1');
596+
597+
// Both keys with encoded reserved characters sort by decoded code point
598+
t.is(normalizeUrl('https://example.com/?%3A=1&%2F=2'), 'https://example.com/?%2F=2&%3A=1'); // / (47) < : (58)
599+
t.is(normalizeUrl('https://example.com/?%5B=1&%3A=2'), 'https://example.com/?%3A=2&%5B=1'); // : (58) < [ (91)
600+
601+
// Encoded reserved characters in both keys and values with sort reordering
602+
t.is(normalizeUrl('https://example.com/?z%3A=val%2F&a%2F=val%3A'), 'https://example.com/?a%2F=val%3A&z%3A=val%2F');
603+
604+
// Multiple different encoded reserved characters in a single value
605+
t.is(normalizeUrl('https://example.com/?q=%3A%2F%3F'), 'https://example.com/?q=%3A%2F%3F');
606+
607+
// Interaction with removeQueryParameters
608+
t.is(normalizeUrl('https://example.com/?utm_source=test&token=a%2Fb'), 'https://example.com/?token=a%2Fb');
609+
t.is(normalizeUrl('https://example.com/?token=a/b&utm_source=test'), 'https://example.com/?token=a/b');
610+
t.is(
611+
normalizeUrl('https://example.com/?foo%3Abar=1&baz=2', {removeQueryParameters: ['foo:bar']}),
612+
'https://example.com/?baz=2'
613+
);
614+
t.is(
615+
normalizeUrl('https://example.com/?foo%3Abar=1&baz=2', {removeQueryParameters: false, keepQueryParameters: ['foo:bar']}),
616+
'https://example.com/?foo%3Abar=1'
617+
);
618+
619+
// Preserved when sortQueryParameters is disabled
620+
t.is(normalizeUrl('https://example.com/?token=a%2Fb', {sortQueryParameters: false}), 'https://example.com/?token=a%2Fb');
621+
t.is(normalizeUrl('https://example.com/?token=%3A%2F%3F', {sortQueryParameters: false}), 'https://example.com/?token=%3A%2F%3F');
622+
623+
// Malformed percent-encoding should not allow token-collision rewrites in user data
624+
t.is(
625+
normalizeUrl('https://example.com/?broken=%E0%A4&literal=%5F%5Fnormalize_url_encoded_reserved__2F&token=a%2Fb'),
626+
'https://example.com/?broken=%EF%BF%BD&literal=__normalize_url_encoded_reserved__2F&token=a%2Fb'
627+
);
628+
t.is(
629+
normalizeUrl('https://example.com/?broken=%E0%A4&literal=%5F%5Fnormalize_url_encoded_reserved__0__2F&token=a%2Fb'),
630+
'https://example.com/?broken=%EF%BF%BD&literal=__normalize_url_encoded_reserved__0__2F&token=a%2Fb'
631+
);
632+
633+
const longPadding = '_'.repeat(4000);
634+
t.is(
635+
normalizeUrl(`https://example.com/?token=a%2Fb&literal=${longPadding}__normalize_url_encoded_reserved__${longPadding}`),
636+
`https://example.com/?literal=${longPadding}__normalize_url_encoded_reserved__${longPadding}&token=a%2Fb`
637+
);
638+
});
639+
572640
test('emptyQueryValue option', t => {
573641
// Default 'preserve' behavior - keeps original format
574642
t.is(normalizeUrl('https://example.com?key'), 'https://example.com/?key');
@@ -659,10 +727,10 @@ test('emptyQueryValue option', t => {
659727
t.is(normalizeUrl('https://example.com?key=a=b=c'), 'https://example.com/?key=a=b=c');
660728
t.is(normalizeUrl('https://example.com?data=abc=='), 'https://example.com/?data=abc==');
661729

662-
// Encoded = (%3D) in values gets decoded (= is safe unencoded in values)
663-
t.is(normalizeUrl('https://example.com?key=val%3Due'), 'https://example.com/?key=val=ue');
664-
t.is(normalizeUrl('https://example.com?key=%3D'), 'https://example.com/?key==');
665-
t.is(normalizeUrl('https://example.com?key=val%3Due', {sortQueryParameters: false}), 'https://example.com/?key=val=ue');
730+
// Encoded = (%3D) in values is preserved
731+
t.is(normalizeUrl('https://example.com?key=val%3Due'), 'https://example.com/?key=val%3Due');
732+
t.is(normalizeUrl('https://example.com?key=%3D'), 'https://example.com/?key=%3D');
733+
t.is(normalizeUrl('https://example.com?key=val%3Due', {sortQueryParameters: false}), 'https://example.com/?key=val%3Due');
666734

667735
// All params removed leaves no query string
668736
t.is(normalizeUrl('https://example.com?utm_source=test&utm_medium=web'), 'https://example.com');

0 commit comments

Comments
 (0)