Make piped header copying opt-in and preserve explicit headers

Author: sindresorhus

documentation/2-options.md

@@ -500,51 +500,51 @@ Therefore this option has no effect when using HTTP/2.
 ### `copyPipedHeaders`
 
 **Type: `boolean`**\
-**Default: `true`**
+**Default: `false`**
 
 Automatically copy headers from piped streams.
 
 When piping a request into a Got stream (e.g., `request.pipe(got.stream(url))`), this controls whether headers from the source stream are automatically merged into the Got request headers.
 
-**Note:** Piped headers overwrite any explicitly set headers with the same name. To override this, either set `copyPipedHeaders` to `false` and manually copy safe headers, or use a `beforeRequest` hook to force specific header values after piping.
+**Note:** Explicitly set headers take precedence over piped headers. Piped headers are only copied when a header is not already explicitly set.
 
-Useful for proxy scenarios, but you may want to disable this to filter out headers like `Host`, `Connection`, `Authorization`, etc.
+Useful for proxy scenarios when explicitly enabled, but you may still want to filter out headers like `Host`, `Connection`, `Authorization`, etc.
 
-**Example: Disable automatic header copying and manually copy only safe headers**
+**Example: Opt in to automatic header copying for proxy scenarios**
 
 ```js
 import got from 'got';
 import {pipeline} from 'node:stream/promises';
 
 server.get('/proxy', async (request, response) => {
 	const gotStream = got.stream('https://example.com', {
-		copyPipedHeaders: false,
+		copyPipedHeaders: true,
+		// Explicit headers win over piped headers
 		headers: {
-			'user-agent': request.headers['user-agent'],
-			'accept': request.headers['accept'],
-			// Explicitly NOT copying host, connection, authorization, etc.
+			host: 'example.com',
 		}
 	});
 
 	await pipeline(request, gotStream, response);
 });
 ```
 
-**Example: Override piped headers using beforeRequest hook**
+**Example: Keep it disabled and manually copy only safe headers**
 
 ```js
 import got from 'got';
+import {pipeline} from 'node:stream/promises';
 
-const gotStream = got.stream('https://example.com', {
-	hooks: {
-		beforeRequest: [
-			options => {
-				// Force specific header values after piping
-				options.headers.host = 'example.com';
-				delete options.headers.authorization;
-			}
-		]
-	}
+server.get('/proxy', async (request, response) => {
+	const gotStream = got.stream('https://example.com', {
+		headers: {
+			'user-agent': request.headers['user-agent'],
+			'accept': request.headers['accept'],
+			// Explicitly NOT copying host, connection, authorization, etc.
+		}
+	});
+
+	await pipeline(request, gotStream, response);
 });
 ```
 

documentation/migration-guides/request.md

@@ -95,6 +95,10 @@ Readability is very important to us, so we have different names for these option
 - No `removeRefererHeader` option.\
   You can remove the `referer` header in a [`beforeRequest` hook](../9-hooks.md#beforerequest).
 - No `followAllRedirects` option.
+- [`copyPipedHeaders`](../2-options.md#copypipedheaders) defaults to `false`.\
+  Piped request headers are no longer copied automatically. Opt in with `copyPipedHeaders: true` for proxy scenarios.
+- With `copyPipedHeaders: true`, explicitly set headers win over piped headers.\
+  Piped headers only fill headers that were not explicitly set.
 
 Hooks are very powerful. [Read more](../9-hooks.md) to see what else you achieve using hooks.
 
@@ -110,7 +114,7 @@ http.createServer((serverRequest, serverResponse) => {
 });
 ```
 
-The cool feature here is that Request can proxy headers with the stream, but Got can do that too!
+Request can proxy headers with the stream. Got can do that too, but it is opt-in:
 
 ```js
 import {pipeline as streamPipeline} from 'node:stream/promises';
@@ -119,7 +123,8 @@ import got from 'got';
 const server = http.createServer(async (serverRequest, serverResponse) => {
 	if (serverRequest.url === '/doodle.png') {
 		await streamPipeline(
-			got.stream('https://example.com/doodle.png'),
+			serverRequest,
+			got.stream('https://example.com/doodle.png', {copyPipedHeaders: true}),
 			serverResponse
 		);
 	}
@@ -128,7 +133,7 @@ const server = http.createServer(async (serverRequest, serverResponse) => {
 server.listen(8080);
 ```
 
-In terms of streams nothing has really changed.
+In terms of stream usage, nothing has really changed, but header proxying is opt-in via `copyPipedHeaders: true`.
 
 #### Convenience methods
 

source/core/errors.ts

@@ -1,4 +1,5 @@
 import is from '@sindresorhus/is';
+import stripUrlAuth from './utils/strip-url-auth.js';
 import type {Timings} from './utils/timer.js';
 import type Options from './options.js';
 import type {TimeoutError as TimedOutTimeoutError} from './timed-out.js';
@@ -99,7 +100,7 @@ export class HTTPError<T = unknown> extends RequestError<T> {
 	declare readonly timings: Timings;
 
 	constructor(response: PlainResponse) {
-		super(`Request failed with status code ${response.statusCode} (${response.statusMessage!}): ${response.request.options.method} ${response.request.options.url!.toString()}`, {}, response.request);
+		super(`Request failed with status code ${response.statusCode} (${response.statusMessage!}): ${response.request.options.method} ${stripUrlAuth(response.request.options.url!)}`, {}, response.request);
 	}
 }
 

source/core/index.ts

@@ -21,6 +21,7 @@ import getBodySize from './utils/get-body-size.js';
 import proxyEvents from './utils/proxy-events.js';
 import timedOut, {TimeoutError as TimedOutTimeoutError} from './timed-out.js';
 import urlToOptions from './utils/url-to-options.js';
+import stripUrlAuth from './utils/strip-url-auth.js';
 import WeakableMap from './utils/weakable-map.js';
 import calculateRetryDelay from './calculate-retry-delay.js';
 import Options, {
@@ -201,6 +202,7 @@ const normalizeError = (error: unknown): Error => {
 type UrlType = ConstructorParameters<typeof Options>[0];
 type OptionsType = ConstructorParameters<typeof Options>[1];
 type DefaultsType = ConstructorParameters<typeof Options>[2];
+const getSanitizedUrl = (options?: Options): string => options?.url ? stripUrlAuth(options.url) : '';
 
 export default class Request extends Duplex implements RequestEvents<Request> {
 	// @ts-expect-error - Ignoring for now.
@@ -254,7 +256,15 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 
 		this.on('pipe', (source: NodeJS.ReadableStream & {headers?: Record<string, string | string[] | undefined>}) => {
 			if (this.options.copyPipedHeaders && source?.headers) {
-				Object.assign(this.options.headers, source.headers);
+				for (const [header, value] of Object.entries(source.headers)) {
+					const normalizedHeader = header.toLowerCase();
+
+					if (!this.options.shouldCopyPipedHeader(normalizedHeader)) {
+						continue;
+					}
+
+					this.options.setPipedHeader(normalizedHeader, value);
+				}
 			}
 		});
 
@@ -280,7 +290,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 			// Publish request creation event
 			publishRequestCreate({
 				requestId: this._requestId,
-				url: this.options.url?.toString() ?? '',
+				url: getSanitizedUrl(this.options),
 				method: this.options.method,
 			});
 		} catch (error: unknown) {
@@ -751,7 +761,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 
 	private async _finalizeBody(): Promise<void> {
 		const {options} = this;
-		const {headers} = options;
+		const headers = options.getInternalHeaders();
 
 		const isForm = !is.undefined(options.form);
 		// eslint-disable-next-line @typescript-eslint/naming-convention
@@ -800,7 +810,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 				options.body = options.stringifyJson(json);
 			}
 
-			const uploadBodySize = getBodySize(options.body, options.headers);
+			const uploadBodySize = getBodySize(options.body, headers);
 
 			// See https://tools.ietf.org/html/rfc7230#section-3.3.2
 			// A user agent SHOULD send a Content-Length in a request message when
@@ -816,8 +826,8 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 			}
 		}
 
-		if (options.responseType === 'json' && !('accept' in options.headers)) {
-			options.headers.accept = 'application/json';
+		if (options.responseType === 'json' && !('accept' in headers)) {
+			headers.accept = 'application/json';
 		}
 
 		this._bodySize = Number(headers['content-length']) || undefined;
@@ -873,7 +883,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 		}
 
 		typedResponse.statusMessage ||= http.STATUS_CODES[statusCode]; // eslint-disable-line @typescript-eslint/prefer-nullish-coalescing -- The status message can be empty.
-		typedResponse.url = options.url!.toString();
+		typedResponse.url = stripUrlAuth(options.url!);
 		typedResponse.requestUrl = this.requestUrl!;
 		typedResponse.redirectUrls = this.redirectUrls;
 		typedResponse.request = this;
@@ -981,7 +991,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 					updatedOptions.json = undefined;
 					updatedOptions.form = undefined;
 
-					delete updatedOptions.headers['content-length'];
+					updatedOptions.deleteInternalHeader('content-length');
 					// Only clear `_bodySize` when the redirect drops the request body.
 					this._bodySize = undefined;
 				}
@@ -998,21 +1008,24 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 
 					// Redirecting to a different site, clear sensitive data.
 					// For UNIX sockets, different socket paths are also different origins.
-					const isDifferentOrigin = redirectUrl.hostname !== (url as URL).hostname
+					const isDifferentOrigin = redirectUrl.protocol !== (url as URL).protocol
+						|| redirectUrl.hostname !== (url as URL).hostname
 						|| redirectUrl.port !== (url as URL).port
 						|| getUnixSocketPath(url as URL) !== getUnixSocketPath(redirectUrl);
 
 					if (isDifferentOrigin) {
-						if ('host' in updatedOptions.headers) {
-							delete updatedOptions.headers.host;
+						const updatedHeaders = updatedOptions.getInternalHeaders();
+
+						if ('host' in updatedHeaders) {
+							updatedOptions.deleteInternalHeader('host');
 						}
 
-						if ('cookie' in updatedOptions.headers) {
-							delete updatedOptions.headers.cookie;
+						if ('cookie' in updatedHeaders) {
+							updatedOptions.deleteInternalHeader('cookie');
 						}
 
-						if ('authorization' in updatedOptions.headers) {
-							delete updatedOptions.headers.authorization;
+						if ('authorization' in updatedHeaders) {
+							updatedOptions.deleteInternalHeader('authorization');
 						}
 
 						if (updatedOptions.username || updatedOptions.password) {
@@ -1221,7 +1234,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 		// Publish request start event
 		publishRequestStart({
 			requestId: this._requestId,
-			url: url?.toString() ?? '',
+			url: getSanitizedUrl(this.options),
 			method: options.method,
 			headers: options.headers,
 		});
@@ -1651,13 +1664,13 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 
 	private async _makeRequest(): Promise<void> {
 		const {options} = this;
-		const {headers, username, password} = options;
+		const headers = options.getInternalHeaders();
+		const {username, password} = options;
 		const cookieJar = options.cookieJar as PromiseCookieJar | undefined;
 
 		for (const key in headers) {
 			if (is.undefined(headers[key])) {
-				// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-				delete headers[key];
+				options.deleteInternalHeader(key);
 			} else if (is.null(headers[key])) {
 				throw new TypeError(`Use \`undefined\` instead of \`null\` to delete the \`${key}\` header`);
 			}
@@ -1673,20 +1686,20 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 				encodings.push('zstd');
 			}
 
-			headers['accept-encoding'] = encodings.join(', ');
+			options.setInternalHeader('accept-encoding', encodings.join(', '));
 		}
 
 		if (username || password) {
 			const credentials = Buffer.from(`${username}:${password}`).toString('base64');
-			headers.authorization = `Basic ${credentials}`;
+			options.setInternalHeader('authorization', `Basic ${credentials}`);
 		}
 
 		// Set cookies
 		if (cookieJar) {
 			const cookieString: string = await cookieJar.getCookieString(options.url!.toString());
 
 			if (is.nonEmptyString(cookieString)) {
-				headers.cookie = cookieString;
+				options.setInternalHeader('cookie', cookieString);
 			}
 		}
 
@@ -1790,7 +1803,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 		// Publish error event
 		publishError({
 			requestId: this._requestId,
-			url: this.options?.url?.toString() ?? '',
+			url: getSanitizedUrl(this.options),
 			error,
 			timings: this.timings,
 		});