diff --git a/README.md b/README.md index f16e624c..cb682c1b 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,99 @@ server.on('requestFailed', ({ request, error }) => { }); ``` +## Run a simple HTTPS proxy server + +This example demonstrates how to create an HTTPS proxy server with a self-signed certificate. The HTTPS proxy server works identically to the HTTP version but with TLS encryption. + +```javascript +const fs = require('fs'); +const path = require('path'); +const ProxyChain = require('proxy-chain'); + +(async () => { + // TODO: update these lines to use your own key and cert + const sslKey = fs.readFileSync(path.join(__dirname, 'ssl.key')); + const sslCrt = fs.readFileSync(path.join(__dirname, 'ssl.crt')); + + const server = new ProxyChain.Server({ + // Main difference between 'http' and 'https' is additional event listening: + // + // http + // -> listen for 'connection' events to track raw TCP sockets + // + // https: + // -> listen for 'securedConnection' events (instead of 'connection') to track only post-TLS-handshake sockets + // -> additionally listen for 'tlsError' events to handle TLS handshake errors + // + // Default value is 'http' + serverType: 'https', + + // Provide the TLS certificate and private key + httpsOptions: { + key: sslKey, + cert: sslCrt, + }, + + // Port where the server will listen + port: 8443, + + // Enable verbose logging to see what's happening + verbose: true, + + // Optional: Add authentication and upstream proxy configuration + prepareRequestFunction: ({ username, hostname, port }) => { + console.log(`Request to ${hostname}:${port} from user: ${username || 'anonymous'}`); + + // Allow the request + return {}; + }, + }); + + // Handle failed HTTP/HTTPS requests + server.on('requestFailed', ({ request, error }) => { + console.log(`Request ${request.url} failed`); + console.error(error); + }); + + // Handle TLS handshake errors + server.on('tlsError', ({ error, socket }) => { + console.error(`TLS error from ${socket.remoteAddress}: ${error.message}`); + }); + + // Emitted when HTTP/HTTPS connection is closed + server.on('connectionClosed', ({ connectionId, stats }) => { + console.log(`Connection ${connectionId} closed`); + console.dir(stats); + }); + + // Start the server + await server.listen(); + + // Handle graceful shutdown + process.on('SIGINT', async () => { + console.log('\nShutting down server...'); + await server.close(true); + console.log('Server closed.'); + process.exit(0); + }); + + // Keep the server running + await new Promise(() => { }); +})(); +``` + +Run server: + +```bash +node https_proxy_server.js +``` + +Send request via proxy: + +```bash +curl --proxy-insecure -x https://localhost:8443 -k https://example.com +``` + ## Use custom HTTP agents for connection pooling You can provide custom HTTP/HTTPS agents to enable connection pooling and reuse with upstream proxies. This is particularly useful for maintaining sticky IP addresses or reducing connection overhead: diff --git a/package.json b/package.json index 1d760f98..6127829d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "proxy-chain", - "version": "2.6.1", + "version": "2.7.0", "description": "Node.js implementation of a proxy server (think Squid) with support for SSL, authentication, upstream proxy chaining, and protocol tunneling.", "main": "dist/index.js", "keywords": [ @@ -37,8 +37,9 @@ "clean": "rimraf dist", "prepublishOnly": "npm run build", "local-proxy": "node ./dist/run_locally.js", - "test": "nyc cross-env NODE_OPTIONS=--insecure-http-parser mocha --bail", - "test:docker": "docker build --tag proxy-chain-tests --file test/Dockerfile . && docker run proxy-chain-tests", + "test": "nyc cross-env NODE_OPTIONS=--insecure-http-parser mocha", + "test:docker": "docker build --tag proxy-chain-tests --file test/Dockerfile . && docker run proxy-chain-tests", + "test:docker:all": "bash scripts/test-docker-all.sh", "lint": "eslint .", "lint:fix": "eslint . --fix" }, diff --git a/scripts/test-docker-all.sh b/scripts/test-docker-all.sh new file mode 100755 index 00000000..2df81ec0 --- /dev/null +++ b/scripts/test-docker-all.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +echo "Starting parallel Docker tests for Node 14, 16, and 18..." + +# Run builds in parallel, capture PIDs. +docker build --build-arg NODE_IMAGE=node:14.21.3-bullseye --tag proxy-chain-tests:node14 --file test/Dockerfile . && docker run proxy-chain-tests:node14 & +pid14=$! +docker build --build-arg NODE_IMAGE=node:16.20.2-bookworm --tag proxy-chain-tests:node16 --file test/Dockerfile . && docker run proxy-chain-tests:node16 & +pid16=$! +docker build --build-arg NODE_IMAGE=node:18.20.8-bookworm --tag proxy-chain-tests:node18 --file test/Dockerfile . && docker run proxy-chain-tests:node18 & +pid18=$! + +# Wait for all and capture exit codes. +wait $pid14 +ec14=$? +wait $pid16 +ec16=$? +wait $pid18 +ec18=$? + +echo "" +echo "========== Results ==========" +echo "Node 14: $([ $ec14 -eq 0 ] && echo 'PASS' || echo 'FAIL')" +echo "Node 16: $([ $ec16 -eq 0 ] && echo 'PASS' || echo 'FAIL')" +echo "Node 18: $([ $ec18 -eq 0 ] && echo 'PASS' || echo 'FAIL')" +echo "=============================" + +# Exit with non-zero if any failed. +exit $((ec14 + ec16 + ec18)) diff --git a/src/server.ts b/src/server.ts index d9fec52c..6295c724 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,7 +3,7 @@ import { Buffer } from 'node:buffer'; import type dns from 'node:dns'; import { EventEmitter } from 'node:events'; import http from 'node:http'; -import type https from 'node:https'; +import https from 'node:https'; import type net from 'node:net'; import { URL } from 'node:url'; import util from 'node:util'; @@ -19,7 +19,7 @@ import type { HandlerOpts as ForwardOpts } from './forward'; import { forward } from './forward'; import { forwardSocks } from './forward_socks'; import { RequestError } from './request_error'; -import type { Socket } from './socket'; +import type { Socket, TLSSocket } from './socket'; import { badGatewayStatusCodes } from './statuses'; import { getTargetStats } from './utils/count_target_bytes'; import { nodeify } from './utils/nodeify'; @@ -41,10 +41,23 @@ export const SOCKS_PROTOCOLS = ['socks:', 'socks4:', 'socks4a:', 'socks5:', 'soc const DEFAULT_AUTH_REALM = 'ProxyChain'; const DEFAULT_PROXY_SERVER_PORT = 8000; +const HTTPS_DEFAULT_OPTIONS = { + // Disable TLS 1.0 and 1.1 (deprecated, insecure). + // All other TLS settings use Node.js defaults for cipher selection (automatically updated). + minVersion: 'TLSv1.2', +} as const; + +/** + * Connection statistics for bandwidth tracking. + */ export type ConnectionStats = { + // Bytes sent by proxy to client. srcTxBytes: number; + // Bytes received by proxy from client. srcRxBytes: number; + // Bytes sent by proxy to target. trgTxBytes: number | null; + // Bytes received by proxy from target. trgRxBytes: number | null; }; @@ -96,10 +109,31 @@ export type PrepareRequestFunctionResult = { type Promisable = T | Promise; export type PrepareRequestFunction = (opts: PrepareRequestFunctionOpts) => Promisable; +type ServerOptionsBase = { + port?: number; + host?: string; + prepareRequestFunction?: PrepareRequestFunction; + verbose?: boolean; + authRealm?: unknown; +}; + +export type HttpServerOptions = ServerOptionsBase & { + serverType?: 'http'; +}; + +export type HttpsServerOptions = ServerOptionsBase & { + serverType: 'https'; + httpsOptions: https.ServerOptions; +}; + +export type ServerOptions = HttpServerOptions | HttpsServerOptions; + /** * Represents the proxy server. * It emits the 'requestFailed' event on unexpected request errors, with the following parameter `{ error, request }`. * It emits the 'connectionClosed' event when connection to proxy server is closed, with parameter `{ connectionId, stats }`. + * It emits the 'tlsError' event on TLS handshake failures (HTTPS servers only), with parameter `{ error, socket }`. + * with parameter `{ connectionId, reason, hasParent, parentType }`. */ export class Server extends EventEmitter { port: number; @@ -112,7 +146,9 @@ export class Server extends EventEmitter { verbose: boolean; - server: http.Server; + server: http.Server | https.Server; + + serverType: 'http' | 'https'; lastHandlerId: number; @@ -124,6 +160,9 @@ export class Server extends EventEmitter { * Initializes a new instance of Server class. * @param options * @param [options.port] Port where the server will listen. By default 8000. + * @param [options.serverType] Type of server to create: 'http' or 'https'. By default 'http'. + * @param [options.httpsOptions] HTTPS server options (required when serverType is 'https'). + * Accepts standard Node.js https.ServerOptions including key, cert, ca, passphrase, etc. * @param [options.prepareRequestFunction] Custom function to authenticate proxy requests, * provide URL to upstream proxy or potentially provide a function that generates a custom response to HTTP requests. * It accepts a single parameter which is an object: @@ -154,13 +193,7 @@ export class Server extends EventEmitter { * @param [options.authRealm] Realm used in the Proxy-Authenticate header and also in the 'Server' HTTP header. By default it's `ProxyChain`. * @param [options.verbose] If true, the server will output logs */ - constructor(options: { - port?: number, - host?: string, - prepareRequestFunction?: PrepareRequestFunction, - verbose?: boolean, - authRealm?: unknown, - } = {}) { + constructor(options: ServerOptions = {}) { super(); if (options.port === undefined || options.port === null) { @@ -174,11 +207,43 @@ export class Server extends EventEmitter { this.authRealm = options.authRealm || DEFAULT_AUTH_REALM; this.verbose = !!options.verbose; - this.server = http.createServer(); + // Keep legacy behavior (http) as default behavior. + this.serverType = options.serverType === 'https' ? 'https' : 'http'; + + if (options.serverType === 'https') { + if (!options.httpsOptions) { + throw new Error('httpsOptions is required when serverType is "https"'); + } + + // Apply secure TLS defaults (user options can override). + const effectiveOptions: https.ServerOptions = { + ...HTTPS_DEFAULT_OPTIONS, + honorCipherOrder: true, + ...options.httpsOptions, + }; + + this.server = https.createServer(effectiveOptions); + } else { + this.server = http.createServer(); + } + + // Attach common event handlers (same for both HTTP and HTTPS). this.server.on('clientError', this.onClientError.bind(this)); this.server.on('request', this.onRequest.bind(this)); this.server.on('connect', this.onConnect.bind(this)); - this.server.on('connection', this.onConnection.bind(this)); + + // Attach connection tracking based on server type. + // Only listen to one connection event to avoid double registration. + if (this.serverType === 'https') { + // For HTTPS: Track only post-TLS-handshake sockets (secureConnection). + // This ensures we track the TLS-wrapped socket with correct bytesRead/bytesWritten. + this.server.on('secureConnection', this.onConnection.bind(this)); + // Handle TLS handshake errors to prevent server crashes. + this.server.on('tlsClientError', this.onTLSClientError.bind(this)); + } else { + // For HTTP: Track raw TCP sockets (connection). + this.server.on('connection', this.onConnection.bind(this)); + } this.lastHandlerId = 0; this.stats = { @@ -189,6 +254,29 @@ export class Server extends EventEmitter { this.connections = new Map(); } + /** + * Handles TLS handshake errors for HTTPS servers. + * Without this handler, unhandled TLS errors can crash the server. + * Common errors: ECONNRESET, ERR_SSL_SSLV3_ALERT_CERTIFICATE_UNKNOWN, + * ERR_SSL_TLSV1_ALERT_PROTOCOL_VERSION, ERR_SSL_SSLV3_ALERT_HANDSHAKE_FAILURE + */ + onTLSClientError(err: NodeJS.ErrnoException, tlsSocket: TLSSocket): void { + const connectionId = (tlsSocket as TLSSocket).proxyChainId; + this.log(connectionId, `TLS handshake failed: ${err.message}`); + + // Emit event in first place before any return statement. + this.emit('tlsError', { error: err, socket: tlsSocket }); + + // If connection already reset or socket not writable, nothing more to do. + if (err.code === 'ECONNRESET' || !tlsSocket.writable) { + return; + } + + // TLS handshake failed before HTTP, cannot send HTTP response. + // Destroy the socket to clean up. + tlsSocket.destroy(err); + } + log(connectionId: unknown, str: string): void { if (this.verbose) { const logPrefix = connectionId != null ? `${String(connectionId)} | ` : ''; diff --git a/test/Dockerfile b/test/Dockerfile index d8aad04b..643033e1 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,4 +1,5 @@ -FROM node:18.20.8-bookworm@sha256:c6ae79e38498325db67193d391e6ec1d224d96c693a8a4d943498556716d3783 +ARG NODE_IMAGE=node:18.20.8-bookworm +FROM ${NODE_IMAGE} RUN apt-get update && apt-get install -y --no-install-recommends chromium \ && rm -rf /var/lib/apt/lists/* diff --git a/test/https-server.js b/test/https-server.js new file mode 100644 index 00000000..e9b5ffd6 --- /dev/null +++ b/test/https-server.js @@ -0,0 +1,339 @@ +const fs = require('fs'); +const path = require('path'); +const tls = require('tls'); +const { expect } = require('chai'); +const http = require('http'); +const { Server } = require('../src/index'); + +const sslKey = fs.readFileSync(path.join(__dirname, 'ssl.key')); +const sslCrt = fs.readFileSync(path.join(__dirname, 'ssl.crt')); + +const wait = (timeout) => new Promise((resolve) => setTimeout(resolve, timeout)); + +it('handles TLS handshake failures gracefully and continues accepting connections', async function () { + this.timeout(10000); + + const tlsErrors = []; + + let server = new Server({ + port: 0, + serverType: 'https', + httpsOptions: { + key: sslKey, + cert: sslCrt, + }, + }); + + server.on('tlsError', ({ error }) => { + tlsErrors.push(error); + }); + + await server.listen(); + const serverPort = server.port; + + // Make invalid TLS connection. + const badSocket = tls.connect({ + port: serverPort, + host: '127.0.0.1', + rejectUnauthorized: false, + minVersion: 'TLSv1', + maxVersion: 'TLSv1', + }); + + const badSocketErrorOccurred = await new Promise((resolve, reject) => { + let errorOccurred = false; + + badSocket.on('error', () => { + errorOccurred = true; + // Expected: TLS handshake will fail due to version mismatch. + }); + + badSocket.on('close', () => { + resolve(errorOccurred); + }); + + badSocket.setTimeout(5000, () => { + badSocket.destroy(); + reject(new Error('Bad socket timed out before error')); + }); + + }); + + await wait(100); + + expect(badSocketErrorOccurred).to.equal(true); + + // Make a valid TLS connection to prove server still works. + const goodSocket = tls.connect({ + port: serverPort, + host: '127.0.0.1', + rejectUnauthorized: false, + }); + + // Wait for secure connection. + const goodSocketConnected = await new Promise((resolve, reject) => { + let isConnected = false; + + const timeout = setTimeout(() => { + goodSocket.destroy(); + reject(new Error('Good socket connection timed out')); + }, 5000); + + goodSocket.on('error', (err) => { + clearTimeout(timeout); + goodSocket.destroy(); + reject(err); + }); + + goodSocket.on('secureConnect', () => { + isConnected = true; + clearTimeout(timeout); + resolve(isConnected); + }); + + goodSocket.on('close', () => { + clearTimeout(timeout); + }); + }); + + expect(goodSocketConnected).to.equal(true, 'Good socket should have connected'); + + // Write the CONNECT request. + goodSocket.write('CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n'); + + const response = await new Promise((resolve, reject) => { + const goodSocketTimeout = setTimeout(() => { + goodSocket.destroy(); + reject(new Error('Good socket connection timed out')); + }, 5000); + + goodSocket.on('error', (err) => { + clearTimeout(goodSocketTimeout); + goodSocket.destroy(); + reject(err); + }); + + goodSocket.on('data', (data) => { + clearTimeout(goodSocketTimeout); + goodSocket.destroy(); + resolve(data.toString()); + }); + + goodSocket.on('close', () => { + clearTimeout(goodSocketTimeout); + }); + }); + + await wait(100); + + expect(response).to.be.equal('HTTP/1.1 200 Connection Established\r\n\r\n'); + + expect(tlsErrors.length).to.be.equal(1); + expect(tlsErrors[0].library).to.be.equal('SSL routines'); + expect(tlsErrors[0].reason).to.be.equal('unsupported protocol'); + expect(tlsErrors[0].code).to.be.equal('ERR_SSL_UNSUPPORTED_PROTOCOL'); + + // Cleanup. + server.close(true); + server = null; +}); + +describe('HTTPS proxy server resource cleanup', () => { + let server; + + beforeEach(async () => { + server = new Server({ + port: 0, + serverType: 'https', + httpsOptions: { + key: sslKey, + cert: sslCrt, + }, + }); + await server.listen(); + }); + + afterEach(async () => { + if (server) { + await server.close(true); + server = null; + } + }); + + it('cleans up connections when client disconnects abruptly', async function () { + this.timeout(5000); + + const closedConnections = []; + server.on('connectionClosed', ({ connectionId }) => { + closedConnections.push(connectionId); + }); + + const socket = tls.connect({ + port: server.port, + host: '127.0.0.1', + rejectUnauthorized: false, + }); + + await new Promise((resolve) => socket.on('secureConnect', resolve)); + + // Small delay to ensure server-side connection registration completes. + await wait(100); + + const connectionsBefore = server.getConnectionIds().length; + expect(connectionsBefore).to.equal(1); + + // Abruptly destroy the connection (simulating client crash). + socket.destroy(); + + await new Promise((resolve) => socket.on('close', resolve)); + await wait(100); + + expect(server.getConnectionIds()).to.be.empty; + expect(closedConnections.length).to.equal(1); + }); + + it('cleans up when client closes immediately after CONNECT 200', async function () { + this.timeout(5000); + + const closedConnections = []; + server.on('connectionClosed', ({ connectionId, stats }) => { + closedConnections.push({ connectionId, stats }); + }); + + const socket = tls.connect({ + port: server.port, + host: '127.0.0.1', + rejectUnauthorized: false, + }); + + await new Promise((resolve) => socket.on('secureConnect', resolve)); + + socket.write('CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n'); + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Timeout waiting for CONNECT response')), 3000); + + socket.on('data', (data) => { + if (data.toString().includes('200')) { + clearTimeout(timeout); + socket.destroy(); // Abrupt close. + resolve(); + } + }); + + socket.on('error', () => {}); + }); + + await new Promise((resolve) => socket.on('close', resolve)); + await wait(500); + + expect(server.getConnectionIds()).to.be.empty; + expect(closedConnections.length).to.equal(1); + }); + + it('handles multiple HTTP requests over single TLS connection (keep-alive)', async function () { + this.timeout(10000); + + const targetServer = http.createServer((_, res) => { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('Hello world!'); + }); + + await new Promise((resolve) => targetServer.listen(0, resolve)); + const targetServerPort = targetServer.address().port; + + try { + const socket = tls.connect({ + port: server.port, + host: '127.0.0.1', + rejectUnauthorized: false, + }); + + await new Promise((resolve) => socket.on('secureConnect', resolve)); + + const responses = []; + + for (let i = 0; i < 3; i++) { + socket.write( + `GET http://127.0.0.1:${targetServerPort}/hello-world HTTP/1.1\r\n` + + `Host: 127.0.0.1\r\n` + + `Connection: keep-alive\r\n\r\n` + ); + + const response = await new Promise((resolve) => { + let data = ''; + const onData = (chunk) => { + data += chunk.toString(); + if (data.includes('Hello world')) { + socket.removeListener('data', onData); + resolve(data); + } + }; + socket.on('data', onData); + }); + + responses.push(response); + + // Verify keep-alive: socket still alive, exactly one connection. + expect(socket.destroyed).to.equal(false); + expect(server.getConnectionIds().length).to.equal(1); + } + + socket.destroy(); + + // Wait a bit for socket cleanup. + await wait(100); + + expect(server.getConnectionIds().length).to.equal(0); + + expect(responses.length).to.equal(3); + responses.forEach((r) => { + expect(r).to.include('200 OK'); + expect(r).to.include('Hello world'); + }); + } finally { + await new Promise((resolve) => targetServer.close(resolve)); + } + }); + + it('handles multiple sequential TLS failures without leaking connections', async function () { + this.timeout(10000); + + const tlsErrors = []; + server.on('tlsError', ({ error }) => tlsErrors.push(error)); + + // 10 sequential failures (sanity check). + for (let i = 0; i < 10; i++) { + const badSocket = tls.connect({ + port: server.port, + host: '127.0.0.1', + minVersion: 'TLSv1', + maxVersion: 'TLSv1', + }); + + await new Promise((resolve) => { + badSocket.on('error', () => {}); + badSocket.on('close', resolve); + }); + } + + await wait(200); + + expect(tlsErrors.length).to.equal(10); + expect(server.getConnectionIds()).to.be.empty; + + // Verify server still works. + const goodSocket = tls.connect({ + port: server.port, + host: '127.0.0.1', + rejectUnauthorized: false, + }); + + await new Promise((resolve, reject) => { + goodSocket.on('secureConnect', resolve); + goodSocket.on('error', reject); + }); + + goodSocket.destroy(); + }); +}); diff --git a/test/https-stress-test.js b/test/https-stress-test.js new file mode 100644 index 00000000..63b1715d --- /dev/null +++ b/test/https-stress-test.js @@ -0,0 +1,161 @@ +const fs = require('fs'); +const path = require('path'); +const tls = require('tls'); +const util = require('util'); +const request = require('request'); +const { expect } = require('chai'); +const { Server } = require('../src/index'); +const { TargetServer } = require('./utils/target_server'); + +const sslKey = fs.readFileSync(path.join(__dirname, 'ssl.key')); +const sslCrt = fs.readFileSync(path.join(__dirname, 'ssl.crt')); + +const requestPromised = util.promisify(request); + +describe('HTTPS proxy stress testing', function () { + this.timeout(60000); + + let server; + let targetServer; + let targetServerPort; + + before(async () => { + targetServer = new TargetServer({ port: 0, useSsl: false }); + await targetServer.listen(); + targetServerPort = targetServer.httpServer.address().port; + }); + + after(async () => { + if (targetServer) await targetServer.close(); + }); + + beforeEach(async () => { + server = new Server({ + port: 0, + serverType: 'https', + httpsOptions: { key: sslKey, cert: sslCrt }, + }); + await server.listen(); + }); + + afterEach(async () => { + if (server) await server.close(true); + }); + + it('handles 100 concurrent HTTP requests with correct responses', async () => { + const REQUESTS = 100; + const results = []; + + const promises = []; + for (let i = 0; i < REQUESTS; i++) { + promises.push( + requestPromised({ + url: `http://127.0.0.1:${targetServerPort}/hello-world`, + proxy: `https://127.0.0.1:${server.port}`, + strictSSL: false, + }).then((response) => { + results.push({ + status: response.statusCode, + body: response.body, + }); + }).catch((err) => { + results.push({ error: err.message }); + }) + ); + } + + await Promise.all(promises); + + const successful = results.filter((r) => r.status === 200 && r.body === 'Hello world!'); + expect(successful.length).to.equal(REQUESTS); + }); + + // Not specific for https but still worth to have. + it('handles 100 concurrent CONNECT tunnels with data verification', async () => { + const TUNNEL_COUNT = 100; + const results = []; + + const promises = []; + for (let i = 0; i < TUNNEL_COUNT; i++) { + promises.push(new Promise((resolve) => { + const socket = tls.connect({ + port: server.port, + host: '127.0.0.1', + rejectUnauthorized: false, + }); + + let requestSent = false; + + socket.on('secureConnect', () => { + socket.write(`CONNECT 127.0.0.1:${targetServerPort} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n`); + }); + + let data = ''; + socket.on('data', (chunk) => { + data += chunk.toString(); + + if (data.includes('200 Connection Established') && !requestSent) { + requestSent = true; + socket.write('GET /hello-world HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n'); + } + + if (data.includes('Hello world')) { + socket.destroy(); + results.push({ success: true }); + resolve(); + } + }); + + socket.on('error', (err) => { + results.push({ error: err.message }); + resolve(); + }); + + setTimeout(() => { + socket.destroy(); + if (!results.some((r) => r.success || r.error)) { + results.push({ error: 'timeout' }); + } + resolve(); + }, 10000); + })); + } + + await Promise.all(promises); + + const successful = results.filter((r) => r.success); + expect(successful.length).to.equal(TUNNEL_COUNT); + }); + + it('tracks accurate statistics for 100 concurrent requests', async () => { + const REQUESTS = 100; + const allStats = []; + + server.on('connectionClosed', ({ stats }) => { + allStats.push(stats); + }); + + const promises = []; + for (let i = 0; i < REQUESTS; i++) { + promises.push( + requestPromised({ + url: `http://127.0.0.1:${targetServerPort}/hello-world`, + proxy: `https://127.0.0.1:${server.port}`, + strictSSL: false, + }) + ); + } + + await Promise.all(promises); + await new Promise((r) => setTimeout(r, 500)); + + expect(allStats.length).to.equal(REQUESTS); + + allStats.forEach((stats) => { + // These are application-layer bytes only (no TLS overhead). + // srcRxBytes > trgTxBytes because hop-by-hop headers (e.g., Proxy-Connection) + // are stripped when forwarding the request to target. + expect(stats).to.be.deep.equal({ srcTxBytes: 174, srcRxBytes: 93, trgTxBytes: 71, trgRxBytes: 174 }); + }); + }); +}); diff --git a/test/server.js b/test/server.js index 5f884f3a..8089dd08 100644 --- a/test/server.js +++ b/test/server.js @@ -3,6 +3,7 @@ const zlib = require('zlib'); const path = require('path'); const stream = require('stream'); const childProcess = require('child_process'); +const tls = require('tls'); const net = require('net'); const dns = require('dns'); const util = require('util'); @@ -75,14 +76,32 @@ const puppeteerGet = (url, proxyUrl) => { return (async () => { const parsed = proxyUrl ? new URL(proxyUrl) : undefined; - const browser = await puppeteer.launch({ - env: parsed ? { - HTTP_PROXY: parsed.origin, - } : {}, + const args = [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage' + ]; + + const launchOpts = { ignoreHTTPSErrors: true, - headless: "new", - args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'] - }); + headless: 'new', + args + }; + + if (parsed) { + if (parsed.protocol === 'https:') { + args.push(`--proxy-server=${parsed.origin}`); + // For HTTPS proxies with self-signed certificates, + // ignore certificate errors on the proxy connection itself. + args.push('--ignore-certificate-errors'); + } else { + launchOpts.env = { + HTTP_PROXY: parsed.origin, + }; + } + } + + const browser = await puppeteer.launch(launchOpts); try { const page = await browser.newPage(); @@ -110,8 +129,13 @@ const puppeteerGet = (url, proxyUrl) => { // This is a regression test for that situation const curlGet = (url, proxyUrl, returnResponse) => { let cmd = 'curl --insecure '; // ignore SSL errors - if (proxyUrl) cmd += `-x ${proxyUrl} `; // use proxy - if (returnResponse) cmd += `--silent --output - ${url}`; // print response to stdout + if (proxyUrl) { + if (proxyUrl.startsWith('https://')) { + cmd += '--proxy-insecure '; + } + cmd += `-x ${proxyUrl} `; // use proxy + } + if (returnResponse) cmd += `--silent --show-error --output - ${url}`; // print response to stdout else cmd += `${url}`; // console.log(`curlGet(): ${cmd}`); @@ -129,7 +153,7 @@ const curlGet = (url, proxyUrl, returnResponse) => { * @return {function(...[*]=)} */ const createTestSuite = ({ - useSsl, useMainProxy, mainProxyAuth, useUpstreamProxy, upstreamProxyAuth, testCustomResponse, + useSsl, useMainProxy, mainProxyAuth, mainProxyServerType, useUpstreamProxy, upstreamProxyAuth, testCustomResponse, }) => { return function () { this.timeout(30 * 1000); @@ -162,13 +186,21 @@ const createTestSuite = ({ let baseUrl; let mainProxyUrl; const getRequestOpts = (pathOrUrl) => { - return { + const opts = { url: pathOrUrl[0] === '/' ? `${baseUrl}${pathOrUrl}` : pathOrUrl, key: sslKey, proxy: mainProxyUrl, headers: {}, timeout: 30000, }; + + // Accept self-signed certificates when connecting to HTTPS proxy. + if (mainProxyServerType === 'https') { + opts.strictSSL = false; + opts.rejectUnauthorized = false; + } + + return opts; }; let counter = 0; @@ -411,6 +443,15 @@ const createTestSuite = ({ opts.authRealm = AUTH_REALM; + // Configure HTTPS proxy server if requested. + if (mainProxyServerType === 'https') { + opts.serverType = 'https'; + opts.httpsOptions = { + key: sslKey, + cert: sslCrt, + }; + } + mainProxyServer = new Server(opts); mainProxyServer.on('connectionClosed', ({ connectionId, stats }) => { @@ -437,7 +478,8 @@ const createTestSuite = ({ if (useMainProxy) { let auth = ''; if (mainProxyAuth) auth = `${mainProxyAuth.username}:${mainProxyAuth.password}@`; - mainProxyUrl = `http://${auth}127.0.0.1:${mainProxyServerPort}`; + const proxySchema = mainProxyServerType === 'https' ? 'https' : 'http'; + mainProxyUrl = `${proxySchema}://${auth}127.0.0.1:${mainProxyServerPort}`; } }); }); @@ -520,9 +562,10 @@ const createTestSuite = ({ upstreamProxyHostname = '127.0.0.1'; } }); - } else if (useMainProxy && process.versions.node.split('.')[0] >= 15) { + } else if (useMainProxy && process.versions.node.split('.')[0] >= 15 && mainProxyServerType !== 'https') { // Version check is required because HTTP/2 negotiation // is not supported on Node.js < 15. + // Note: Skipped for HTTPS proxy - got-scraping has issues with IPv6 + HTTPS proxy combination. _it('direct ipv6', async () => { const opts = getRequestOpts('/hello-world'); @@ -545,9 +588,10 @@ const createTestSuite = ({ expect(response.body).to.eql('Hello world!'); expect(response.statusCode).to.eql(200); }); - } else if (!useSsl && process.versions.node.split('.')[0] >= 15) { + } else if (!useSsl && process.versions.node.split('.')[0] >= 15 && mainProxyServerType !== 'https') { // Version check is required because HTTP/2 negotiation // is not supported on Node.js < 15. + // Note: Skipped for HTTPS proxy - got-scraping has issues with IPv6 + HTTPS proxy combination. _it('forward ipv6', async () => { const opts = getRequestOpts('/hello-world'); @@ -666,6 +710,7 @@ const createTestSuite = ({ }); }); + // TODO: investigate https case. if (!useSsl) { _it('handles double Host header', () => { // This is a regression test, duplication of Host headers caused the proxy to throw @@ -691,10 +736,21 @@ const createTestSuite = ({ + 'Host: dummy2.example.com\r\n\r\n'; } - const client = net.createConnection({ port }, () => { - // console.log('connected to server! sending msg: ' + httpMsg); - client.write(httpMsg); - }); + let client; + if (mainProxyServerType === 'https') { + client = tls.connect({ + port, + host: 'localhost', + rejectUnauthorized: false, + }, () => { + client.write(httpMsg); + }); + } else { + client = net.createConnection({ port }, () => { + client.write(httpMsg); + }); + } + client.on('data', (data) => { // console.log('received data: ' + data.toString()); try { @@ -826,7 +882,11 @@ const createTestSuite = ({ }); }); - if (!mainProxyAuth || (mainProxyAuth.username && mainProxyAuth.password)) { + // Skip on Node 14: HTTPS proxy with upstream proxy causes EPIPE errors. + const isNode14 = process.versions.node.split('.')[0] === '14'; + const skipPuppeteerOnNode14 = isNode14 && mainProxyServerType === 'https' && useUpstreamProxy && !mainProxyAuth; + + if ((!mainProxyAuth || (mainProxyAuth.username && mainProxyAuth.password)) && !skipPuppeteerOnNode14) { it('handles GET request using puppeteer', async () => { const phantomUrl = `${useSsl ? 'https' : 'http'}://${LOCALHOST_TEST}:${targetServerPort}/hello-world`; const response = await puppeteerGet(phantomUrl, mainProxyUrl); @@ -837,7 +897,8 @@ const createTestSuite = ({ if (!useSsl && mainProxyAuth && mainProxyAuth.username && mainProxyAuth.password) { it('handles GET request using puppeteer with invalid credentials', async () => { const phantomUrl = `${useSsl ? 'https' : 'http'}://${LOCALHOST_TEST}:${targetServerPort}/hello-world`; - const response = await puppeteerGet(phantomUrl, `http://bad:password@127.0.0.1:${mainProxyServerPort}`); + const proxySchema = mainProxyServerType === 'https' ? 'https' : 'http'; + const response = await puppeteerGet(phantomUrl, `${proxySchema}://bad:password@127.0.0.1:${mainProxyServerPort}`); expect(response).to.contain('Proxy credentials required'); }); } @@ -855,8 +916,9 @@ const createTestSuite = ({ if (mainProxyAuth && mainProxyAuth.username) { it('handles GET request from curl with invalid credentials', async () => { const curlUrl = `${useSsl ? 'https' : 'http'}://${LOCALHOST_TEST}:${targetServerPort}/hello-world`; + const proxySchema = mainProxyServerType === 'https' ? 'https' : 'http'; // For SSL, we need to return curl's stderr to check what kind of error was there - const output = await curlGet(curlUrl, `http://bad:password@127.0.0.1:${mainProxyServerPort}`, !useSsl); + const output = await curlGet(curlUrl, `${proxySchema}://bad:password@127.0.0.1:${mainProxyServerPort}`, !useSsl); if (useSsl) { expect(output).to.contain.oneOf([ // Old error message before dafdb20a26d0c890e83dea61a104b75408481ebd @@ -927,12 +989,15 @@ const createTestSuite = ({ } it('handles invalid CONNECT path', (done) => { - const req = http.request(mainProxyUrl, { + const requestModule = mainProxyServerType === 'https' ? https : http; + const req = requestModule.request(mainProxyUrl, { method: 'CONNECT', path: ':443', headers: { host: ':443', }, + // Accept self-signed certificates for HTTPS proxy. + rejectUnauthorized: false, }); req.once('connect', (response, socket, head) => { expect(response.statusCode).to.equal(400); @@ -985,14 +1050,26 @@ const createTestSuite = ({ }); server.listen(0, () => { - const req = http.request(mainProxyUrl, { + const proxyUrl = new URL(mainProxyUrl); + const requestModule = proxyUrl.protocol === 'https:' ? https : http; + + const requestOpts = { + hostname: proxyUrl.hostname, + port: proxyUrl.port, method: 'CONNECT', path: `127.0.0.1:${server.address().port}`, headers: { host: `127.0.0.1:${server.address().port}`, 'proxy-authorization': `Basic ${Buffer.from('nopassword').toString('base64')}`, }, - }); + }; + + // Accept self-signed certificates for HTTPS prpxy. + if (proxyUrl.protocol === 'https:') { + requestOpts.rejectUnauthorized = false; + } + + const req = requestModule.request(requestOpts); req.once('connect', (response, socket, head) => { expect(response.statusCode).to.equal(200); expect(head.length).to.equal(0); @@ -1008,29 +1085,31 @@ const createTestSuite = ({ }); it('returns 407 for invalid credentials', () => { + const proxySchema = mainProxyServerType === 'https' ? 'https' : 'http'; + return Promise.resolve() .then(() => { // Test no username and password const opts = getRequestOpts('/whatever'); - opts.proxy = `http://127.0.0.1:${mainProxyServerPort}`; + opts.proxy = `${proxySchema}://127.0.0.1:${mainProxyServerPort}`; return testForErrorResponse(opts, 407); }) .then(() => { // Test good username and invalid password const opts = getRequestOpts('/whatever'); - opts.proxy = `http://${mainProxyAuth.username}:bad-password@127.0.0.1:${mainProxyServerPort}`; + opts.proxy = `${proxySchema}://${mainProxyAuth.username}:bad-password@127.0.0.1:${mainProxyServerPort}`; return testForErrorResponse(opts, 407); }) .then(() => { // Test invalid username and good password const opts = getRequestOpts('/whatever'); - opts.proxy = `http://bad-username:${mainProxyAuth.password}@127.0.0.1:${mainProxyServerPort}`; + opts.proxy = `${proxySchema}://bad-username:${mainProxyAuth.password}@127.0.0.1:${mainProxyServerPort}`; return testForErrorResponse(opts, 407); }) .then(() => { - // Test invalid username and good password + // Test invalid username and bad password const opts = getRequestOpts('/whatever'); - opts.proxy = `http://bad-username:bad-password@127.0.0.1:${mainProxyServerPort}`; + opts.proxy = `${proxySchema}://bad-username:bad-password@127.0.0.1:${mainProxyServerPort}`; return testForErrorResponse(opts, 407); }) .then((response) => { @@ -1579,6 +1658,11 @@ describe('supports ignoreUpstreamProxyCertificate', () => { }); // Run all combinations of test parameters +const mainProxyServerTypeVariants = [ + 'http', + 'https', +]; + const useSslVariants = [ false, true, @@ -1601,48 +1685,53 @@ const upstreamProxyAuthVariants = [ { type: 'Basic', username: 'us%erB', password: 'p$as%sA' }, ]; -useSslVariants.forEach((useSsl) => { - mainProxyAuthVariants.forEach((mainProxyAuth) => { - const baseDesc = `Server (${useSsl ? 'HTTPS' : 'HTTP'} -> Main proxy`; - - // Test custom response separately (it doesn't use upstream proxies) - describe(`${baseDesc} -> Target + custom responses)`, createTestSuite({ - useMainProxy: true, - useSsl, - mainProxyAuth, - testCustomResponse: true, - })); - - useUpstreamProxyVariants.forEach((useUpstreamProxy) => { - // If useUpstreamProxy is not used, only try one variant of upstreamProxyAuth - let variants = upstreamProxyAuthVariants; - if (!useUpstreamProxy) variants = [null]; - - variants.forEach((upstreamProxyAuth) => { - let desc = `${baseDesc} `; - - if (mainProxyAuth) { - if (!mainProxyAuth) desc += 'public '; - else if (mainProxyAuth.username && mainProxyAuth.password) desc += 'with username:password '; - else if (mainProxyAuth.username) desc += 'with username only '; - else desc += 'with password only '; - } - if (useUpstreamProxy) { - desc += '-> Upstream proxy '; - if (!upstreamProxyAuth) desc += 'public '; - else if (upstreamProxyAuth.username && upstreamProxyAuth.password) desc += 'with username:password '; - else if (upstreamProxyAuth.username) desc += 'with username only '; - else desc += 'with password only '; - } - desc += '-> Target)'; - - describe(desc, createTestSuite({ - useMainProxy: true, - useSsl, - useUpstreamProxy, - mainProxyAuth, - upstreamProxyAuth, - })); +mainProxyServerTypeVariants.forEach((mainProxyServerType) => { + useSslVariants.forEach((useSsl) => { + mainProxyAuthVariants.forEach((mainProxyAuth) => { + const proxyTypeLabel = mainProxyServerType === 'https' ? 'HTTPS' : 'HTTP'; + const baseDesc = `Server (${useSsl ? 'HTTPS' : 'HTTP'} -> ${proxyTypeLabel} Main proxy`; + + // Test custom response separately (it doesn't use upstream proxies) + describe(`${baseDesc} -> Target + custom responses)`, createTestSuite({ + useMainProxy: true, + useSsl, + mainProxyAuth, + mainProxyServerType, + testCustomResponse: true, + })); + + useUpstreamProxyVariants.forEach((useUpstreamProxy) => { + // If useUpstreamProxy is not used, only try one variant of upstreamProxyAuth + let variants = upstreamProxyAuthVariants; + if (!useUpstreamProxy) variants = [null]; + + variants.forEach((upstreamProxyAuth) => { + let desc = `${baseDesc} `; + + if (mainProxyAuth) { + if (!mainProxyAuth) desc += 'public '; + else if (mainProxyAuth.username && mainProxyAuth.password) desc += 'with username:password '; + else if (mainProxyAuth.username) desc += 'with username only '; + else desc += 'with password only '; + } + if (useUpstreamProxy) { + desc += '-> Upstream proxy '; + if (!upstreamProxyAuth) desc += 'public '; + else if (upstreamProxyAuth.username && upstreamProxyAuth.password) desc += 'with username:password '; + else if (upstreamProxyAuth.username) desc += 'with username only '; + else desc += 'with password only '; + } + desc += '-> Target)'; + + describe(desc, createTestSuite({ + useMainProxy: true, + useSsl, + useUpstreamProxy, + mainProxyAuth, + mainProxyServerType, + upstreamProxyAuth, + })); + }); }); }); }); @@ -1707,3 +1796,43 @@ describe('Socket error handler regression test', () => { }); }); }); + +describe('Server constructor', () => { + it('should default to "http" when serverType is not specified', async () => { + const server = new Server({ port: 0 }); + await server.listen(); + expect(server.serverType).to.equal('http'); + expect(server.server).to.be.instanceOf(http.Server); + await server.close(true); + }); + + it('should use "http" when explicitly specified', async () => { + const server = new Server({ port: 0, serverType: 'http' }); + await server.listen(); + expect(server.serverType).to.equal('http'); + expect(server.server).to.be.instanceOf(http.Server); + await server.close(true); + }); + + it('should use "https" when explicitly specified with httpsOptions', async () => { + const server = new Server({ + port: 0, + serverType: 'https', + httpsOptions: { key: sslKey, cert: sslCrt } + }); + await server.listen(); + expect(server.serverType).to.equal('https'); + expect(server.server).to.be.instanceOf(https.Server); + await server.close(true); + }); + + it('requires httpsOptions when serverType is "https"', () => { + expect(() => { + new Server({ + port: 0, + serverType: 'https', + }); + }).to.throw('httpsOptions is required when serverType is "https"'); + }); +}); +