Outbound proxy ?

[dani]

I'm trying to run whishlist in an env where using an outbound proxy is mandatory for external web access. It seems setting the (almost) standard http_proxy / https_proxy env vars is not working. Anyone knows how this can be achieved ?

[cmintey]

There has not been any explicit support for those env vars added. What issues are you seeing caused by the lack of support?

[dani]

to be more precise, the issue is when I hit the /api/product?url=xxxx, I get a 500 err after a while, with this error in the logs

2025-11-08 21:33:24.932 {"level":"ERROR","time":"2025-11-08T21:33:24.930Z","status":500,"method":"GET","uri":"/api/product?url=https%3A%2F%2Fskullshaver.fr%2Fcollections%2Fmen-s-shavers%2Fproducts%2Fskullshaver-silver-pro-sx5","err":{"code":"ETIMEDOUT","event":"request","message":"Timeout awaiting 'request' for 60000ms: Timeout awaiting 'request' for 60000ms","name":"TimeoutError","options":{"agent":{"http":{"agent":{"_events":"[Object]","_eventsCount":2,"defaultPort":80,"freeSockets":{},"keepAlive":true,"keepAliveMsecs":1000,"maxFreeSockets":256,"maxSockets":null,"maxTotalSockets":null,"options":"[Object]","protocol":"http:","requests":{},"scheduling":"lifo","sockets":{},"totalSocketCount":0}},"https":{"agent":{"_events":"[Object]","_eventsCount":2,"_sessionCache":"[Object]","defaultPort":443,"freeSockets":{},"keepAlive":true,"keepAliveMsecs":1000,"maxCachedSessions":100,"maxFreeSockets":256,"maxSockets":null,"maxTotalSockets":null,"options":"[Object]","protocol":"https:","requests":{},"scheduling":"lifo","sockets":"[Object]","totalSocketCount":1}}},"allowGetBody":false,"cacheOptions":{},"context":{"headerGenerator":{"globalOptions":{"browserListQuery":"","browsers":"[Array]","devices":"[Array]","httpVersion":"2","locales":"[Array]","operatingSystems":"[Array]","strict":false},"headerGeneratorNetwork":{"nodesByName":"[Object]","nodesInSamplingOrder":"[Array]"},"headersOrder":{"chrome":"[Array]","edge":"[Array]","firefox":"[Array]","safari":"[Array]"},"inputGeneratorNetwork":{"nodesByName":"[Object]","nodesInSamplingOrder":"[Array]"},"relaxationOrder":["locales","devices","operatingSystems","browsers","browserListQuery"],"uniqueBrowsers":["[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","... 12 items not stringified"]},"headerGeneratorOptions":{"devices":["desktop"],"locales":["fr-FR","fr"]},"insecureHTTPParser":true,"useHeaderGenerator":true},"decompress":true,"enableUnixSockets":false,"followRedirect":true,"headers":{"accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7","accept-encoding":"gzip, deflate, br","accept-language":"fr-FR,fr;q=0.9","connection":"keep-alive","sec-ch-ua":"\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"138\", \"Google Chrome\";v=\"138\"","sec-ch-ua-mobile":"?0","sec-ch-ua-platform":"\"macOS\"","sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"same-site","sec-fetch-user":"?1","upgrade-insecure-requests":"1","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"},"hooks":{"afterResponse":[],"beforeError":[],"beforeRedirect":[null],"beforeRequest":[null,null,null,null,null,null],"beforeRetry":[],"init":[null,null]},"http2":true,"https":{"ciphers":"TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA","ecdhCurve":"X25519:prime256v1:secp384r1","maxVersion":"TLSv1.3","minVersion":"TLSv1","rejectUnauthorized":false,"signatureAlgorithms":"ecdsa_secp256r1_sha256:rsa_pss_rsae_sha256:rsa_pkcs1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha384:rsa_pkcs1_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha512"},"ignoreInvalidCookies":false,"isStream":false,"maxRedirects":10,"method":"GET","methodRewriting":false,"pagination":{"backoff":0,"countLimit":null,"requestLimit":10000,"stackAllItems":false},"password":"","prefixUrl":"","resolveBodyOnly":false,"responseType":"text","retry":{"backoffLimit":null,"errorCodes":["ETIMEDOUT","ECONNRESET","EADDRINUSE","ECONNREFUSED","EPIPE","ENOTFOUND","ENETUNREACH","EAI_AGAIN"],"limit":0,"methods":["GET","PUT","HEAD","DELETE","OPTIONS","TRACE"],"noise":100,"statusCodes":[408,413,429,500,502,503,504,521,522,524]},"setHost":true,"throwHttpErrors":false,"timeout":{"request":60000},"url":"https://skullshaver.fr/collections/men-s-shavers/products/skullshaver-silver-pro-sx5","username":""},"stack":"RequestError: Timeout awaiting 'request' for 60000ms\n    at ClientRequest.<anonymous> (file:///app/node_modules/.pnpm/got@14.4.7/node_modules/got/dist/source/core/index.js:673:61)\n    at Object.onceWrapper (node:events:633:26)\n    at ClientRequest.emit (node:events:530:35)\n    at request.emit (file:///app/node_modules/.pnpm/got-scraping@4.1.2/node_modules/got-scraping/dist/index.js:695:14)\n    at emitErrorEvent (node:_http_client:104:11)\n    at TLSSocket.socketErrorListener (node:_http_client:518:5)\n    at TLSSocket.emit (node:events:518:28)\n    at emitErrorNT (node:internal/streams/destroy:170:8)\n    at emitErrorCloseNT (node:internal/streams/destroy:129:3)\n    at process.processTicksAndRejections (node:internal/process/task_queues:90:21)\n    at Timeout._onTimeout (file:///app/node_modules/.pnpm/got@14.4.7/node_modules/got/dist/source/core/timed-out.js:39:33)\n    at listOnTimeout (node:internal/timers:588:17)\n    at process.processTimers (node:internal/timers:523:7)\ncaused by: TimeoutError: Timeout awaiting 'request' for 60000ms\n    at Timeout._onTimeout (file:///app/node_modules/.pnpm/got@14.4.7/node_modules/got/dist/source/core/timed-out.js:39:33)\n    at listOnTimeout (node:internal/timers:588:17)\n    at process.processTimers (node:internal/timers:523:7)","timings":{"error":1762637604927,"lookup":1762637544903,"phases":{"dns":1,"total":60028,"wait":3},"socket":1762637544902,"start":1762637544899},"type":"TimeoutError"},"msg":"Internal Error"}
2025-11-08 21:37:21.483 {"level":"ERROR","time":"2025-11-08T21:37:21.482Z","status":500,"method":"GET","uri":"/api/product?url=https%3A%2F%2Fskullshaver.fr%2Fcollections%2Fmen-s-shavers%2Fproducts%2Fskullshaver-silver-pro-sx5","err":{"code":"ETIMEDOUT","event":"request","message":"Timeout awaiting 'request' for 60000ms: Timeout awaiting 'request' for 60000ms","name":"TimeoutError","options":{"agent":{"http":{"agent":{"_events":"[Object]","_eventsCount":2,"defaultPort":80,"freeSockets":{},"keepAlive":true,"keepAliveMsecs":1000,"maxFreeSockets":256,"maxSockets":null,"maxTotalSockets":null,"options":"[Object]","protocol":"http:","requests":{},"scheduling":"lifo","sockets":{},"totalSocketCount":0}},"https":{"agent":{"_events":"[Object]","_eventsCount":2,"_sessionCache":"[Object]","defaultPort":443,"freeSockets":{},"keepAlive":true,"keepAliveMsecs":1000,"maxCachedSessions":100,"maxFreeSockets":256,"maxSockets":null,"maxTotalSockets":null,"options":"[Object]","protocol":"https:","requests":{},"scheduling":"lifo","sockets":"[Object]","totalSocketCount":1}}},"allowGetBody":false,"cacheOptions":{},"context":{"headerGenerator":{"globalOptions":{"browserListQuery":"","browsers":"[Array]","devices":"[Array]","httpVersion":"2","locales":"[Array]","operatingSystems":"[Array]","strict":false},"headerGeneratorNetwork":{"nodesByName":"[Object]","nodesInSamplingOrder":"[Array]"},"headersOrder":{"chrome":"[Array]","edge":"[Array]","firefox":"[Array]","safari":"[Array]"},"inputGeneratorNetwork":{"nodesByName":"[Object]","nodesInSamplingOrder":"[Array]"},"relaxationOrder":["locales","devices","operatingSystems","browsers","browserListQuery"],"uniqueBrowsers":["[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","[Object]","... 12 items not stringified"]},"headerGeneratorOptions":{"devices":["desktop"],"locales":["fr","fr-FR"]},"insecureHTTPParser":true,"useHeaderGenerator":true},"decompress":true,"enableUnixSockets":false,"followRedirect":true,"headers":{"accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7","accept-encoding":"gzip, deflate, br","accept-language":"fr-FR,fr;q=0.9","connection":"keep-alive","sec-ch-ua":"\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"138\", \"Google Chrome\";v=\"138\"","sec-ch-ua-mobile":"?0","sec-ch-ua-platform":"\"Windows\"","sec-fetch-dest":"document","sec-fetch-mode":"navigate","sec-fetch-site":"same-site","sec-fetch-user":"?1","upgrade-insecure-requests":"1","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"},"hooks":{"afterResponse":[],"beforeError":[],"beforeRedirect":[null],"beforeRequest":[null,null,null,null,null,null],"beforeRetry":[],"init":[null,null]},"http2":true,"https":{"ciphers":"TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA","ecdhCurve":"X25519:prime256v1:secp384r1","maxVersion":"TLSv1.3","minVersion":"TLSv1","rejectUnauthorized":false,"signatureAlgorithms":"ecdsa_secp256r1_sha256:rsa_pss_rsae_sha256:rsa_pkcs1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha384:rsa_pkcs1_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha512"},"ignoreInvalidCookies":false,"isStream":false,"maxRedirects":10,"method":"GET","methodRewriting":false,"pagination":{"backoff":0,"countLimit":null,"requestLimit":10000,"stackAllItems":false},"password":"","prefixUrl":"","resolveBodyOnly":false,"responseType":"text","retry":{"backoffLimit":null,"errorCodes":["ETIMEDOUT","ECONNRESET","EADDRINUSE","ECONNREFUSED","EPIPE","ENOTFOUND","ENETUNREACH","EAI_AGAIN"],"limit":0,"methods":["GET","PUT","HEAD","DELETE","OPTIONS","TRACE"],"noise":100,"statusCodes":[408,413,429,500,502,503,504,521,522,524]},"setHost":true,"throwHttpErrors":false,"timeout":{"request":60000},"url":"https://skullshaver.fr/collections/men-s-shavers/products/skullshaver-silver-pro-sx5","username":""},"stack":"RequestError: Timeout awaiting 'request' for 60000ms\n    at ClientRequest.<anonymous> (file:///app/node_modules/.pnpm/got@14.4.7/node_modules/got/dist/source/core/index.js:673:61)\n    at Object.onceWrapper (node:events:633:26)\n    at ClientRequest.emit (node:events:530:35)\n    at request.emit (file:///app/node_modules/.pnpm/got-scraping@4.1.2/node_modules/got-scraping/dist/index.js:695:14)\n    at emitErrorEvent (node:_http_client:104:11)\n    at TLSSocket.socketErrorListener (node:_http_client:518:5)\n    at TLSSocket.emit (node:events:518:28)\n    at emitErrorNT (node:internal/streams/destroy:170:8)\n    at emitErrorCloseNT (node:internal/streams/destroy:129:3)\n    at process.processTicksAndRejections (node:internal/process/task_queues:90:21)\n    at Timeout._onTimeout (file:///app/node_modules/.pnpm/got@14.4.7/node_modules/got/dist/source/core/timed-out.js:39:33)\n    at listOnTimeout (node:internal/timers:588:17)\n    at process.processTimers (node:internal/timers:523:7)\ncaused by: TimeoutError: Timeout awaiting 'request' for 60000ms\n    at Timeout._onTimeout (file:///app/node_modules/.pnpm/got@14.4.7/node_modules/got/dist/source/core/timed-out.js:39:33)\n    at listOnTimeout (node:internal/timers:588:17)\n    at process.processTimers (node:internal/timers:523:7)","timings":{"error":1762637841481,"lookup":1762637781462,"phases":{"dns":3,"total":60023,"wait":1},"socket":1762637781459,"start":1762637781458},"type":"TimeoutError"},"msg":"Internal Error"}

[cmintey]

Gotcha, thanks for the additional information. That API uses the package "got-scraping" to fetch the page. I'm sure I can configure it to be proxy-aware

[dani]

Something like this seems to make it working

diff --git a/src/routes/api/product/+server.ts b/src/routes/api/product/+server.ts
index 1d3e672..9c54818 100644
--- a/src/routes/api/product/+server.ts
+++ b/src/routes/api/product/+server.ts
@@ -9,12 +9,14 @@ import shopping from "$lib/server/shopping";
 import { parseAcceptLanguageHeader } from "$lib/i18n";
 import { getFormatter } from "$lib/server/i18n";
 import { requireLoginOrError } from "$lib/server/auth";
+import { env } from "$env/dynamic/private";
 
 const scraper = metascraper([shopping(), metascraperTitle(), metascraperImage()]);
 
 const goShopping = async (targetUrl: string, locales: string[]) => {
     const resp = await gotScraping({
         url: targetUrl,
+        proxyUrl: env.HTTP_PROXY || "",
         headerGeneratorOptions: {
             devices: ["desktop"],
             locales

(but I couldn't check it doesn't break when no proxy is set as I can't make requests without a proxy in my env)

[cmintey]

I’d be happy to accept a PR with the change! I can test it out without a proxy

[dani]

Here it is #482