improvements to the HTTP transports

Author: caffix

engine/sessions/http_clients.go

@@ -49,16 +49,16 @@ func NewClients(perHost int) (*Clients, error) {
 	}
 
 	return &Clients{
-		General: &http.Client{Transport: genTr, Timeout: 20 * time.Second},
+		General: &http.Client{Transport: genTr, Timeout: 30 * time.Second},
 		Probe: &http.Client{
 			Transport: probTr,
 			// for probes, prefer per-request context timeouts; keep a hard cap anyway
-			Timeout: 8 * time.Second,
+			Timeout: 12 * time.Second,
 		},
 		Crawl: &http.Client{
 			Transport: crwlTr,
 			// crawls can legitimately take longer. Use request contexts to bound if needed
-			Timeout: 45 * time.Second,
+			Timeout: 60 * time.Second,
 			Jar:     jar,
 		},
 		genTr:  genTr,
@@ -70,15 +70,15 @@ func NewClients(perHost int) (*Clients, error) {
 func newGeneralTransport() *http.Transport {
 	return &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           amassnet.NewDialContext(5 * time.Second),
+		DialContext:           amassnet.NewDialContext(8 * time.Second),
 		ForceAttemptHTTP2:     true,
 		MaxIdleConns:          200,
 		MaxIdleConnsPerHost:   20,
-		MaxConnsPerHost:       100,
+		MaxConnsPerHost:       64,
 		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   5 * time.Second,
-		ExpectContinueTimeout: 4 * time.Second,
-		ResponseHeaderTimeout: 8 * time.Second,
+		TLSHandshakeTimeout:   8 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		ResponseHeaderTimeout: 15 * time.Second,
 		// prefer correct TLS verification for APIs
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
@@ -90,40 +90,39 @@ func newGeneralTransport() *http.Transport {
 func newProbeTransport(perHost int) *http.Transport {
 	return &http.Transport{
 		Proxy:             http.ProxyFromEnvironment,
-		DialContext:       amassnet.NewDialContext(2 * time.Second),
-		ForceAttemptHTTP2: true,
+		DialContext:       amassnet.NewDialContext(5 * time.Second),
+		ForceAttemptHTTP2: false,
 		// keep this lower: probes spray across many hosts; idle pools become “memory”
-		MaxIdleConns:          100,
+		MaxIdleConns:          64,
 		MaxIdleConnsPerHost:   perHost,
 		MaxConnsPerHost:       perHost * 3,
-		IdleConnTimeout:       5 * time.Second,
-		TLSHandshakeTimeout:   3 * time.Second,
+		IdleConnTimeout:       15 * time.Second,
+		TLSHandshakeTimeout:   6 * time.Second,
 		ExpectContinueTimeout: 0,
-		ResponseHeaderTimeout: 4 * time.Second,
+		ResponseHeaderTimeout: 8 * time.Second,
 		// often you’ll hit junk certs during probing; keep verification on by default.
 		// if you *must* allow insecure probing, fork a separate transport with InsecureSkipVerify=true
 		// but be intentional about it
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},
-		DisableKeepAlives:  true,
 		DisableCompression: true, // avoid spending CPU on gzip for tiny probe responses
 	}
 }
 
 func newCrawlTransport() *http.Transport {
 	return &http.Transport{
 		Proxy:             http.ProxyFromEnvironment,
-		DialContext:       amassnet.NewDialContext(6 * time.Second),
+		DialContext:       amassnet.NewDialContext(8 * time.Second),
 		ForceAttemptHTTP2: true,
 		// crawling usually hits same hosts repeatedly; keep-alives pay off
-		MaxIdleConns:          512,
+		MaxIdleConns:          256,
 		MaxIdleConnsPerHost:   32,
-		MaxConnsPerHost:       128,
+		MaxConnsPerHost:       96,
 		IdleConnTimeout:       120 * time.Second,
-		TLSHandshakeTimeout:   6 * time.Second,
+		TLSHandshakeTimeout:   8 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
-		ResponseHeaderTimeout: 12 * time.Second,
+		ResponseHeaderTimeout: 20 * time.Second,
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},

internal/net/network.go

@@ -86,7 +86,10 @@ type DialContext func(ctx context.Context, network, addr string) (net.Conn, erro
 
 // NewDialContext performs the dial using global variables (e.g. LocalAddr).
 func NewDialContext(timeout time.Duration) DialContext {
-	d := &net.Dialer{Timeout: timeout}
+	d := &net.Dialer{
+		Timeout:   timeout,
+		KeepAlive: 30 * time.Second,
+	}
 
 	return func(ctx context.Context, network, addr string) (net.Conn, error) {
 		_, p, err := net.SplitHostPort(addr)