Making headless httpclient more similar to real browsers

v2/pkg/protocols/headless/engine/http_client.go

@@ -3,6 +3,7 @@ package engine
 import (
 	"crypto/tls"
 	"net/http"
+	"net/http/cookiejar"
 	"net/url"
 	"time"
 
@@ -30,5 +31,17 @@ func newhttpClient(options *types.Options) *http.Client {
 		}
 	}
 
-	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}
+	jar, _ := cookiejar.New(nil)
+
+	httpclient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Duration(options.Timeout*3) * time.Second,
+		Jar:       jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// the browser should follow redirects not us
+			return http.ErrUseLastResponse
+		},
+	}
+
+	return httpclient
 }

v2/pkg/protocols/headless/engine/rules.go

@@ -8,6 +8,9 @@ import (
 
 // routingRuleHandler handles proxy rule for actions related to request/response modification
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
+	// usually browsers don't use chunked transfer encoding so we set the content-length nevertheless
+	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
+
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue