From c8134a8b86c8420dbca510cbe8601ba4dda5765e Mon Sep 17 00:00:00 2001 From: UUBulb <35923940+uubulb@users.noreply.github.com> Date: Wed, 28 Aug 2024 23:21:25 +0800 Subject: [PATCH] replace cloudflare-bpc with utls (#58) * replace cloudflare-bpc with utls * fix test * ci: skip utls test --- .github/workflows/test-on-pr.yml | 4 +- .github/workflows/test.yml | 4 +- cmd/agent/main.go | 21 +-- go.mod | 10 +- go.sum | 23 +-- pkg/monitor/myip.go | 4 +- pkg/util/util.go | 11 ++ pkg/utls/roundtripper.go | 276 +++++++++++++++++++++++++++++++ pkg/utls/roundtripper_test.go | 52 ++++++ 9 files changed, 368 insertions(+), 37 deletions(-) create mode 100644 pkg/utls/roundtripper.go create mode 100644 pkg/utls/roundtripper_test.go diff --git a/.github/workflows/test-on-pr.yml b/.github/workflows/test-on-pr.yml index 3cfabf0..9f0019e 100644 --- a/.github/workflows/test-on-pr.yml +++ b/.github/workflows/test-on-pr.yml @@ -17,8 +17,10 @@ jobs: with: go-version: "1.20.13" - name: Unit test + # Skip TestCloudflareDetection here, as most IP addresses of Github's action + # runners have been marked as bot. run: | - go test -v ./... + go test -skip TestCloudflareDetection -v ./... - name: Build test run: | go build ./cmd/agent diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 021cc5b..ca9b38c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,8 +25,10 @@ jobs: with: go-version: "1.20.13" - name: Unit test + # Skip TestCloudflareDetection here, as most IP addresses of Github's action + # runners have been marked as bot. run: | - go test -v ./... + go test -skip TestCloudflareDetection -v ./... #- name: Run Gosec Security Scanner # run: | # go install github.com/securego/gosec/v2/cmd/gosec@v2.19.0 diff --git a/cmd/agent/main.go b/cmd/agent/main.go index dcc9c94..628bb47 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -17,13 +17,13 @@ import ( "strings" "time" - bpc "github.com/DaRealFreak/cloudflare-bp-go" "github.com/blang/semver" "github.com/ebi-yade/altsvc-go" "github.com/go-ping/ping" "github.com/nezhahq/go-github-selfupdate/selfupdate" "github.com/nezhahq/service" "github.com/quic-go/quic-go/http3" + utls "github.com/refraction-networking/utls" "github.com/shirou/gopsutil/v4/host" "github.com/spf13/cobra" "google.golang.org/grpc" @@ -36,6 +36,7 @@ import ( "github.com/nezhahq/agent/pkg/processgroup" "github.com/nezhahq/agent/pkg/pty" "github.com/nezhahq/agent/pkg/util" + utlsx "github.com/nezhahq/agent/pkg/utls" pb "github.com/nezhahq/agent/proto" ) @@ -95,7 +96,6 @@ var ( const ( delayWhenError = time.Second * 10 // Agent 重连间隔 networkTimeOut = time.Second * 5 // 普通网络超时 - macOSChromeUA = "" ) func init() { @@ -121,15 +121,12 @@ func init() { return nil, err } + headers := util.BrowserHeaders() http.DefaultClient.Timeout = time.Second * 30 - httpClient.Transport = bpc.AddCloudFlareByPass(httpClient.Transport, bpc.Options{ - AddMissingHeaders: true, - Headers: map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "User-Agent": monitor.MacOSChromeUA, - }, - }) + httpClient.Transport = utlsx.NewUTLSHTTPRoundTripperWithProxy( + utls.HelloChrome_Auto, new(utls.Config), + http.DefaultTransport, nil, headers, + ) ex, err := os.Executable() if err != nil { @@ -725,7 +722,7 @@ func handleTerminalTask(task *pb.Task) { if remoteData, err = remoteIO.Recv(); err != nil { return } - if remoteData.Data == nil || len(remoteData.Data) == 0 { + if len(remoteData.Data) == 0 { return } switch remoteData.Data[0] { @@ -838,7 +835,7 @@ func handleFMTask(task *pb.Task) { if remoteData, err = remoteIO.Recv(); err != nil { return } - if remoteData.Data == nil || len(remoteData.Data) == 0 { + if len(remoteData.Data) == 0 { return } fmc.DoTask(remoteData) diff --git a/go.mod b/go.mod index 72b05f9..b407083 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.20 require ( github.com/AlecAivazis/survey/v2 v2.3.7 - github.com/DaRealFreak/cloudflare-bp-go v1.0.4 github.com/UserExistsError/conpty v0.1.4 github.com/artdarek/go-unzip v1.0.0 github.com/blang/semver v3.5.1+incompatible @@ -19,9 +18,11 @@ require ( github.com/nezhahq/go-github-selfupdate v0.0.0-20240713123605-d560a87d03a0 github.com/nezhahq/service v0.0.0-20240704142721-eba37f9cc709 github.com/quic-go/quic-go v0.40.1 + github.com/refraction-networking/utls v1.6.1 github.com/shirou/gopsutil/v4 v4.24.6 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 + golang.org/x/net v0.26.0 golang.org/x/sys v0.22.0 google.golang.org/grpc v1.64.1 google.golang.org/protobuf v1.34.2 @@ -30,11 +31,10 @@ require ( require ( gitee.com/naibahq/go-gitee v0.0.0-20240713052758-bc992e4c5b2c // indirect - github.com/EDDYCJY/fake-useragent v0.2.0 // indirect - github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/StackExchange/wmi v1.2.1 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/antihax/optional v1.0.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -48,6 +48,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jaypipes/pcidb v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/lufia/plan9stats v0.0.0-20230110061619-bbe2e5e100de // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -82,7 +83,6 @@ require ( golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/term v0.21.0 // indirect diff --git a/go.sum b/go.sum index 6b36ccf..6b52100 100644 --- a/go.sum +++ b/go.sum @@ -3,20 +3,14 @@ gitee.com/naibahq/go-gitee v0.0.0-20240713052758-bc992e4c5b2c h1:rFMPP1jR4CIOcxU gitee.com/naibahq/go-gitee v0.0.0-20240713052758-bc992e4c5b2c/go.mod h1:9gFPAuMAO9HJv5W73eoLV1NX71Ko5MhzGe+NwOJkm24= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/DaRealFreak/cloudflare-bp-go v1.0.4 h1:33X8Z0YMV1DEVvL/kYLku+rjb4wF712+VIh3xBoifQ0= -github.com/DaRealFreak/cloudflare-bp-go v1.0.4/go.mod h1:oBI9KAKb9FqdoB42uUqHU6pdP+YDWlKjpZRSk8JTuwk= -github.com/EDDYCJY/fake-useragent v0.2.0 h1:Jcnkk2bgXmDpX0z+ELlUErTkoLb/mxFBNd2YdcpvJBs= -github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/UserExistsError/conpty v0.1.4 h1:+3FhJhiqhyEJa+K5qaK3/w6w+sN3Nh9O9VbJyBS02to= github.com/UserExistsError/conpty v0.1.4/go.mod h1:PDglKIkX3O/2xVk0MV9a6bCWxRmPVfxqZoTG/5sSd9I= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/artdarek/go-unzip v1.0.0 h1:Ja9wfhiXyl67z5JT37rWjTSb62KXDP+9jHRkdSREUvg= @@ -26,6 +20,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= @@ -87,6 +83,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/lufia/plan9stats v0.0.0-20230110061619-bbe2e5e100de h1:V53FWzU6KAZVi1tPp5UIsMoUWJ2/PNwYIDXnu7QuBCE= @@ -133,6 +131,8 @@ github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5 github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q= github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c= +github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA= +github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -200,9 +200,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -221,12 +219,10 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -234,15 +230,12 @@ golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/monitor/myip.go b/pkg/monitor/myip.go index 51b8c0c..c488d03 100644 --- a/pkg/monitor/myip.go +++ b/pkg/monitor/myip.go @@ -11,8 +11,6 @@ import ( "github.com/nezhahq/agent/pkg/util" ) -const MacOSChromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" - var ( cfList = []string{ "https://blog.cloudflare.com/cdn-cgi/trace", @@ -116,6 +114,6 @@ func httpGetWithUA(client *http.Client, url string) (*http.Response, error) { if err != nil { return nil, err } - req.Header.Add("User-Agent", MacOSChromeUA) + req.Header.Add("User-Agent", util.MacOSChromeUA) return client.Do(req) } diff --git a/pkg/util/util.go b/pkg/util/util.go index d92bea9..b3751d7 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,6 +2,7 @@ package util import ( "fmt" + "net/http" "os" "time" @@ -9,6 +10,8 @@ import ( "github.com/nezhahq/service" ) +const MacOSChromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + var ( Json = jsoniter.ConfigCompatibleWithStandardLibrary Logger service.Logger = service.ConsoleLogger @@ -29,3 +32,11 @@ func Printf(enabled bool, format string, v ...interface{}) { Logger.Infof("NEZHA@%s>> "+format, append([]interface{}{time.Now().Format("2006-01-02 15:04:05")}, v...)...) } } + +func BrowserHeaders() *http.Header { + return &http.Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"}, + "Accept-Language": {"en,zh-CN;q=0.9,zh;q=0.8"}, + "User-Agent": {MacOSChromeUA}, + } +} diff --git a/pkg/utls/roundtripper.go b/pkg/utls/roundtripper.go new file mode 100644 index 0000000..6cf2bf5 --- /dev/null +++ b/pkg/utls/roundtripper.go @@ -0,0 +1,276 @@ +// SPDX-FileCopyrightText: Copyright (c) 2016, Serene Han, Arlo Breault +// SPDX-FileCopyrightText: Copyright (c) 2019-2020, The Tor Project, Inc +// SPDX-License-Identifier: BSD-3-Clause +// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/blob/main/common/utls/roundtripper.go + +package utls + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "math/rand" + "net" + "net/http" + "net/url" + "sync" + "time" + + utls "github.com/refraction-networking/utls" + "golang.org/x/net/http2" + "golang.org/x/net/proxy" +) + +// NewUTLSHTTPRoundTripperWithProxy creates an instance of RoundTripper that dial to remote HTTPS endpoint with +// an alternative version of TLS implementation that attempts to imitate browsers' fingerprint. +// clientHelloID is the clientHello that uTLS attempts to imitate +// uTlsConfig is the TLS Configuration template +// backdropTransport is the transport that will be used for non-https traffic +// returns a RoundTripper: its behaviour is documented at +// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/merge_requests/76#note_2777161 +func NewUTLSHTTPRoundTripperWithProxy(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config, + backdropTransport http.RoundTripper, proxy *url.URL, header *http.Header) http.RoundTripper { + rtImpl := &uTLSHTTPRoundTripperImpl{ + clientHelloID: clientHelloID, + config: uTlsConfig, + connectWithH1: map[string]bool{}, + backdropTransport: backdropTransport, + pendingConn: map[pendingConnKey]*unclaimedConnection{}, + proxyAddr: proxy, + headers: header, + } + rtImpl.init() + return rtImpl +} + +type uTLSHTTPRoundTripperImpl struct { + clientHelloID utls.ClientHelloID + config *utls.Config + + accessConnectWithH1 sync.Mutex + connectWithH1 map[string]bool + + httpsH1Transport http.RoundTripper + httpsH2Transport http.RoundTripper + backdropTransport http.RoundTripper + + accessDialingConnection sync.Mutex + pendingConn map[pendingConnKey]*unclaimedConnection + + proxyAddr *url.URL + + headers *http.Header +} + +type pendingConnKey struct { + isH2 bool + dest string +} + +var ( + errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN") + errEAGAINTooMany = errors.New("incorrect ALPN negotiated") + errExpired = errors.New("connection have expired") +) + +func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header = *r.headers + + if req.URL.Scheme != "https" { + return r.backdropTransport.RoundTrip(req) + } + for retryCount := 0; retryCount < 5; retryCount++ { + effectivePort := req.URL.Port() + if effectivePort == "" { + effectivePort = "443" + } + if r.getShouldConnectWithH1(fmt.Sprintf("%v:%v", req.URL.Hostname(), effectivePort)) { + resp, err := r.httpsH1Transport.RoundTrip(req) + if errors.Is(err, errEAGAIN) { + continue + } + return resp, err + } + resp, err := r.httpsH2Transport.RoundTrip(req) + if errors.Is(err, errEAGAIN) { + continue + } + return resp, err + } + return nil, errEAGAINTooMany +} + +func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool { + r.accessConnectWithH1.Lock() + defer r.accessConnectWithH1.Unlock() + if value, set := r.connectWithH1[domainName]; set { + return value + } + return false +} + +func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) { + r.accessConnectWithH1.Lock() + defer r.accessConnectWithH1.Unlock() + r.connectWithH1[domainName] = true +} + +func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) { + r.accessConnectWithH1.Lock() + defer r.accessConnectWithH1.Unlock() + r.connectWithH1[domainName] = false +} + +func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey { + return pendingConnKey{isH2: alpnIsH2, dest: dest} +} + +func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) { + connId := getPendingConnectionID(addr, alpnIsH2) + r.pendingConn[connId] = NewUnclaimedConnection(conn, time.Minute) +} + +func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn { + connId := getPendingConnectionID(addr, alpnIsH2) + if conn, ok := r.pendingConn[connId]; ok { + delete(r.pendingConn, connId) + if claimedConnection, err := conn.claimConnection(); err == nil { + return claimedConnection + } + } + return nil +} + +func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) { + r.accessDialingConnection.Lock() + defer r.accessDialingConnection.Unlock() + + if r.getShouldConnectWithH1(addr) == expectedH2 { + return nil, errEAGAIN + } + + //Get a cached connection if possible to reduce preflight connection closed without sending data + if gconn := r.getConn(addr, expectedH2); gconn != nil { + return gconn, nil + } + + conn, err := r.dialTLS(ctx, addr) + if err != nil { + return nil, err + } + + protocol := conn.ConnectionState().NegotiatedProtocol + + protocolIsH2 := protocol == http2.NextProtoTLS + + if protocolIsH2 == expectedH2 { + return conn, err + } + + r.putConn(addr, protocolIsH2, conn) + + if protocolIsH2 { + r.clearShouldConnectWithH1(addr) + } else { + r.setShouldConnectWithH1(addr) + } + + return nil, errEAGAIN +} + +// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac53b09..98bdffa1706dfc041d1e99b86c47f29d72ad3a0c +// by dcf1 +func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) { + config := r.config.Clone() + + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + config.ServerName = host + + systemDialer := &net.Dialer{} + + var dialer proxy.ContextDialer + dialer = systemDialer + + if r.proxyAddr != nil { + proxyDialer, err := proxy.FromURL(r.proxyAddr, systemDialer) + if err != nil { + return nil, err + } + dialer = proxyDialer.(proxy.ContextDialer) + } + + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + uconn := utls.UClient(conn, config, r.clientHelloID) + if net.ParseIP(config.ServerName) != nil { + err := uconn.RemoveSNIExtension() + if err != nil { + uconn.Close() + return nil, err + } + } + + err = uconn.Handshake() + if err != nil { + return nil, err + } + return uconn, nil +} + +func (r *uTLSHTTPRoundTripperImpl) init() { + min := 1 << 13 + max := 1 << 14 + + r.httpsH2Transport = &http2.Transport{ + DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) { + return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true) + }, + MaxReadFrameSize: 16384, + MaxDecoderHeaderTableSize: uint32(rand.Intn(max-min) + min), + } + r.httpsH1Transport = &http.Transport{ + DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) { + return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false) + }, + } +} + +func NewUnclaimedConnection(conn net.Conn, expireTime time.Duration) *unclaimedConnection { + c := &unclaimedConnection{ + Conn: conn, + } + time.AfterFunc(expireTime, c.tick) + return c +} + +type unclaimedConnection struct { + net.Conn + claimed bool + access sync.Mutex +} + +func (c *unclaimedConnection) claimConnection() (net.Conn, error) { + c.access.Lock() + defer c.access.Unlock() + if !c.claimed { + c.claimed = true + return c.Conn, nil + } + return nil, errExpired +} + +func (c *unclaimedConnection) tick() { + c.access.Lock() + defer c.access.Unlock() + if !c.claimed { + c.claimed = true + c.Conn.Close() + c.Conn = nil + } +} diff --git a/pkg/utls/roundtripper_test.go b/pkg/utls/roundtripper_test.go new file mode 100644 index 0000000..a3a6f9d --- /dev/null +++ b/pkg/utls/roundtripper_test.go @@ -0,0 +1,52 @@ +package utls_test + +import ( + "net/http" + "testing" + + utls "github.com/refraction-networking/utls" + + "github.com/nezhahq/agent/pkg/util" + utlsx "github.com/nezhahq/agent/pkg/utls" +) + +const url = "https://www.patreon.com/login" + +func TestCloudflareDetection(t *testing.T) { + client := http.DefaultClient + + t.Logf("testing connection to %s", url) + resp, err := doRequest(client, url) + if err != nil { + t.Errorf("Get %s failed: %v", url, err) + } + + if resp.StatusCode == 403 { + t.Log("Default client is detected, switching to client with utls transport") + headers := util.BrowserHeaders() + client.Transport = utlsx.NewUTLSHTTPRoundTripperWithProxy( + utls.HelloChrome_Auto, new(utls.Config), + http.DefaultTransport, nil, headers, + ) + resp, err = doRequest(client, url) + if err != nil { + t.Errorf("Get %s failed: %v", url, err) + } + if resp.StatusCode == 403 { + t.Fail() + } else { + t.Log("Client with utls transport passed Cloudflare detection") + } + } else { + t.Log("Default client passed Cloudflare detection") + } +} + +func doRequest(client *http.Client, url string) (*http.Response, error) { + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return resp, nil +}