replace cloudflare-bpc with utls (#58)
* replace cloudflare-bpc with utls * fix test * ci: skip utls test
This commit is contained in:
UUBulb
+1
-3
@@ -11,8 +11,6 @@ import (
|
||||
"github.com/nezhahq/agent/pkg/util"
|
||||
)
|
||||
|
||||
const MacOSChromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
|
||||
|
||||
var (
|
||||
cfList = []string{
|
||||
"https://blog.cloudflare.com/cdn-cgi/trace",
|
||||
@@ -116,6 +114,6 @@ func httpGetWithUA(client *http.Client, url string) (*http.Response, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Add("User-Agent", MacOSChromeUA)
|
||||
req.Header.Add("User-Agent", util.MacOSChromeUA)
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
@@ -9,6 +10,8 @@ import (
|
||||
"github.com/nezhahq/service"
|
||||
)
|
||||
|
||||
const MacOSChromeUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
|
||||
var (
|
||||
Json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
Logger service.Logger = service.ConsoleLogger
|
||||
@@ -29,3 +32,11 @@ func Printf(enabled bool, format string, v ...interface{}) {
|
||||
Logger.Infof("NEZHA@%s>> "+format, append([]interface{}{time.Now().Format("2006-01-02 15:04:05")}, v...)...)
|
||||
}
|
||||
}
|
||||
|
||||
func BrowserHeaders() *http.Header {
|
||||
return &http.Header{
|
||||
"Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"},
|
||||
"Accept-Language": {"en,zh-CN;q=0.9,zh;q=0.8"},
|
||||
"User-Agent": {MacOSChromeUA},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2016, Serene Han, Arlo Breault
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2019-2020, The Tor Project, Inc
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/blob/main/common/utls/roundtripper.go
|
||||
|
||||
package utls
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
utls "github.com/refraction-networking/utls"
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
// NewUTLSHTTPRoundTripperWithProxy creates an instance of RoundTripper that dial to remote HTTPS endpoint with
|
||||
// an alternative version of TLS implementation that attempts to imitate browsers' fingerprint.
|
||||
// clientHelloID is the clientHello that uTLS attempts to imitate
|
||||
// uTlsConfig is the TLS Configuration template
|
||||
// backdropTransport is the transport that will be used for non-https traffic
|
||||
// returns a RoundTripper: its behaviour is documented at
|
||||
// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/merge_requests/76#note_2777161
|
||||
func NewUTLSHTTPRoundTripperWithProxy(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config,
|
||||
backdropTransport http.RoundTripper, proxy *url.URL, header *http.Header) http.RoundTripper {
|
||||
rtImpl := &uTLSHTTPRoundTripperImpl{
|
||||
clientHelloID: clientHelloID,
|
||||
config: uTlsConfig,
|
||||
connectWithH1: map[string]bool{},
|
||||
backdropTransport: backdropTransport,
|
||||
pendingConn: map[pendingConnKey]*unclaimedConnection{},
|
||||
proxyAddr: proxy,
|
||||
headers: header,
|
||||
}
|
||||
rtImpl.init()
|
||||
return rtImpl
|
||||
}
|
||||
|
||||
type uTLSHTTPRoundTripperImpl struct {
|
||||
clientHelloID utls.ClientHelloID
|
||||
config *utls.Config
|
||||
|
||||
accessConnectWithH1 sync.Mutex
|
||||
connectWithH1 map[string]bool
|
||||
|
||||
httpsH1Transport http.RoundTripper
|
||||
httpsH2Transport http.RoundTripper
|
||||
backdropTransport http.RoundTripper
|
||||
|
||||
accessDialingConnection sync.Mutex
|
||||
pendingConn map[pendingConnKey]*unclaimedConnection
|
||||
|
||||
proxyAddr *url.URL
|
||||
|
||||
headers *http.Header
|
||||
}
|
||||
|
||||
type pendingConnKey struct {
|
||||
isH2 bool
|
||||
dest string
|
||||
}
|
||||
|
||||
var (
|
||||
errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN")
|
||||
errEAGAINTooMany = errors.New("incorrect ALPN negotiated")
|
||||
errExpired = errors.New("connection have expired")
|
||||
)
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
req.Header = *r.headers
|
||||
|
||||
if req.URL.Scheme != "https" {
|
||||
return r.backdropTransport.RoundTrip(req)
|
||||
}
|
||||
for retryCount := 0; retryCount < 5; retryCount++ {
|
||||
effectivePort := req.URL.Port()
|
||||
if effectivePort == "" {
|
||||
effectivePort = "443"
|
||||
}
|
||||
if r.getShouldConnectWithH1(fmt.Sprintf("%v:%v", req.URL.Hostname(), effectivePort)) {
|
||||
resp, err := r.httpsH1Transport.RoundTrip(req)
|
||||
if errors.Is(err, errEAGAIN) {
|
||||
continue
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
resp, err := r.httpsH2Transport.RoundTrip(req)
|
||||
if errors.Is(err, errEAGAIN) {
|
||||
continue
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
return nil, errEAGAINTooMany
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool {
|
||||
r.accessConnectWithH1.Lock()
|
||||
defer r.accessConnectWithH1.Unlock()
|
||||
if value, set := r.connectWithH1[domainName]; set {
|
||||
return value
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) {
|
||||
r.accessConnectWithH1.Lock()
|
||||
defer r.accessConnectWithH1.Unlock()
|
||||
r.connectWithH1[domainName] = true
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) {
|
||||
r.accessConnectWithH1.Lock()
|
||||
defer r.accessConnectWithH1.Unlock()
|
||||
r.connectWithH1[domainName] = false
|
||||
}
|
||||
|
||||
func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey {
|
||||
return pendingConnKey{isH2: alpnIsH2, dest: dest}
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) {
|
||||
connId := getPendingConnectionID(addr, alpnIsH2)
|
||||
r.pendingConn[connId] = NewUnclaimedConnection(conn, time.Minute)
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn {
|
||||
connId := getPendingConnectionID(addr, alpnIsH2)
|
||||
if conn, ok := r.pendingConn[connId]; ok {
|
||||
delete(r.pendingConn, connId)
|
||||
if claimedConnection, err := conn.claimConnection(); err == nil {
|
||||
return claimedConnection
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) {
|
||||
r.accessDialingConnection.Lock()
|
||||
defer r.accessDialingConnection.Unlock()
|
||||
|
||||
if r.getShouldConnectWithH1(addr) == expectedH2 {
|
||||
return nil, errEAGAIN
|
||||
}
|
||||
|
||||
//Get a cached connection if possible to reduce preflight connection closed without sending data
|
||||
if gconn := r.getConn(addr, expectedH2); gconn != nil {
|
||||
return gconn, nil
|
||||
}
|
||||
|
||||
conn, err := r.dialTLS(ctx, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
protocol := conn.ConnectionState().NegotiatedProtocol
|
||||
|
||||
protocolIsH2 := protocol == http2.NextProtoTLS
|
||||
|
||||
if protocolIsH2 == expectedH2 {
|
||||
return conn, err
|
||||
}
|
||||
|
||||
r.putConn(addr, protocolIsH2, conn)
|
||||
|
||||
if protocolIsH2 {
|
||||
r.clearShouldConnectWithH1(addr)
|
||||
} else {
|
||||
r.setShouldConnectWithH1(addr)
|
||||
}
|
||||
|
||||
return nil, errEAGAIN
|
||||
}
|
||||
|
||||
// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac53b09..98bdffa1706dfc041d1e99b86c47f29d72ad3a0c
|
||||
// by dcf1
|
||||
func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) {
|
||||
config := r.config.Clone()
|
||||
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config.ServerName = host
|
||||
|
||||
systemDialer := &net.Dialer{}
|
||||
|
||||
var dialer proxy.ContextDialer
|
||||
dialer = systemDialer
|
||||
|
||||
if r.proxyAddr != nil {
|
||||
proxyDialer, err := proxy.FromURL(r.proxyAddr, systemDialer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialer = proxyDialer.(proxy.ContextDialer)
|
||||
}
|
||||
|
||||
conn, err := dialer.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uconn := utls.UClient(conn, config, r.clientHelloID)
|
||||
if net.ParseIP(config.ServerName) != nil {
|
||||
err := uconn.RemoveSNIExtension()
|
||||
if err != nil {
|
||||
uconn.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = uconn.Handshake()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return uconn, nil
|
||||
}
|
||||
|
||||
func (r *uTLSHTTPRoundTripperImpl) init() {
|
||||
min := 1 << 13
|
||||
max := 1 << 14
|
||||
|
||||
r.httpsH2Transport = &http2.Transport{
|
||||
DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
|
||||
return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true)
|
||||
},
|
||||
MaxReadFrameSize: 16384,
|
||||
MaxDecoderHeaderTableSize: uint32(rand.Intn(max-min) + min),
|
||||
}
|
||||
r.httpsH1Transport = &http.Transport{
|
||||
DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
|
||||
return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NewUnclaimedConnection(conn net.Conn, expireTime time.Duration) *unclaimedConnection {
|
||||
c := &unclaimedConnection{
|
||||
Conn: conn,
|
||||
}
|
||||
time.AfterFunc(expireTime, c.tick)
|
||||
return c
|
||||
}
|
||||
|
||||
type unclaimedConnection struct {
|
||||
net.Conn
|
||||
claimed bool
|
||||
access sync.Mutex
|
||||
}
|
||||
|
||||
func (c *unclaimedConnection) claimConnection() (net.Conn, error) {
|
||||
c.access.Lock()
|
||||
defer c.access.Unlock()
|
||||
if !c.claimed {
|
||||
c.claimed = true
|
||||
return c.Conn, nil
|
||||
}
|
||||
return nil, errExpired
|
||||
}
|
||||
|
||||
func (c *unclaimedConnection) tick() {
|
||||
c.access.Lock()
|
||||
defer c.access.Unlock()
|
||||
if !c.claimed {
|
||||
c.claimed = true
|
||||
c.Conn.Close()
|
||||
c.Conn = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package utls_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
utls "github.com/refraction-networking/utls"
|
||||
|
||||
"github.com/nezhahq/agent/pkg/util"
|
||||
utlsx "github.com/nezhahq/agent/pkg/utls"
|
||||
)
|
||||
|
||||
const url = "https://www.patreon.com/login"
|
||||
|
||||
func TestCloudflareDetection(t *testing.T) {
|
||||
client := http.DefaultClient
|
||||
|
||||
t.Logf("testing connection to %s", url)
|
||||
resp, err := doRequest(client, url)
|
||||
if err != nil {
|
||||
t.Errorf("Get %s failed: %v", url, err)
|
||||
}
|
||||
|
||||
if resp.StatusCode == 403 {
|
||||
t.Log("Default client is detected, switching to client with utls transport")
|
||||
headers := util.BrowserHeaders()
|
||||
client.Transport = utlsx.NewUTLSHTTPRoundTripperWithProxy(
|
||||
utls.HelloChrome_Auto, new(utls.Config),
|
||||
http.DefaultTransport, nil, headers,
|
||||
)
|
||||
resp, err = doRequest(client, url)
|
||||
if err != nil {
|
||||
t.Errorf("Get %s failed: %v", url, err)
|
||||
}
|
||||
if resp.StatusCode == 403 {
|
||||
t.Fail()
|
||||
} else {
|
||||
t.Log("Client with utls transport passed Cloudflare detection")
|
||||
}
|
||||
} else {
|
||||
t.Log("Default client passed Cloudflare detection")
|
||||
}
|
||||
}
|
||||
|
||||
func doRequest(client *http.Client, url string) (*http.Response, error) {
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return resp, nil
|
||||
}
|
||||
Reference in New Issue
Block a user