implement real Encrypted ClientHello probe

cmd/ooniprobe/internal/nettests/echcheck.go

@@ -13,5 +13,9 @@ func (n ECHCheck) Run(ctl *Controller) error {
 	}
 	// providing an input containing an empty string causes the experiment
 	// to recognize the empty string and use the default URL
-	return ctl.Run(builder, []model.ExperimentTarget{model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("")})
+	return ctl.Run(builder, []model.ExperimentTarget{
+		model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://cloudflare-ech.com/cdn-cgi/trace"),
+		model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://cloudflare-ech.com:443"),
+		model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://min-ng.test.defo.ie:15443"),
+	})
 }

go.mod

@@ -1,8 +1,6 @@
 module github.com/ooni/probe-cli/v3
 
-go 1.21.0
-
-toolchain go1.22.2
+go 1.23.4
 
 require (
 	filippo.io/age v1.2.0

go.sum

@@ -406,8 +406,6 @@ github.com/ooni/oocrypto v0.7.0 h1:FBwabkaDoroWcw7cqnxD065IZv/cvWiww729T2xWDOw=
 github.com/ooni/oocrypto v0.7.0/go.mod h1:kzRkZ1AGWFWLJW+ncw94c9ulpEI0oyDGDj59tGKpFuc=
 github.com/ooni/oohttp v0.8.0 h1:D256SKWc8FFN5WvNpG0ImomtXP3deAiRmxoN2GfKUeQ=
 github.com/ooni/oohttp v0.8.0/go.mod h1:6KnSv/hwqZFegFugPEHUGFghmby/9LavhA3BtCE+RQ4=
-github.com/ooni/probe-assets v0.24.0 h1:9y6bF9PyXrPBHu/RmyRZY8JOXHC6W2ZNRC7kaPcuHHk=
-github.com/ooni/probe-assets v0.24.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU=
 github.com/ooni/probe-assets v0.25.0 h1:W/zqKRjkRkTYKHURhiFIuflh+Trm1WaPUWSfVU/y2VA=
 github.com/ooni/probe-assets v0.25.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU=
 github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=

internal/experiment/echcheck/generate.go

@@ -4,91 +4,74 @@ package echcheck
 // ietf.org/archive/id/draft-ietf-tls-esni-14.html
 
 import (
-	"fmt"
+	"io"
+
 	"github.com/cloudflare/circl/hpke"
 	"golang.org/x/crypto/cryptobyte"
-	"io"
 )
 
-const clientHelloOuter uint8 = 0
-
-// echExtension is the Encrypted Client Hello extension that is part of
-// ClientHelloOuter as specified in:
-// ietf.org/archive/id/draft-ietf-tls-esni-14.html#section-5
-type echExtension struct {
-	kdfID    uint16
-	aeadID   uint16
-	configID uint8
-	enc      []byte
-	payload  []byte
-}
-
-func (ech *echExtension) marshal() []byte {
-	var b cryptobyte.Builder
-	b.AddUint8(clientHelloOuter)
-	b.AddUint16(ech.kdfID)
-	b.AddUint16(ech.aeadID)
-	b.AddUint8(ech.configID)
-	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
-		b.AddBytes(ech.enc)
-	})
-	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
-		b.AddBytes(ech.payload)
-	})
-	return b.BytesOrPanic()
-}
-
-// generateGreaseExtension generates an ECH extension with random values as
-// specified in ietf.org/archive/id/draft-ietf-tls-esni-14.html#section-6.2
-func generateGreaseExtension(rand io.Reader) ([]byte, error) {
-	// initialize HPKE suite parameters
-	kem := hpke.KEM(uint16(hpke.KEM_X25519_HKDF_SHA256))
-	kdf := hpke.KDF(uint16(hpke.KDF_HKDF_SHA256))
-	aead := hpke.AEAD(uint16(hpke.AEAD_AES128GCM))
-
-	if !kem.IsValid() || !kdf.IsValid() || !aead.IsValid() {
-		return nil, fmt.Errorf("required parameters not supported")
+// ECH Config List per:
+// https://www.ietf.org/archive/id/draft-ietf-tls-esni-22.html#name-encrypted-clienthello-confi
+func generateGreaseyECHConfigList(rand io.Reader, publicName string) ([]byte, error) {
+	// Start ECHConfig
+	var c cryptobyte.Builder
+	version := uint16(0xfe0d)
+	c.AddUint16(version)
+
+	// Start ECHConfigContents
+	var ecc cryptobyte.Builder
+	// Start HpkeKeyConfig
+	randConfigId := make([]byte, 1)
+	if _, err := io.ReadFull(rand, randConfigId); err != nil {
+		return nil, err
 	}
-
-	defaultHPKESuite := hpke.NewSuite(kem, kdf, aead)
-
-	// generate a public key to place in 'enc' field
+	ecc.AddUint8(randConfigId[0])
+	ecc.AddUint16(uint16(hpke.KEM_X25519_HKDF_SHA256))
+	// Generate a public key
+	kem := hpke.KEM(uint16(hpke.KEM_X25519_HKDF_SHA256))
 	publicKey, _, err := kem.Scheme().GenerateKeyPair()
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate key pair: %s", err)
-	}
-
-	// initiate HPKE Sender
-	sender, err := defaultHPKESuite.NewSender(publicKey, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create sender: %s", err)
-	}
-
-	// Set ECH Extension Fields
-	var ech echExtension
-
-	ech.kdfID = uint16(kdf)
-	ech.aeadID = uint16(aead)
-
-	randomByte := make([]byte, 1)
-	_, err = io.ReadFull(rand, randomByte)
 	if err != nil {
 		return nil, err
 	}
-	ech.configID = randomByte[0]
-
-	ech.enc, _, err = sender.Setup(rand)
+	publicKeyBytes, err := publicKey.MarshalBinary()
 	if err != nil {
 		return nil, err
 	}
+	ecc.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(publicKeyBytes)
+	})
+	// Start HpkeSymmetricCipherSuite
+	kdf := hpke.KDF(uint16(hpke.KDF_HKDF_SHA256))
+	aead := hpke.AEAD(uint16(hpke.AEAD_AES128GCM))
+	var cs cryptobyte.Builder
+	cs.AddUint16(uint16(kdf))
+	cs.AddUint16(uint16(aead))
+	ecc.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(cs.BytesOrPanic())
+	})
+	// End HpkeSymmetricCipherSuite
+	// End HpkeKeyConfig
+	maxNameLength := uint8(42)
+	ecc.AddUint8(maxNameLength)
+	publicNameBytes := []byte(publicName)
+	ecc.AddUint8LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(publicNameBytes)
+	})
+	// Start ECHConfigExtension
+	var ece cryptobyte.Builder
+	ecc.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(ece.BytesOrPanic())
+	})
+	// End ECHConfigExtension
+	// End ECHConfigContents
+	c.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(ecc.BytesOrPanic())
+	})
+	// End ECHConfig
+	var l cryptobyte.Builder
+	l.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(c.BytesOrPanic())
+	})
 
-	// TODO: compute this correctly as per https://www.ietf.org/archive/id/draft-ietf-tls-esni-14.html#name-recommended-padding-scheme
-	randomEncodedClientHelloInnerLen := 100
-	cipherLen := int(aead.CipherLen(uint(randomEncodedClientHelloInnerLen)))
-	ech.payload = make([]byte, randomEncodedClientHelloInnerLen+cipherLen)
-	if _, err = io.ReadFull(rand, ech.payload); err != nil {
-		return nil, err
-	}
-
-	return ech.marshal(), nil
+	return l.BytesOrPanic(), nil
 }

internal/experiment/echcheck/generate_test.go

@@ -0,0 +1,17 @@
+package echcheck
+
+import (
+	"crypto/rand"
+	"testing"
+)
+
+func TestParseableGREASEConfigList(t *testing.T) {
+	// A GREASE extension that can't be parsed is invalid.
+	grease, err := generateGreaseyECHConfigList(rand.Reader, "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := parseECHConfigList(grease); err != nil {
+		t.Fatal(err)
+	}
+}

internal/experiment/echcheck/handshake.go

@@ -2,26 +2,19 @@ package echcheck
 
 import (
 	"context"
-	"crypto/rand"
 	"crypto/tls"
+	"encoding/base64"
 	"net"
+	"net/url"
 	"time"
 
-	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/logx"
 	"github.com/ooni/probe-cli/v3/internal/measurexlite"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
-	utls "gitlab.com/yawning/utls.git"
 )
 
-const echExtensionType uint16 = 0xfe0d
-
-func connectAndHandshake(
-	ctx context.Context,
-	startTime time.Time,
-	address string, sni string, outerSni string,
-	logger model.Logger) (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
+func connectAndHandshake(ctx context.Context, echConfigList []byte, isGrease bool, startTime time.Time, address string, target *url.URL, outerServerName string, logger model.Logger) (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
 
 	channel := make(chan model.ArchivalTLSOrQUICHandshakeResult)
 
@@ -33,94 +26,65 @@ func connectAndHandshake(
 		return nil, netxlite.NewErrWrapper(netxlite.ClassifyGenericError, netxlite.ConnectOperation, err)
 	}
 
+	tlsConfig := genEchTLSConfig(target.Hostname(), echConfigList)
+
 	go func() {
-		var res *model.ArchivalTLSOrQUICHandshakeResult
-		if outerSni == "" {
-			res = handshake(
-				ctx,
-				conn,
-				startTime,
-				address,
-				sni,
-				logger,
-			)
-		} else {
-			res = handshakeWithEch(
-				ctx,
-				conn,
-				startTime,
-				address,
-				outerSni,
-				logger,
-			)
-			// We need to set this explicitly because otherwise it will get
-			// overridden with the outerSni in the case of ECH
-			res.ServerName = sni
-		}
-		channel <- *res
+		hs := handshake(ctx, conn, echConfigList, isGrease, startTime, address, logger, tlsConfig)
+		hs.OuterServerName = outerServerName
+		channel <- *hs
 	}()
 
 	return channel, nil
 }
 
-func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time,
-	address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
-	return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{}, logger)
-}
-
-func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time,
-	address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
-	payload, err := generateGreaseExtension(rand.Reader)
-	if err != nil {
-		panic("failed to generate grease ECH: " + err.Error())
+func handshake(ctx context.Context, conn net.Conn, echConfigList []byte, isGrease bool, startTime time.Time, address string, logger model.Logger, tlsConfig *tls.Config) *model.ArchivalTLSOrQUICHandshakeResult {
+	var d string
+	if isGrease {
+		d = " (GREASE)"
+	} else if len(echConfigList) > 0 {
+		d = " (RealECH)"
 	}
+	ol := logx.NewOperationLogger(logger, "echcheck: DialTLS%s", d)
+	start := time.Now()
 
-	var utlsEchExtension utls.GenericExtension
-
-	utlsEchExtension.Id = echExtensionType
-	utlsEchExtension.Data = payload
-
-	hs := handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension}, logger)
-	hs.ECHConfig = "GREASE"
-	hs.OuterServerName = sni
-	return hs
-}
+	maybeTLSConn := tls.Client(conn, tlsConfig)
+	err := maybeTLSConn.HandshakeContext(ctx)
 
-func handshakeMaybePrintWithECH(doprint bool) string {
-	if doprint {
-		return "WithECH"
+	if echErr, ok := err.(*tls.ECHRejectionError); ok && isGrease {
+		if len(echErr.RetryConfigList) > 0 {
+			tlsConfig.EncryptedClientHelloConfigList = echErr.RetryConfigList
+			maybeTLSConn, err = tls.Dial("tcp", address, tlsConfig)
+		}
 	}
-	return ""
-}
-
-func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string,
-	extensions []utls.TLSExtension, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
-	tlsConfig := genTLSConfig(sni)
-
-	handshakerConstructor := newHandshakerWithExtensions(extensions)
-	tracedHandshaker := handshakerConstructor(log.Log, &utls.HelloFirefox_Auto)
-
-	ol := logx.NewOperationLogger(logger, "echcheck: TLSHandshake%s", handshakeMaybePrintWithECH(len(extensions) > 0))
-	start := time.Now()
-	maybeTLSConn, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig)
 	finish := time.Now()
 	ol.Stop(err)
 
-	connState := netxlite.MaybeTLSConnectionState(maybeTLSConn)
-	return measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(zeroTime), "tcp", address, tlsConfig,
-		connState, err, finish.Sub(zeroTime))
+	var connState tls.ConnectionState
+	// If there's been an error, processing maybeTLSConn can panic.
+	if err != nil {
+		connState = tls.ConnectionState{}
+	} else {
+		connState = netxlite.MaybeTLSConnectionState(maybeTLSConn)
+	}
+	hs := measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(startTime),
+		"tcp", address, tlsConfig, connState, err, finish.Sub(startTime))
+	if isGrease {
+		hs.ECHConfig = "GREASE"
+	} else {
+		hs.ECHConfig = base64.StdEncoding.EncodeToString(echConfigList)
+	}
+	return hs
 }
 
-// We are creating the pool just once because there is a performance penalty
-// when creating it every time. See https://github.com/ooni/probe/issues/2413.
-var certpool = netxlite.NewMozillaCertPool()
-
-// genTLSConfig generates tls.Config from a given SNI
-func genTLSConfig(sni string) *tls.Config {
-	return &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
-		RootCAs:            certpool,
-		ServerName:         sni,
-		NextProtos:         []string{"h2", "http/1.1"},
-		InsecureSkipVerify: true, // #nosec G402 - it's fine to skip verify in a nettest
+func genEchTLSConfig(host string, echConfigList []byte) *tls.Config {
+	if len(echConfigList) == 0 {
+		return &tls.Config{ServerName: host}
+	}
+	return &tls.Config{
+		EncryptedClientHelloConfigList: echConfigList,
+		// This will be used as the inner SNI and we will validate
+		// we get a certificate for this name.  The outer SNI will
+		// be set based on the ECH config.
+		ServerName: host,
 	}
 }