feat: introduce ech experiment based on go1.23 stdlib

cmd/ooniprobe/internal/nettests/echcheck.go

@@ -13,5 +13,9 @@ func (n ECHCheck) Run(ctl *Controller) error {
 	}
 	// providing an input containing an empty string causes the experiment
 	// to recognize the empty string and use the default URL
-	return ctl.Run(builder, []model.ExperimentTarget{model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("")})
+	return ctl.Run(builder, []model.ExperimentTarget{
+		model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://cloudflare-ech.com/cdn-cgi/trace"),
+		// Use ECH on a non-standard port.
+		model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://min-ng.test.defo.ie:15443"),
+	})
 }

internal/experiment/echcheck/generate.go

@@ -4,91 +4,74 @@ package echcheck
 // ietf.org/archive/id/draft-ietf-tls-esni-14.html
 
 import (
-	"fmt"
+	"io"
+
 	"github.com/cloudflare/circl/hpke"
 	"golang.org/x/crypto/cryptobyte"
-	"io"
 )
 
-const clientHelloOuter uint8 = 0
-
-// echExtension is the Encrypted Client Hello extension that is part of
-// ClientHelloOuter as specified in:
-// ietf.org/archive/id/draft-ietf-tls-esni-14.html#section-5
-type echExtension struct {
-	kdfID    uint16
-	aeadID   uint16
-	configID uint8
-	enc      []byte
-	payload  []byte
-}
-
-func (ech *echExtension) marshal() []byte {
-	var b cryptobyte.Builder
-	b.AddUint8(clientHelloOuter)
-	b.AddUint16(ech.kdfID)
-	b.AddUint16(ech.aeadID)
-	b.AddUint8(ech.configID)
-	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
-		b.AddBytes(ech.enc)
-	})
-	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
-		b.AddBytes(ech.payload)
-	})
-	return b.BytesOrPanic()
-}
-
-// generateGreaseExtension generates an ECH extension with random values as
-// specified in ietf.org/archive/id/draft-ietf-tls-esni-14.html#section-6.2
-func generateGreaseExtension(rand io.Reader) ([]byte, error) {
-	// initialize HPKE suite parameters
-	kem := hpke.KEM(uint16(hpke.KEM_X25519_HKDF_SHA256))
-	kdf := hpke.KDF(uint16(hpke.KDF_HKDF_SHA256))
-	aead := hpke.AEAD(uint16(hpke.AEAD_AES128GCM))
-
-	if !kem.IsValid() || !kdf.IsValid() || !aead.IsValid() {
-		return nil, fmt.Errorf("required parameters not supported")
+// ECH Config List per:
+// https://www.ietf.org/archive/id/draft-ietf-tls-esni-22.html#name-encrypted-clienthello-confi
+func generateGreaseyECHConfigList(rand io.Reader, publicName string) ([]byte, error) {
+	// Start ECHConfig
+	var c cryptobyte.Builder
+	version := uint16(0xfe0d)
+	c.AddUint16(version)
+
+	// Start ECHConfigContents
+	var ecc cryptobyte.Builder
+	// Start HpkeKeyConfig
+	randConfigId := make([]byte, 1)
+	if _, err := io.ReadFull(rand, randConfigId); err != nil {
+		return nil, err
 	}
-
-	defaultHPKESuite := hpke.NewSuite(kem, kdf, aead)
-
-	// generate a public key to place in 'enc' field
+	ecc.AddUint8(randConfigId[0])
+	ecc.AddUint16(uint16(hpke.KEM_X25519_HKDF_SHA256))
+	// Generate a public key
+	kem := hpke.KEM(uint16(hpke.KEM_X25519_HKDF_SHA256))
 	publicKey, _, err := kem.Scheme().GenerateKeyPair()
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate key pair: %s", err)
-	}
-
-	// initiate HPKE Sender
-	sender, err := defaultHPKESuite.NewSender(publicKey, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create sender: %s", err)
-	}
-
-	// Set ECH Extension Fields
-	var ech echExtension
-
-	ech.kdfID = uint16(kdf)
-	ech.aeadID = uint16(aead)
-
-	randomByte := make([]byte, 1)
-	_, err = io.ReadFull(rand, randomByte)
 	if err != nil {
 		return nil, err
 	}
-	ech.configID = randomByte[0]
-
-	ech.enc, _, err = sender.Setup(rand)
+	publicKeyBytes, err := publicKey.MarshalBinary()
 	if err != nil {
 		return nil, err
 	}
+	ecc.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(publicKeyBytes)
+	})
+	// Start HpkeSymmetricCipherSuite
+	kdf := hpke.KDF(uint16(hpke.KDF_HKDF_SHA256))
+	aead := hpke.AEAD(uint16(hpke.AEAD_AES128GCM))
+	var cs cryptobyte.Builder
+	cs.AddUint16(uint16(kdf))
+	cs.AddUint16(uint16(aead))
+	ecc.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(cs.BytesOrPanic())
+	})
+	// End HpkeSymmetricCipherSuite
+	// End HpkeKeyConfig
+	maxNameLength := uint8(42)
+	ecc.AddUint8(maxNameLength)
+	publicNameBytes := []byte(publicName)
+	ecc.AddUint8LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(publicNameBytes)
+	})
+	// Start ECHConfigExtension
+	var ece cryptobyte.Builder
+	ecc.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(ece.BytesOrPanic())
+	})
+	// End ECHConfigExtension
+	// End ECHConfigContents
+	c.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(ecc.BytesOrPanic())
+	})
+	// End ECHConfig
+	var l cryptobyte.Builder
+	l.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(c.BytesOrPanic())
+	})
 
-	// TODO: compute this correctly as per https://www.ietf.org/archive/id/draft-ietf-tls-esni-14.html#name-recommended-padding-scheme
-	randomEncodedClientHelloInnerLen := 100
-	cipherLen := int(aead.CipherLen(uint(randomEncodedClientHelloInnerLen)))
-	ech.payload = make([]byte, randomEncodedClientHelloInnerLen+cipherLen)
-	if _, err = io.ReadFull(rand, ech.payload); err != nil {
-		return nil, err
-	}
-
-	return ech.marshal(), nil
+	return l.BytesOrPanic(), nil
 }

internal/experiment/echcheck/generate_test.go

@@ -0,0 +1,17 @@
+package echcheck
+
+import (
+	"crypto/rand"
+	"testing"
+)
+
+func TestParseableGREASEConfigList(t *testing.T) {
+	// A GREASE extension that can't be parsed is invalid.
+	grease, err := generateGreaseyECHConfigList(rand.Reader, "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := parseECHConfigList(grease); err != nil {
+		t.Fatal(err)
+	}
+}

internal/experiment/echcheck/handshake.go

@@ -2,125 +2,126 @@ package echcheck
 
 import (
 	"context"
-	"crypto/rand"
 	"crypto/tls"
-	"net"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"net/url"
 	"time"
 
-	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/logx"
 	"github.com/ooni/probe-cli/v3/internal/measurexlite"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
-	utls "gitlab.com/yawning/utls.git"
 )
 
-const echExtensionType uint16 = 0xfe0d
-
-func connectAndHandshake(
-	ctx context.Context,
-	startTime time.Time,
-	address string, sni string, outerSni string,
-	logger model.Logger) (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
-
-	channel := make(chan model.ArchivalTLSOrQUICHandshakeResult)
-
-	ol := logx.NewOperationLogger(logger, "echcheck: TCPConnect %s", address)
-	var dialer net.Dialer
-	conn, err := dialer.DialContext(ctx, "tcp", address)
-	ol.Stop(err)
+// We can't see which outerservername go std lib selects, so we preemptively
+// make sure it's unambiguous for a given ECH Config List.  If the ecl is
+// empty, return an empty string.
+func getUnambiguousOuterServerName(ecl []byte) (string, error) {
+	if len(ecl) == 0 {
+		return "", nil
+	}
+	configs, err := parseECHConfigList(ecl)
 	if err != nil {
-		return nil, netxlite.NewErrWrapper(netxlite.ClassifyGenericError, netxlite.ConnectOperation, err)
+		return "", fmt.Errorf("failed to parse ECH config: %w", err)
 	}
-
-	go func() {
-		var res *model.ArchivalTLSOrQUICHandshakeResult
-		if outerSni == "" {
-			res = handshake(
-				ctx,
-				conn,
-				startTime,
-				address,
-				sni,
-				logger,
-			)
-		} else {
-			res = handshakeWithEch(
-				ctx,
-				conn,
-				startTime,
-				address,
-				outerSni,
-				logger,
-			)
-			// We need to set this explicitly because otherwise it will get
-			// overridden with the outerSni in the case of ECH
-			res.ServerName = sni
+	outerServerName := string(configs[0].PublicName)
+	for _, ec := range configs {
+		if string(ec.PublicName) != outerServerName {
+			// It's perfectly valid to have multiple ECH configs with different
+			// `PublicName`s. But, since we can't see which one is selected by
+			// go's tls package, we can't accurately record OuterServerName.
+			return "", fmt.Errorf("ambigious OuterServerName for config")
 		}
-		channel <- *res
-	}()
-
-	return channel, nil
+	}
+	return outerServerName, nil
 }
 
-func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time,
-	address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
-	return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{}, logger)
-}
+func startHandshake(ctx context.Context, echConfigList []byte, isGrease bool, startTime time.Time, address string, target *url.URL, logger model.Logger, testOnlyRootCAs *x509.CertPool) (chan TestKeys, error) {
 
-func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time,
-	address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
-	payload, err := generateGreaseExtension(rand.Reader)
-	if err != nil {
-		panic("failed to generate grease ECH: " + err.Error())
-	}
+	channel := make(chan TestKeys)
 
-	var utlsEchExtension utls.GenericExtension
+	tlsConfig := genEchTLSConfig(target.Hostname(), echConfigList, testOnlyRootCAs)
 
-	utlsEchExtension.Id = echExtensionType
-	utlsEchExtension.Data = payload
+	go func() {
+		tk := TestKeys{}
+		tk = handshake(ctx, isGrease, startTime, address, logger, tlsConfig, tk)
+		channel <- tk
+	}()
 
-	hs := handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension}, logger)
-	hs.ECHConfig = "GREASE"
-	hs.OuterServerName = sni
-	return hs
+	return channel, nil
 }
 
-func handshakeMaybePrintWithECH(doprint bool) string {
-	if doprint {
-		return "WithECH"
+// Add to tk TestKeys all events as they occur.  May call self recursively using retry_configs.
+func handshake(ctx context.Context, isGrease bool, startTime time.Time, address string, logger model.Logger, tlsConfig *tls.Config, tk TestKeys) TestKeys {
+	var d string
+	if isGrease {
+		d = " (GREASE)"
+	} else if len(tlsConfig.EncryptedClientHelloConfigList) > 0 {
+		d = " (RealECH)"
 	}
-	return ""
-}
-
-func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string,
-	extensions []utls.TLSExtension, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
-	tlsConfig := genTLSConfig(sni)
 
-	handshakerConstructor := newHandshakerWithExtensions(extensions)
-	tracedHandshaker := handshakerConstructor(log.Log, &utls.HelloFirefox_Auto)
+	ol1 := logx.NewOperationLogger(logger, "echcheck: TCPConnect %s", address)
+	trace := measurexlite.NewTrace(0, startTime)
+	dialer := trace.NewDialerWithoutResolver(logger)
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	ol1.Stop(err)
+	tk.TCPConnects = append(tk.TCPConnects, trace.TCPConnects()...)
+	tk.NetworkEvents = append(tk.NetworkEvents, trace.NetworkEvents()...)
 
-	ol := logx.NewOperationLogger(logger, "echcheck: TLSHandshake%s", handshakeMaybePrintWithECH(len(extensions) > 0))
+	ol2 := logx.NewOperationLogger(logger, "echcheck: DialTLS%s", d)
 	start := time.Now()
-	maybeTLSConn, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig)
+	maybeTLSConn := tls.Client(conn, tlsConfig)
+	err = maybeTLSConn.HandshakeContext(ctx)
 	finish := time.Now()
-	ol.Stop(err)
+	ol2.Stop(err)
+
+	retryConfigs := []byte{}
+	if echErr, ok := err.(*tls.ECHRejectionError); ok && isGrease {
+		if len(echErr.RetryConfigList) > 0 {
+			retryConfigs = echErr.RetryConfigList
+		}
+		// We ignore this error in crafting our TLSOrQUICHandshakeResult
+		// since the *golang* error is expected and merely indicates we
+		// didn't get the ECH setup we wanted.  It does NOT indicate that
+		// that the handshake itself was a failure.
+		// TODO: Can we *confirm* there wasn't a separate TLS failure?  This might be ambiguous :-(
+		// TODO: Confirm above semantics with OONI team.
+		err = nil
+	}
 
 	connState := netxlite.MaybeTLSConnectionState(maybeTLSConn)
-	return measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(zeroTime), "tcp", address, tlsConfig,
-		connState, err, finish.Sub(zeroTime))
+	hs := measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(startTime),
+		"tcp", address, tlsConfig, connState, err, finish.Sub(startTime))
+	if isGrease {
+		hs.ECHConfig = "GREASE"
+	} else {
+		hs.ECHConfig = base64.StdEncoding.EncodeToString(tlsConfig.EncryptedClientHelloConfigList)
+	}
+	osn, err := getUnambiguousOuterServerName(tlsConfig.EncryptedClientHelloConfigList)
+	if err != nil {
+		msg := fmt.Sprintf("can't determine OuterServerName: %s", err)
+		hs.SoError = &msg
+	}
+	hs.OuterServerName = osn
+	tk.TLSHandshakes = append(tk.TLSHandshakes, hs)
+
+	if len(retryConfigs) > 0 {
+		tlsConfig.EncryptedClientHelloConfigList = retryConfigs
+		tk = handshake(ctx, false, startTime, address, logger, tlsConfig, tk)
+	}
+
+	return tk
 }
 
-// We are creating the pool just once because there is a performance penalty
-// when creating it every time. See https://github.com/ooni/probe/issues/2413.
-var certpool = netxlite.NewMozillaCertPool()
-
-// genTLSConfig generates tls.Config from a given SNI
-func genTLSConfig(sni string) *tls.Config {
-	return &tls.Config{ // #nosec G402 - we need to use a large TLS versions range for measuring
-		RootCAs:            certpool,
-		ServerName:         sni,
-		NextProtos:         []string{"h2", "http/1.1"},
-		InsecureSkipVerify: true, // #nosec G402 - it's fine to skip verify in a nettest
+func genEchTLSConfig(host string, echConfigList []byte, testOnlyRootCAs *x509.CertPool) *tls.Config {
+	c := &tls.Config{ServerName: host}
+	if len(echConfigList) > 0 {
+		c.EncryptedClientHelloConfigList = echConfigList
+	}
+	if testOnlyRootCAs != nil {
+		c.RootCAs = testOnlyRootCAs
 	}
+	return c
 }