Merge remote-tracking branch 'origin/master' into iosbuild

Author: bassosimone

GOVERSION

@@ -1 +1 @@
-1.20.8
+1.20.10

Readme.md

@@ -61,7 +61,7 @@ Debian/Ubuntu. Once `ooniprobe` is installed, refer to the
 ## Developer instructions
 
 This repository requires _exactly_ the Go version mentioned by the
-[GOVERSION](GOVERSION) file (i.e., go1.20.8). Using a different version of
+[GOVERSION](GOVERSION) file (i.e., go1.20.10). Using a different version of
 Go _may_ work as intended but is not recommended: we depend
 on packages forked from the standard library; so, it is
 more robust to use the same version of Go from which
@@ -88,10 +88,10 @@ sudo apt install golang build-essential ca-certificates git
 sudo apt install mingw-w64
 
 # install the required go version binary
-go install -v golang.org/dl/go1.20.8@latest
+go install -v golang.org/dl/go1.20.10@latest
 
 # fetch the whole go distribution
-$HOME/go/bin/go1.20.8 download
+$HOME/go/bin/go1.20.10 download
 ```
 
 ### Fedora developer setup
@@ -107,10 +107,10 @@ sudo dnf install golang make gcc gcc-c++ git
 sudo dnf install mingw64-gcc mingw64-gcc-c++
 
 # install the required go version binary
-go install -v golang.org/dl/go1.20.8@latest
+go install -v golang.org/dl/go1.20.10@latest
 
 # fetch the whole go distribution
-$HOME/go/bin/go1.20.8 download
+$HOME/go/bin/go1.20.10 download
 ```
 
 ### macOS developer setup
@@ -127,10 +127,10 @@ Then, you need to follow these instructions:
 brew install go
 
 # install the required go version binary
-go install -v golang.org/dl/go1.20.8@latest
+go install -v golang.org/dl/go1.20.10@latest
 
 # fetch the whole go distribution
-$HOME/go/bin/go1.20.8 download
+$HOME/go/bin/go1.20.10 download
 ```
 
 ### Build instructions
@@ -139,7 +139,7 @@ Once you have installed the correct Go version and a C compiler,
 you can compile `ooniprobe` using:
 
 ```bash
-$HOME/go/bin/go1.20.8 build -v -ldflags '-s -w' ./cmd/ooniprobe
+$HOME/go/bin/go1.20.10 build -v -ldflags '-s -w' ./cmd/ooniprobe
 ```
 
 This command will generate a stripped binary called `ooniprobe`
@@ -148,7 +148,7 @@ in the toplevel directory.
 Likewise, you can compile `miniooni` using:
 
 ```bash
-$HOME/go/bin/go1.20.8 build -v -ldflags '-s -w' ./internal/cmd/miniooni
+$HOME/go/bin/go1.20.10 build -v -ldflags '-s -w' ./internal/cmd/miniooni
 ```
 
 This command will generate a stripped binary called `miniooni`
@@ -157,7 +157,7 @@ in the toplevel directory.
 And `oohelperd` using:
 
 ```bash
-$HOME/go/bin/go1.20.8 build -v -ldflags '-s -w' ./internal/cmd/oohelperd
+$HOME/go/bin/go1.20.10 build -v -ldflags '-s -w' ./internal/cmd/oohelperd
 ```
 
 This command will generate a stripped binary called `oohelperd`

cmd/ooniprobe/internal/nettests/echcheck.go

@@ -0,0 +1,15 @@
+package nettests
+
+// ECHCheck nettest implementation.
+type ECHCheck struct{}
+
+// Run starts the nettest.
+func (n ECHCheck) Run(ctl *Controller) error {
+	builder, err := ctl.Session.NewExperimentBuilder("echcheck")
+	if err != nil {
+		return err
+	}
+	// providing an input containing an empty string causes the experiment
+	// to recognize the empty string and use the default URL
+	return ctl.Run(builder, []string{""})
+}

cmd/ooniprobe/internal/nettests/groups.go

@@ -55,7 +55,9 @@ var All = map[string]Group{
 		Label: "Experimental Nettests",
 		Nettests: []Nettest{
 			DNSCheck{},
+			ECHCheck{},
 			STUNReachability{},
+			RiseupVPN{},
 			TorSf{},
 			VanillaTor{},
 		},

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/cloudflare/circl v1.3.3
 	github.com/cretz/bine v0.2.0
 	github.com/fatih/color v1.15.0
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/google/gopacket v1.1.19
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.3.1
@@ -24,8 +24,8 @@ require (
 	github.com/montanaflynn/stats v0.7.1
 	github.com/ooni/go-libtor v1.1.8
 	github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107
-	github.com/ooni/oocrypto v0.5.4
-	github.com/ooni/oohttp v0.6.4
+	github.com/ooni/oocrypto v0.5.5
+	github.com/ooni/oohttp v0.6.5
 	github.com/ooni/probe-assets v0.19.0
 	github.com/pborman/getopt/v2 v2.1.0
 	github.com/pion/stun v0.6.1
@@ -40,7 +40,7 @@ require (
 	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0
 	gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1
 	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.16.0
+	golang.org/x/net v0.17.0
 	golang.org/x/sys v0.13.0
 )
 

go.sum

@@ -159,8 +159,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ=
@@ -326,10 +326,10 @@ github.com/ooni/go-libtor v1.1.8 h1:Wo3V3DVTxl5vZdxtQakqYP+DAHx7pPtAFSl1bnAa08w=
 github.com/ooni/go-libtor v1.1.8/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI=
 github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107 h1:PktaCPQ1NYZOaK+J8pQGYiPCYFkGR5H3ZURg9zPkQsI=
 github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107/go.mod h1:5X3Lk4+cnrwrQiYgRlCWXgV33IMDgLaO5s1x0DD/fO0=
-github.com/ooni/oocrypto v0.5.4 h1:/AkVZd+aq54+OXgOtWEmK8xgZsFQtlmtPf2VgY20YWw=
-github.com/ooni/oocrypto v0.5.4/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk=
-github.com/ooni/oohttp v0.6.4 h1:QZyOO4e88AzLOHGTgapXmsjtn1EVR7Wl+BtHd8okIf4=
-github.com/ooni/oohttp v0.6.4/go.mod h1:RipdYAUiw1UTnpm0ISd0r1Kiv/CGaRUgn08xbK1JgVo=
+github.com/ooni/oocrypto v0.5.5 h1:x0wIgtBfghVu8Ok0tR/xVyfHlo646hN1LB/5bzuXcIg=
+github.com/ooni/oocrypto v0.5.5/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk=
+github.com/ooni/oohttp v0.6.5 h1:hjMnX2fGNHYHqh1JmfxoTfnN9JmdgT0fa6yIEjoYhG8=
+github.com/ooni/oohttp v0.6.5/go.mod h1:RipdYAUiw1UTnpm0ISd0r1Kiv/CGaRUgn08xbK1JgVo=
 github.com/ooni/probe-assets v0.19.0 h1:XloDJQt6uxn6EYVwfWCOnlgsJZbmzO7VPFsJ8RPW8Ns=
 github.com/ooni/probe-assets v0.19.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU=
 github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
@@ -616,8 +616,8 @@ golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
 golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
-golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

internal/checkincache/checkincache.go

@@ -3,6 +3,7 @@ package checkincache
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/model"
@@ -25,6 +26,9 @@ type checkInFlagsWrapper struct {
 }
 
 // Store stores the result of the latest check-in in the given key-value store.
+//
+// We store check-in feature flags in a file called checkinflags.state. These flags
+// are valid for 24 hours, after which we consider them stale.
 func Store(kvStore model.KeyValueStore, resp *model.OOAPICheckInResult) error {
 	// store the check-in flags in the key-value store
 	wrapper := &checkInFlagsWrapper{
@@ -52,3 +56,16 @@ func GetFeatureFlag(kvStore model.KeyValueStore, name string) bool {
 	}
 	return wrapper.Flags[name] // works even if map is nil
 }
+
+// ExperimentEnabledKey returns the [model.KeyValueStore] key to use to
+// know whether a disabled experiment has been enabled via check-in.
+func ExperimentEnabledKey(name string) string {
+	return fmt.Sprintf("%s_enabled", name)
+}
+
+// ExperimentEnabled returns whether a given experiment has been enabled by a previous
+// execution of check-in. Some experiments are disabled by default for different reasons
+// and we use the check-in API to control whether and when they should be enabled.
+func ExperimentEnabled(kvStore model.KeyValueStore, name string) bool {
+	return GetFeatureFlag(kvStore, ExperimentEnabledKey(name))
+}

internal/engine/experiment_integration_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/registry"
 )
 
 func TestCreateAll(t *testing.T) {
@@ -21,6 +22,12 @@ func TestCreateAll(t *testing.T) {
 	}
 	sess := newSessionForTesting(t)
 	defer sess.Close()
+
+	// Since https://github.com/ooni/probe-cli/pull/1355, some experiments are disabled
+	// by default and we need an environment variable to instantiate them
+	os.Setenv(registry.OONI_FORCE_ENABLE_EXPERIMENT, "1")
+	defer os.Unsetenv(registry.OONI_FORCE_ENABLE_EXPERIMENT)
+
 	for _, name := range AllExperiments() {
 		builder, err := sess.NewExperimentBuilder(name)
 		if err != nil {

internal/engine/experimentbuilder.go

@@ -62,7 +62,7 @@ func (b *experimentBuilder) NewExperiment() model.Experiment {
 
 // newExperimentBuilder creates a new experimentBuilder instance.
 func newExperimentBuilder(session *Session, name string) (*experimentBuilder, error) {
-	factory, err := registry.NewFactory(name)
+	factory, err := registry.NewFactory(name, session.kvStore, session.logger)
 	if err != nil {
 		return nil, err
 	}

internal/engine/inputloader.go

@@ -219,6 +219,12 @@ func StaticBareInputForExperiment(name string) ([]string, error) {
 	// Implementation note: we may be called from pkg/oonimkall
 	// with a non-canonical experiment name, so we need to convert
 	// the experiment name to be canonical before proceeding.
+	//
+	// TODO(https://github.com/ooni/probe/issues/1390): serve DNSCheck
+	// inputs using richer input (aka check-in v2).
+	//
+	// TODO(https://github.com/ooni/probe/issues/2557): server STUNReachability
+	// inputs using richer input (aka check-in v2).
 	switch registry.CanonicalizeExperimentName(name) {
 	case "dnscheck":
 		return dnsCheckDefaultInput, nil

internal/engine/session.go

@@ -11,15 +11,13 @@ import (
 	"sync/atomic"
 
 	"github.com/ooni/probe-cli/v3/internal/bytecounter"
-	"github.com/ooni/probe-cli/v3/internal/checkincache"
 	"github.com/ooni/probe-cli/v3/internal/enginelocate"
 	"github.com/ooni/probe-cli/v3/internal/enginenetx"
 	"github.com/ooni/probe-cli/v3/internal/engineresolver"
 	"github.com/ooni/probe-cli/v3/internal/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/platform"
 	"github.com/ooni/probe-cli/v3/internal/probeservices"
-	"github.com/ooni/probe-cli/v3/internal/registry"
 	"github.com/ooni/probe-cli/v3/internal/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/tunnel"
 	"github.com/ooni/probe-cli/v3/internal/version"
@@ -406,16 +404,6 @@ var ErrAlreadyUsingProxy = errors.New(
 // for the experiment with the given name, or an error if
 // there's no such experiment with the given name
 func (s *Session) NewExperimentBuilder(name string) (model.ExperimentBuilder, error) {
-	name = registry.CanonicalizeExperimentName(name)
-	switch {
-	case name == "web_connectivity" && checkincache.GetFeatureFlag(s.kvStore, "webconnectivity_0.5"):
-		// use LTE rather than the normal webconnectivity when the
-		// feature flag has been set through the check-in API
-		s.Logger().Infof("using webconnectivity LTE")
-		name = "[email protected]"
-	default:
-		// nothing
-	}
 	eb, err := newExperimentBuilder(s, name)
 	if err != nil {
 		return nil, err

internal/experiment/echcheck/handshake.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/logx"
 	"github.com/ooni/probe-cli/v3/internal/measurexlite"
 	"github.com/ooni/probe-cli/v3/internal/model"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
@@ -16,11 +17,13 @@ import (
 
 const echExtensionType uint16 = 0xfe0d
 
-func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string) *model.ArchivalTLSOrQUICHandshakeResult {
-	return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{})
+func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time,
+	address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
+	return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{}, logger)
 }
 
-func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string) *model.ArchivalTLSOrQUICHandshakeResult {
+func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time,
+	address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
 	payload, err := generateGreaseExtension(rand.Reader)
 	if err != nil {
 		panic("failed to generate grease ECH: " + err.Error())
@@ -31,18 +34,28 @@ func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time, ad
 	utlsEchExtension.Id = echExtensionType
 	utlsEchExtension.Data = payload
 
-	return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension})
+	return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension}, logger)
 }
 
-func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string, extensions []utls.TLSExtension) *model.ArchivalTLSOrQUICHandshakeResult {
+func handshakeMaybePrintWithECH(doprint bool) string {
+	if doprint {
+		return "WithECH"
+	}
+	return ""
+}
+
+func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string,
+	extensions []utls.TLSExtension, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
 	tlsConfig := genTLSConfig(sni)
 
 	handshakerConstructor := newHandshakerWithExtensions(extensions)
 	tracedHandshaker := handshakerConstructor(log.Log, &utls.HelloFirefox_Auto)
 
+	ol := logx.NewOperationLogger(logger, "echcheck: TLSHandshake%s", handshakeMaybePrintWithECH(len(extensions) > 0))
 	start := time.Now()
 	maybeTLSConn, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig)
 	finish := time.Now()
+	ol.Stop(err)
 
 	connState := netxlite.MaybeTLSConnectionState(maybeTLSConn)
 	return measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(zeroTime), "tcp", address, tlsConfig,

internal/experiment/echcheck/handshake_test.go

@@ -9,6 +9,8 @@ import (
 	"net/url"
 	"testing"
 	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/model"
 )
 
 func TestHandshake(t *testing.T) {
@@ -31,7 +33,7 @@ func TestHandshake(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	result := handshakeWithEch(ctx, conn, time.Now(), parsed.Host, "example.org")
+	result := handshakeWithEch(ctx, conn, time.Now(), parsed.Host, "crypto.cloudflare.com", model.DiscardLogger)
 	if result == nil {
 		t.Fatal("expected result")
 	}