From d0e1941d109efd5de63e378b69f676e7438b7ef5 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 16:01:35 +0100 Subject: [PATCH 01/11] prototype: each experiment manages its own arguments --- internal/cmd/tinyooni/main.go | 149 +++++++++ internal/cmd/tinyooni/session.go | 85 +++++ internal/cmd/tinyooni/utils.go | 40 +++ internal/cmd/tinyooni/webconnectivity.go | 114 +++++++ internal/engine/mockable/mockable.go | 21 ++ internal/engine/probeservices/collector.go | 15 + internal/engine/session.go | 8 + internal/experiment/webconnectivity/main.go | 325 ++++++++++++++++++++ internal/model/experiment.go | 73 ++++- internal/model/measurement.go | 55 ++++ internal/model/mocks/session.go | 20 ++ internal/model/ooapi.go | 10 + 12 files changed, 913 insertions(+), 2 deletions(-) create mode 100644 internal/cmd/tinyooni/main.go create mode 100644 internal/cmd/tinyooni/session.go create mode 100644 internal/cmd/tinyooni/utils.go create mode 100644 internal/cmd/tinyooni/webconnectivity.go create mode 100644 internal/experiment/webconnectivity/main.go diff --git a/internal/cmd/tinyooni/main.go b/internal/cmd/tinyooni/main.go new file mode 100644 index 0000000000..91cb0817ca --- /dev/null +++ b/internal/cmd/tinyooni/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "os" + + "github.com/ooni/probe-cli/v3/internal/version" + "github.com/spf13/cobra" +) + +// GlobalOptions contains the global options. +type GlobalOptions struct { + Emoji bool + HomeDir string + NoJSON bool + NoCollector bool + ProbeServicesURL string + Proxy string + RepeatEvery int64 + ReportFile string + SnowflakeRendezvous string + TorArgs []string + TorBinary string + Tunnel string + Verbose bool + Yes bool +} + +func main() { + var globalOptions GlobalOptions + rootCmd := &cobra.Command{ + Use: "tinyooni", + Short: "tinyooni is like miniooni but more experimental", + Args: cobra.NoArgs, + Version: version.Version, + } + rootCmd.SetVersionTemplate("{{ .Version }}\n") + flags := rootCmd.PersistentFlags() + + flags.BoolVar( + &globalOptions.Emoji, + "emoji", + false, + "whether to use emojis when logging", + ) + + flags.StringVar( + &globalOptions.HomeDir, + "home", + "", + "force specific home directory", + ) + + flags.BoolVarP( + &globalOptions.NoJSON, + "no-json", + "N", + false, + "disable writing to disk", + ) + + flags.BoolVarP( + &globalOptions.NoCollector, + "no-collector", + "n", + false, + "do not submit measurements to the OONI collector", + ) + + flags.StringVar( + &globalOptions.ProbeServicesURL, + "probe-services", + "", + "URL of the OONI backend instance you want to use", + ) + + flags.StringVar( + &globalOptions.Proxy, + "proxy", + "", + "set proxy URL to communicate with the OONI backend (mutually exclusive with --tunnel)", + ) + + flags.Int64Var( + &globalOptions.RepeatEvery, + "repeat-every", + 0, + "wait the given number of seconds and then repeat the same measurement", + ) + + flags.StringVarP( + &globalOptions.ReportFile, + "reportfile", + "o", + "", + "set the output report file path (default: \"report.jsonl\")", + ) + + flags.StringVar( + &globalOptions.SnowflakeRendezvous, + "snowflake-rendezvous", + "domain_fronting", + "rendezvous method for --tunnel=torsf (one of: \"domain_fronting\" and \"amp\")", + ) + + flags.StringSliceVar( + &globalOptions.TorArgs, + "tor-args", + []string{}, + "extra arguments for the tor binary (may be specified multiple times)", + ) + + flags.StringVar( + &globalOptions.TorBinary, + "tor-binary", + "", + "execute a specific tor binary", + ) + + flags.StringVar( + &globalOptions.Tunnel, + "tunnel", + "", + "tunnel to use to communicate with the OONI backend (one of: psiphon, tor, torsf)", + ) + + flags.BoolVarP( + &globalOptions.Verbose, + "verbose", + "v", + false, + "increase verbosity level", + ) + + flags.BoolVarP( + &globalOptions.Yes, + "yes", + "y", + false, + "assume yes as the answer to all questions", + ) + + rootCmd.MarkFlagsMutuallyExclusive("proxy", "tunnel") + + registerWebConnectivity(rootCmd, &globalOptions) + + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/internal/cmd/tinyooni/session.go b/internal/cmd/tinyooni/session.go new file mode 100644 index 0000000000..04a3cdf299 --- /dev/null +++ b/internal/cmd/tinyooni/session.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "net/url" + "os" + "path" + "path/filepath" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/version" +) + +const ( + softwareName = "tinyooni" + softwareVersion = version.Version +) + +// newSession creates a new measurement session. +func newSession(ctx context.Context, globalOptions *GlobalOptions) (*engine.Session, error) { + ooniDir := maybeGetOONIDir(globalOptions.HomeDir) + if err := os.MkdirAll(ooniDir, 0700); err != nil { + return nil, err + } + + // We cleanup the assets files used by versions of ooniprobe + // older than v3.9.0, where we started embedding the assets + // into the binary and use that directly. This cleanup doesn't + // remove the whole directory but only known files inside it + // and then the directory itself, if empty. We explicitly discard + // the return value as it does not matter to us here. + _, _ = assetsdir.Cleanup(path.Join(ooniDir, "assets")) + + var ( + proxyURL *url.URL + err error + ) + if globalOptions.Proxy != "" { + proxyURL, err = url.Parse(globalOptions.Proxy) + if err != nil { + return nil, err + } + } + + kvstore2dir := filepath.Join(ooniDir, "kvstore2") + kvstore, err := kvstore.NewFS(kvstore2dir) + if err != nil { + return nil, err + } + + tunnelDir := filepath.Join(ooniDir, "tunnel") + if err := os.MkdirAll(tunnelDir, 0700); err != nil { + return nil, err + } + + config := engine.SessionConfig{ + KVStore: kvstore, + Logger: log.Log, + ProxyURL: proxyURL, + SnowflakeRendezvous: globalOptions.SnowflakeRendezvous, + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TorArgs: globalOptions.TorArgs, + TorBinary: globalOptions.TorBinary, + TunnelDir: tunnelDir, + } + if globalOptions.ProbeServicesURL != "" { + config.AvailableProbeServices = []model.OOAPIService{{ + Address: globalOptions.ProbeServicesURL, + Type: "https", + }} + } + + sess, err := engine.NewSession(ctx, config) + if err != nil { + return nil, err + } + + log.Debugf("miniooni temporary directory: %s", sess.TempDir()) + return sess, nil +} diff --git a/internal/cmd/tinyooni/utils.go b/internal/cmd/tinyooni/utils.go new file mode 100644 index 0000000000..be56510e2d --- /dev/null +++ b/internal/cmd/tinyooni/utils.go @@ -0,0 +1,40 @@ +package main + +import ( + "os" + "path/filepath" + "runtime" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// getoonidir returns the $HOME directory. +func getHomeDir() (string, string) { + // See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da + if runtime.GOOS == "windows" { + home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + if home == "" { + home = os.Getenv("USERPROFILE") + } + return home, "ooniprobe" + } + if runtime.GOOS == "linux" { + home := os.Getenv("XDG_CONFIG_HOME") + if home != "" { + return home, "ooniprobe" + } + // fallthrough + } + return os.Getenv("HOME"), ".ooniprobe" +} + +// maybeGetOONIDir returns the $HOME/.ooniprobe equivalent unless optionsHome +// is already set, in which case it just returns optionsHome. +func maybeGetOONIDir(optionsHome string) string { + if optionsHome != "" { + return optionsHome + } + homeDir, dirName := getHomeDir() + runtimex.Assert(homeDir != "", "homeDir is empty") + return filepath.Join(homeDir, dirName) +} diff --git a/internal/cmd/tinyooni/webconnectivity.go b/internal/cmd/tinyooni/webconnectivity.go new file mode 100644 index 0000000000..7175251f31 --- /dev/null +++ b/internal/cmd/tinyooni/webconnectivity.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// webConnectivityOptions contains options for web connectivity. +type webConnectivityOptions struct { + Annotations []string + InputFilePaths []string + Inputs []string + MaxRuntime int64 + Random bool +} + +func registerWebConnectivity(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + options := &webConnectivityOptions{} + + subCmd := &cobra.Command{ + Use: "web_connectivity", + Short: "Runs the webconnectivity experiment", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + webConnectivityMain(globalOptions, options) + }, + Aliases: []string{"webconnectivity"}, + } + rootCmd.AddCommand(subCmd) + flags := subCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + flags.StringSliceVarP( + &options.InputFilePaths, + "input-file", + "f", + []string{}, + "path to file to supply test dependent input (may be specified multiple times)", + ) + + flags.StringSliceVarP( + &options.Inputs, + "input", + "i", + []string{}, + "add test-dependent input (may be specified multiple times)", + ) + + flags.Int64Var( + &options.MaxRuntime, + "max-runtime", + 0, + "maximum runtime in seconds for the experiment (zero means infinite)", + ) + + flags.BoolVar( + &options.Random, + "random", + false, + "randomize the inputs list", + ) +} + +func webConnectivityMain(globalOptions *GlobalOptions, options *webConnectivityOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, globalOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + db, err := database.Open("database.sqlite3") + runtimex.PanicOnError(err, "database.Open failed") + + networkDB, err := db.CreateNetwork(sess) + runtimex.PanicOnError(err, "db.Create failed") + + dbResult, err := db.CreateResult(".", "custom", networkDB.ID) + runtimex.PanicOnError(err, "db.CreateResult failed") + + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db, + Inputs: options.Inputs, + MaxRuntime: options.MaxRuntime, + MeasurementDir: "results.d", + NoCollector: false, + OnWiFi: true, + ResultID: dbResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + + err = webconnectivity.Main(ctx, args) + runtimex.PanicOnError(err, "webconnectivity.Main failed") +} diff --git a/internal/engine/mockable/mockable.go b/internal/engine/mockable/mockable.go index 76b3f08a84..0e0c90092e 100644 --- a/internal/engine/mockable/mockable.go +++ b/internal/engine/mockable/mockable.go @@ -40,6 +40,11 @@ type Session struct { MockableUserAgent string } +// SubmitMeasurementV2 implements model.ExperimentSession +func (*Session) SubmitMeasurementV2(ctx context.Context, measurement *model.Measurement) error { + panic("unimplemented") +} + // GetTestHelpersByName implements ExperimentSession.GetTestHelpersByName func (sess *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { services, okay := sess.MockableTestHelpers[name] @@ -148,4 +153,20 @@ func (sess *Session) UserAgent() string { return sess.MockableUserAgent } +func (sess *Session) CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInNettests, error) { + panic("not implemented") +} + +func (sess *Session) Platform() string { + panic("not implemented") +} + +func (sess *Session) ResolverASNString() string { + panic("not implemented") +} + +func (sess *Session) ResolverNetworkName() string { + panic("not implemented") +} + var _ model.ExperimentSession = &Session{} diff --git a/internal/engine/probeservices/collector.go b/internal/engine/probeservices/collector.go index 54818f2749..8779859d33 100644 --- a/internal/engine/probeservices/collector.go +++ b/internal/engine/probeservices/collector.go @@ -97,6 +97,21 @@ func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) return nil } +var ErrMissingReportID = errors.New("probeservices: missing report ID") + +func (c Client) SubmitMeasurementV2(ctx context.Context, m *model.Measurement) error { + if m.ReportID == "" { + return ErrMissingReportID + } + var updateResponse model.OOAPICollectorUpdateResponse + return c.APIClientTemplate.WithBodyLogging().Build().PostJSON( + ctx, fmt.Sprintf("/report/%s", m.ReportID), model.OOAPICollectorUpdateRequest{ + Format: "json", + Content: m, + }, &updateResponse, + ) +} + // ReportID returns the report ID. func (r reportChan) ReportID() string { return r.ID diff --git a/internal/engine/session.go b/internal/engine/session.go index 2ad9f222c2..578614fbfc 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -556,6 +556,14 @@ func (s *Session) ResolverNetworkName() string { return nn } +func (s *Session) SubmitMeasurementV2(ctx context.Context, meas *model.Measurement) error { + clnt, err := s.NewProbeServicesClient(ctx) + if err != nil { + return err + } + return clnt.SubmitMeasurementV2(ctx, meas) +} + // SoftwareName returns the application name. func (s *Session) SoftwareName() string { return s.softwareName diff --git a/internal/experiment/webconnectivity/main.go b/internal/experiment/webconnectivity/main.go new file mode 100644 index 0000000000..03805a9092 --- /dev/null +++ b/internal/experiment/webconnectivity/main.go @@ -0,0 +1,325 @@ +package webconnectivity + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") + +// Main is the main function of the experiment. +func Main( + ctx context.Context, + args *model.ExperimentMainArgs, +) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := sess.CheckIn(ctx, &model.OOAPICheckInConfig{ + Charging: args.Charging, + OnWiFi: args.OnWiFi, + Platform: sess.Platform(), + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + RunType: args.RunType, + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: args.CategoryCodes, + }, + }) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.WebConnectivity == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.WebConnectivity.ReportID + logger.Infof("ReportID: %s", reportID) + + // Obtain experiment inputs. + inputs := getInputs(args, checkInResp) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: &Config{}} + + // Record when we started running this nettest. + testStartTime := time.Now() + + // Create suitable stop policy. + shouldStop := newStopPolicy(args, testStartTime) + + // Create suitable progress emitter. + progresser := newProgressEmitter(args, inputs, testStartTime) + + // Measure each URL in sequence. + for inputIdx, input := range inputs { + + // Honour max runtime. + if shouldStop() { + break + } + + // Emit progress. + progresser(inputIdx, input.URL) + + // Measure the current URL. + err := measureSingleURL( + ctx, + args, + measurer, + testStartTime, + reportID, + inputIdx, + &input, + ) + + // An error here means stuff like "cannot write to disk". + if err != nil { + return err + } + } + + return nil +} + +// getInputs obtains inputs from either args or checkInResp giving +// priority to user supplied arguments inside args. +func getInputs(args *model.ExperimentMainArgs, checkInResp *model.OOAPICheckInNettests) []model.OOAPIURLInfo { + runtimex.Assert(checkInResp.WebConnectivity != nil, "getInputs passed invalid checkInResp") + inputs := args.Inputs + if len(inputs) < 1 { + return checkInResp.WebConnectivity.URLs + } + outputs := []model.OOAPIURLInfo{} + for _, input := range inputs { + outputs = append(outputs, model.OOAPIURLInfo{ + CategoryCode: "MISC", + CountryCode: "ZZ", + URL: input, + }) + } + return outputs +} + +// newStopPolicy creates a new stop policy depending on the +// arguments passed to the experiment in args. +func newStopPolicy(args *model.ExperimentMainArgs, testStartTime time.Time) func() bool { + if args.MaxRuntime <= 0 { + return func() bool { + return false + } + } + maxRuntime := time.Duration(args.MaxRuntime) * time.Second + return func() bool { + return time.Since(testStartTime) > maxRuntime + } +} + +func newProgressEmitter( + args *model.ExperimentMainArgs, + inputs []model.OOAPIURLInfo, + testStartTime time.Time, +) func(idx int, URL string) { + total := len(inputs) + if total <= 0 { + return func(idx int, URL string) {} // just in case + } + if args.MaxRuntime <= 0 { + return func(idx int, URL string) { + percentage := 100.0 * (float64(idx) / float64(total)) + args.Callbacks.OnProgress(percentage, URL) + } + } + maxRuntime := (time.Duration(args.MaxRuntime) * time.Second) + time.Nanosecond // avoid zero division + return func(idx int, URL string) { + elapsed := time.Since(testStartTime) + percentage := 100.0 * (float64(elapsed) / float64(maxRuntime)) + args.Callbacks.OnProgress(percentage, URL) + } +} + +// measureSingleURL measures a single URL. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - measurer is the measurer; +// +// - testStartTime is when the nettest started; +// +// - inputIdx is the input URL's index; +// +// - input is the current input; +// +// - reportID is the reportID to use. +func measureSingleURL( + ctx context.Context, + args *model.ExperimentMainArgs, + measurer *Measurer, + testStartTime time.Time, + reportID string, + inputIdx int, + input *model.OOAPIURLInfo, +) error { + sess := args.Session + + // Make sure we track this URL into the database. + urlIdx, err := args.Database.CreateOrUpdateURL( + input.URL, + input.CategoryCode, + input.CountryCode, + ) + if err != nil { + return err + } + + // Create a measurement object inside of the database. + dbMeas, err := args.Database.CreateMeasurement( + sql.NullString{ + String: reportID, + Valid: true, + }, + "web_connectivity", + args.MeasurementDir, + inputIdx, + args.ResultID, + sql.NullInt64{ + Int64: urlIdx, + Valid: true, + }, + ) + if err != nil { + return err + } + + // Create the measurement for this URL. + meas := model.NewMeasurement( + sess, + measurer, + reportID, + input.URL, + testStartTime, + args.Annotations, + ) + + // Perform the measurement proper. + err = measurer.Run(ctx, &model.ExperimentArgs{ + Callbacks: args.Callbacks, + Measurement: meas, + Session: sess, + }) + + // In case of error the measurement failed because of some + // fundamental issue, so we don't want to submit. + if err != nil { + failure := err.Error() + return args.Database.Failed(dbMeas, failure) + } + + // Extract the measurement summary and store it inside the database. + summary, err := measurer.GetSummaryKeys(meas) + if err != nil { + return err + } + + err = args.Database.AddTestKeys(dbMeas, summary) + if err != nil { + return err + } + + // Attempt to submit the measurement. + err = submitOrStoreLocally(ctx, args, sess, meas, reportID, input.URL, dbMeas) + if err != nil { + return err + } + + // Mark the measurement as done + return args.Database.Done(dbMeas) +} + +// submitOrStoreLocally submits the measurement or stores it locally. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - args contains the experiment's main arguments; +// +// - sess is the measurement session; +// +// - reportID is the reportID; +// +// - input is the possibly-empty input; +// +// - dbMeas is the database's view of the measurement. +// +// This function will return error only in case of fatal errors such as +// not being able to write onto the local disk. +func submitOrStoreLocally( + ctx context.Context, + args *model.ExperimentMainArgs, + sess model.ExperimentSession, + meas *model.Measurement, + reportID string, + input string, + dbMeas *model.DatabaseMeasurement, +) error { + logger := sess.Logger() + + if !args.NoCollector { + // Submit the measurement to the OONI backend. + err := sess.SubmitMeasurementV2(ctx, meas) + if err == nil { + logger.Infof( + "Measurement: https://explorer.ooni.org/measurement/%s?input=%s", + reportID, + input, + ) + return args.Database.UploadSucceeded(dbMeas) + } + + // Handle the case where we could not submit the measurement. + failure := err.Error() + if err := args.Database.UploadFailed(dbMeas, failure); err != nil { + return err + } + + // Fallthrough and attempt to save measurement on disk + } + + // Serialize to JSON. + data, err := json.Marshal(meas) + if err != nil { + return err + } + + // Write the measurement and return result. + return os.WriteFile(dbMeas.MeasurementFilePath.String, data, 0600) +} diff --git a/internal/model/experiment.go b/internal/model/experiment.go index 1cba6c1b84..1ab02cc784 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -11,8 +11,8 @@ import ( // ExperimentSession is the experiment's view of a session. type ExperimentSession interface { - // GetTestHelpersByName returns a list of test helpers with the given name. - GetTestHelpersByName(name string) ([]OOAPIService, bool) + // CheckIn invokes the check-in API. + CheckIn(ctx context.Context, config *OOAPICheckInConfig) (*OOAPICheckInNettests, error) // DefaultHTTPClient returns the default HTTPClient used by the session. DefaultHTTPClient() HTTPClient @@ -23,15 +23,42 @@ type ExperimentSession interface { // FetchTorTargets returns the targets for the Tor experiment or an error. FetchTorTargets(ctx context.Context, cc string) (map[string]OOAPITorTarget, error) + // GetTestHelpersByName returns a list of test helpers with the given name. + GetTestHelpersByName(name string) ([]OOAPIService, bool) + // Logger returns the logger used by the session. Logger() Logger + // Platform returns the operating system's platform name. + Platform() string + + // ProbeASNString returns the probe's ASN as a string. + ProbeASNString() string + // ProbeCC returns the country code. ProbeCC() string + // ProbeNetworkName is the name of the probes' ASN. + ProbeNetworkName() string + + // ResolverASNString is the resolver ASN as a string. + ResolverASNString() string + // ResolverIP returns the resolver's IP. ResolverIP() string + // ResolverNetworkName is the name of the resolver's ASN. + ResolverNetworkName() string + + // SoftwareName returns the name of the client software. + SoftwareName() string + + // SoftwareVersion returns the version of the client software. + SoftwareVersion() string + + // SubmitMeasurementV2 submits the given measurement. + SubmitMeasurementV2(ctx context.Context, measurement *Measurement) error + // TempDir returns the session's temporary directory. TempDir() string @@ -130,6 +157,48 @@ type ExperimentArgs struct { Session ExperimentSession } +// ExperimentMainArgs contains the args passed to the experiment's main. +type ExperimentMainArgs struct { + // Annotations contains OPTIONAL annotations. + Annotations map[string]string + + // CategoryCodes OPTIONALLY contains the enabled category codes. + CategoryCodes []string + + // Charging OPTIONALLY indicates whether the phone is charging. + Charging bool + + // Callbacks contains MANDATORY experiment callbacks. + Callbacks ExperimentCallbacks + + // Database is the MANDATORY database to use. + Database WritableDatabase + + // Inputs contains OPTIONAL experiment inputs. + Inputs []string + + // MaxRuntime is the OPTIONAL maximum runtime in seconds. + MaxRuntime int64 + + // MeasurementDir is the MANDATORY directory where to save measurements. + MeasurementDir string + + // NoCollector OPTIONALLY disables submitting the measurements. + NoCollector bool + + // OnWiFi OPTIONALLY indicates whether the phone is using Wi-Fi. + OnWiFi bool + + // ResultID contains the MANDATORY result ID. + ResultID int64 + + // RunType OPTIONALLY indicates in which mode we are running. + RunType RunType + + // Session is the MANDATORY session the experiment can use. + Session ExperimentSession +} + // ExperimentMeasurer is the interface that allows to run a // measurement for a specific experiment. type ExperimentMeasurer interface { diff --git a/internal/model/measurement.go b/internal/model/measurement.go index 7cd2a8e13e..668f574c2d 100644 --- a/internal/model/measurement.go +++ b/internal/model/measurement.go @@ -10,9 +10,12 @@ import ( "errors" "fmt" "net" + "runtime" "time" + "github.com/ooni/probe-cli/v3/internal/platform" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" ) const ( @@ -215,3 +218,55 @@ func scrubTestKeys(m *Measurement, currentIP string) error { data = bytes.ReplaceAll(data, []byte(currentIP), []byte(Scrubbed)) return scrubJSONUnmarshalTestKeys(data, &m.TestKeys) } + +// NewMeasurement creates a new measurement instance. +// +// Arguments: +// +// - sess is the measurement session; +// +// - measurer is the experiment's measurer; +// +// - reportID is the report ID; +// +// - input is the OPTIONAL measurement's input; +// +// - testStartTime is when this nettest started executing; +// +// - annotations contains the annotations. +func NewMeasurement( + sess ExperimentSession, + measurer ExperimentMeasurer, + reportID string, + input string, + testStartTime time.Time, + annotations map[string]string, +) *Measurement { + const dateFormat = "2006-01-02 15:04:05" + utctimenow := time.Now().UTC() + m := &Measurement{ + DataFormatVersion: OOAPIReportDefaultDataFormatVersion, + Input: MeasurementTarget(input), + MeasurementStartTime: utctimenow.Format(dateFormat), + MeasurementStartTimeSaved: utctimenow, + ProbeIP: DefaultProbeIP, + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + ProbeNetworkName: sess.ProbeNetworkName(), + ReportID: reportID, + ResolverASN: sess.ResolverASNString(), + ResolverIP: sess.ResolverIP(), + ResolverNetworkName: sess.ResolverNetworkName(), + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + TestName: measurer.ExperimentName(), + TestStartTime: testStartTime.Format(dateFormat), + TestVersion: measurer.ExperimentVersion(), + } + m.AddAnnotations(annotations) // must be before MANDATORY engine annotations + m.AddAnnotation("engine_name", "ooniprobe-engine") + m.AddAnnotation("engine_version", version.Version) + m.AddAnnotation("platform", platform.Name()) + m.AddAnnotation("architecture", runtime.GOARCH) + return m +} diff --git a/internal/model/mocks/session.go b/internal/model/mocks/session.go index aba0c3c9f9..ba248f6f42 100644 --- a/internal/model/mocks/session.go +++ b/internal/model/mocks/session.go @@ -61,6 +61,26 @@ type Session struct { config *model.OOAPICheckInConfig) (*model.OOAPICheckInNettests, error) } +// Platform implements model.ExperimentSession +func (*Session) Platform() string { + panic("unimplemented") +} + +// ResolverASNString implements model.ExperimentSession +func (*Session) ResolverASNString() string { + panic("unimplemented") +} + +// ResolverNetworkName implements model.ExperimentSession +func (*Session) ResolverNetworkName() string { + panic("unimplemented") +} + +// SubmitMeasurementV2 implements model.ExperimentSession +func (*Session) SubmitMeasurementV2(ctx context.Context, measurement *model.Measurement) error { + panic("unimplemented") +} + func (sess *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { return sess.MockGetTestHelpersByName(name) } diff --git a/internal/model/ooapi.go b/internal/model/ooapi.go index 83227871f2..8baa0f5247 100644 --- a/internal/model/ooapi.go +++ b/internal/model/ooapi.go @@ -45,6 +45,13 @@ type OOAPICheckInConfig struct { WebConnectivity OOAPICheckInConfigWebConnectivity `json:"web_connectivity"` } +// OOAPICheckInInfoTelegram contains the Telegram +// part of OOAPICheckInInfo. +type OOAPICheckInInfoTelegram struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + // OOAPICheckInInfoWebConnectivity contains the WebConnectivity // part of OOAPICheckInInfo. type OOAPICheckInInfoWebConnectivity struct { @@ -57,6 +64,9 @@ type OOAPICheckInInfoWebConnectivity struct { // OOAPICheckInNettests contains nettest information returned by the checkin API call. type OOAPICheckInNettests struct { + // Telegram contains Telegram related information. + Telegram *OOAPICheckInInfoTelegram `json:"telegram"` + // WebConnectivity contains WebConnectivity related information. WebConnectivity *OOAPICheckInInfoWebConnectivity `json:"web_connectivity"` } From 81b2ebad9f4b59280125eb455bd701e094e0dbaa Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 16:51:39 +0100 Subject: [PATCH 02/11] feat: also add the telegram experiment --- internal/cmd/tinyooni/main.go | 1 + internal/cmd/tinyooni/telegram.go | 80 ++++++++ internal/engine/experiment/telegram/main.go | 191 ++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 internal/cmd/tinyooni/telegram.go create mode 100644 internal/engine/experiment/telegram/main.go diff --git a/internal/cmd/tinyooni/main.go b/internal/cmd/tinyooni/main.go index 91cb0817ca..1ea67a5091 100644 --- a/internal/cmd/tinyooni/main.go +++ b/internal/cmd/tinyooni/main.go @@ -142,6 +142,7 @@ func main() { rootCmd.MarkFlagsMutuallyExclusive("proxy", "tunnel") registerWebConnectivity(rootCmd, &globalOptions) + registerTelegram(rootCmd, &globalOptions) if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/internal/cmd/tinyooni/telegram.go b/internal/cmd/tinyooni/telegram.go new file mode 100644 index 0000000000..5e6bf85b2a --- /dev/null +++ b/internal/cmd/tinyooni/telegram.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +// telegramOptions contains options for web connectivity. +type telegramOptions struct { + Annotations []string +} + +func registerTelegram(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + options := &telegramOptions{} + + subCmd := &cobra.Command{ + Use: "telegram", + Short: "Runs the telegram experiment", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + telegramMain(globalOptions, options) + }, + Aliases: []string{"webconnectivity"}, + } + rootCmd.AddCommand(subCmd) + flags := subCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) +} + +func telegramMain(globalOptions *GlobalOptions, options *telegramOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, globalOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + db, err := database.Open("database.sqlite3") + runtimex.PanicOnError(err, "database.Open failed") + + networkDB, err := db.CreateNetwork(sess) + runtimex.PanicOnError(err, "db.Create failed") + + dbResult, err := db.CreateResult(".", "custom", networkDB.ID) + runtimex.PanicOnError(err, "db.CreateResult failed") + + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: "results.d", + NoCollector: false, + OnWiFi: true, + ResultID: dbResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + + err = telegram.Main(ctx, args) + runtimex.PanicOnError(err, "telegram.Main failed") +} diff --git a/internal/engine/experiment/telegram/main.go b/internal/engine/experiment/telegram/main.go new file mode 100644 index 0000000000..ae06eccf7f --- /dev/null +++ b/internal/engine/experiment/telegram/main.go @@ -0,0 +1,191 @@ +package telegram + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") + +// Main is the main function of the experiment. +func Main( + ctx context.Context, + args *model.ExperimentMainArgs, +) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := sess.CheckIn(ctx, &model.OOAPICheckInConfig{ + Charging: args.Charging, + OnWiFi: args.OnWiFi, + Platform: sess.Platform(), + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + RunType: args.RunType, + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: args.CategoryCodes, + }, + }) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Telegram == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Telegram.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: Config{}} + + // Record when we started running this nettest. + testStartTime := time.Now() + + // Create a measurement object inside of the database. + dbMeas, err := args.Database.CreateMeasurement( + sql.NullString{ + String: reportID, + Valid: true, + }, + "telegram", + args.MeasurementDir, + 0, + args.ResultID, + sql.NullInt64{ + Int64: 0, + Valid: false, + }, + ) + if err != nil { + return err + } + + // Create the measurement for this URL. + meas := model.NewMeasurement( + sess, + measurer, + reportID, + "", + testStartTime, + args.Annotations, + ) + + // Perform the measurement proper. + err = measurer.Run(ctx, &model.ExperimentArgs{ + Callbacks: args.Callbacks, + Measurement: meas, + Session: sess, + }) + + // In case of error the measurement failed because of some + // fundamental issue, so we don't want to submit. + if err != nil { + failure := err.Error() + return args.Database.Failed(dbMeas, failure) + } + + // Extract the measurement summary and store it inside the database. + summary, err := measurer.GetSummaryKeys(meas) + if err != nil { + return err + } + + err = args.Database.AddTestKeys(dbMeas, summary) + if err != nil { + return err + } + + // Attempt to submit the measurement. + err = submitOrStoreLocally(ctx, args, sess, meas, reportID, "", dbMeas) + if err != nil { + return err + } + + // Mark the measurement as done + return args.Database.Done(dbMeas) +} + +// submitOrStoreLocally submits the measurement or stores it locally. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - args contains the experiment's main arguments; +// +// - sess is the measurement session; +// +// - reportID is the reportID; +// +// - input is the possibly-empty input; +// +// - dbMeas is the database's view of the measurement. +// +// This function will return error only in case of fatal errors such as +// not being able to write onto the local disk. +func submitOrStoreLocally( + ctx context.Context, + args *model.ExperimentMainArgs, + sess model.ExperimentSession, + meas *model.Measurement, + reportID string, + input string, + dbMeas *model.DatabaseMeasurement, +) error { + logger := sess.Logger() + + if !args.NoCollector { + // Submit the measurement to the OONI backend. + err := sess.SubmitMeasurementV2(ctx, meas) + if err == nil { + logger.Infof( + "Measurement: https://explorer.ooni.org/measurement/%s", + reportID, + ) + return args.Database.UploadSucceeded(dbMeas) + } + + // Handle the case where we could not submit the measurement. + failure := err.Error() + if err := args.Database.UploadFailed(dbMeas, failure); err != nil { + return err + } + + // Fallthrough and attempt to save measurement on disk + } + + // Serialize to JSON. + data, err := json.Marshal(meas) + if err != nil { + return err + } + + // Write the measurement and return result. + return os.WriteFile(dbMeas.MeasurementFilePath.String, data, 0600) +} From d0462dfc202da8ae1cf7155796302cc7d429b71c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 16:55:37 +0100 Subject: [PATCH 03/11] note a bunch more todos --- internal/engine/experiment/telegram/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/engine/experiment/telegram/main.go b/internal/engine/experiment/telegram/main.go index ae06eccf7f..c5b59a3ad6 100644 --- a/internal/engine/experiment/telegram/main.go +++ b/internal/engine/experiment/telegram/main.go @@ -14,6 +14,11 @@ import ( // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") +// TODO(bassosimone): I am wondering whether we should have a specific +// MainArgs struct for each experiment rather than a common struct. + +// TODO(bassosimone): ideally, I would like OONI Run v2 to call Main. + // Main is the main function of the experiment. func Main( ctx context.Context, From 6e393fb0f0009347d1832365f7647abdc71c2e78 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 16:57:20 +0100 Subject: [PATCH 04/11] note one more todo --- internal/engine/experiment/telegram/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/engine/experiment/telegram/main.go b/internal/engine/experiment/telegram/main.go index c5b59a3ad6..463d36e455 100644 --- a/internal/engine/experiment/telegram/main.go +++ b/internal/engine/experiment/telegram/main.go @@ -165,6 +165,11 @@ func submitOrStoreLocally( ) error { logger := sess.Logger() + // TODO(bassosimone): this function is basically the same for each + // experiment so we can easily share it. The only "tricky" part + // here is that we should construct the explorer URL differently + // depending on whether there's input. + if !args.NoCollector { // Submit the measurement to the OONI backend. err := sess.SubmitMeasurementV2(ctx, meas) From 34013b31abf894053e13be2a46c1337ebb1af3c7 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 17:23:27 +0100 Subject: [PATCH 05/11] significantly compact the code --- internal/engine/experiment/telegram/main.go | 156 +------------- internal/experiment/experiment.go | 224 ++++++++++++++++++++ internal/experiment/webconnectivity/main.go | 184 +--------------- 3 files changed, 237 insertions(+), 327 deletions(-) create mode 100644 internal/experiment/experiment.go diff --git a/internal/engine/experiment/telegram/main.go b/internal/engine/experiment/telegram/main.go index 463d36e455..3e46fbbeb0 100644 --- a/internal/engine/experiment/telegram/main.go +++ b/internal/engine/experiment/telegram/main.go @@ -2,28 +2,19 @@ package telegram import ( "context" - "database/sql" - "encoding/json" "errors" "os" "time" + "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") -// TODO(bassosimone): I am wondering whether we should have a specific -// MainArgs struct for each experiment rather than a common struct. - -// TODO(bassosimone): ideally, I would like OONI Run v2 to call Main. - // Main is the main function of the experiment. -func Main( - ctx context.Context, - args *model.ExperimentMainArgs, -) error { +func Main(ctx context.Context, args *model.ExperimentMainArgs) error { sess := args.Session logger := sess.Logger() @@ -39,19 +30,7 @@ func Main( // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := sess.CheckIn(ctx, &model.OOAPICheckInConfig{ - Charging: args.Charging, - OnWiFi: args.OnWiFi, - Platform: sess.Platform(), - ProbeASN: sess.ProbeASNString(), - ProbeCC: sess.ProbeCC(), - RunType: args.RunType, - SoftwareName: sess.SoftwareName(), - SoftwareVersion: sess.SoftwareVersion(), - WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ - CategoryCodes: args.CategoryCodes, - }, - }) + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -72,130 +51,13 @@ func Main( // Record when we started running this nettest. testStartTime := time.Now() - // Create a measurement object inside of the database. - dbMeas, err := args.Database.CreateMeasurement( - sql.NullString{ - String: reportID, - Valid: true, - }, - "telegram", - args.MeasurementDir, - 0, - args.ResultID, - sql.NullInt64{ - Int64: 0, - Valid: false, - }, - ) - if err != nil { - return err - } - - // Create the measurement for this URL. - meas := model.NewMeasurement( - sess, + return experiment.MeasurePossiblyNilInput( + ctx, + args, measurer, - reportID, - "", testStartTime, - args.Annotations, + reportID, + 0, // inputIdx + nil, // input ) - - // Perform the measurement proper. - err = measurer.Run(ctx, &model.ExperimentArgs{ - Callbacks: args.Callbacks, - Measurement: meas, - Session: sess, - }) - - // In case of error the measurement failed because of some - // fundamental issue, so we don't want to submit. - if err != nil { - failure := err.Error() - return args.Database.Failed(dbMeas, failure) - } - - // Extract the measurement summary and store it inside the database. - summary, err := measurer.GetSummaryKeys(meas) - if err != nil { - return err - } - - err = args.Database.AddTestKeys(dbMeas, summary) - if err != nil { - return err - } - - // Attempt to submit the measurement. - err = submitOrStoreLocally(ctx, args, sess, meas, reportID, "", dbMeas) - if err != nil { - return err - } - - // Mark the measurement as done - return args.Database.Done(dbMeas) -} - -// submitOrStoreLocally submits the measurement or stores it locally. -// -// Arguments: -// -// - ctx is the context for deadline/cancellation/timeout; -// -// - args contains the experiment's main arguments; -// -// - sess is the measurement session; -// -// - reportID is the reportID; -// -// - input is the possibly-empty input; -// -// - dbMeas is the database's view of the measurement. -// -// This function will return error only in case of fatal errors such as -// not being able to write onto the local disk. -func submitOrStoreLocally( - ctx context.Context, - args *model.ExperimentMainArgs, - sess model.ExperimentSession, - meas *model.Measurement, - reportID string, - input string, - dbMeas *model.DatabaseMeasurement, -) error { - logger := sess.Logger() - - // TODO(bassosimone): this function is basically the same for each - // experiment so we can easily share it. The only "tricky" part - // here is that we should construct the explorer URL differently - // depending on whether there's input. - - if !args.NoCollector { - // Submit the measurement to the OONI backend. - err := sess.SubmitMeasurementV2(ctx, meas) - if err == nil { - logger.Infof( - "Measurement: https://explorer.ooni.org/measurement/%s", - reportID, - ) - return args.Database.UploadSucceeded(dbMeas) - } - - // Handle the case where we could not submit the measurement. - failure := err.Error() - if err := args.Database.UploadFailed(dbMeas, failure); err != nil { - return err - } - - // Fallthrough and attempt to save measurement on disk - } - - // Serialize to JSON. - data, err := json.Marshal(meas) - if err != nil { - return err - } - - // Write the measurement and return result. - return os.WriteFile(dbMeas.MeasurementFilePath.String, data, 0600) } diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go new file mode 100644 index 0000000000..66ca591cdd --- /dev/null +++ b/internal/experiment/experiment.go @@ -0,0 +1,224 @@ +// Package experiment contains common code for implementing experiments. +package experiment + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/url" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// CallCheckIn is a convenience function that calls the +// check-in API using the given arguments. +func CallCheckIn( + ctx context.Context, + args *model.ExperimentMainArgs, + sess model.ExperimentSession, +) (*model.OOAPICheckInNettests, error) { + return sess.CheckIn(ctx, &model.OOAPICheckInConfig{ + Charging: args.Charging, + OnWiFi: args.OnWiFi, + Platform: sess.Platform(), + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + RunType: args.RunType, + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: args.CategoryCodes, + }, + }) +} + +// MeasurePossiblyNilInput measures a possibly nil +// input using the given experiment measurer. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - args contains the experiment-main's arguments; +// +// - measurer is the measurer; +// +// - testStartTime is when the nettest started; +// +// - reportID is the reportID to use; +// +// - inputIdx is the POSSIBLY-ZERO input's index; +// +// - input is the POSSIBLY-NIL input to measure. +// +// This function only returns an error in case there is a +// serious situation (e.g., cannot write to disk). +func MeasurePossiblyNilInput( + ctx context.Context, + args *model.ExperimentMainArgs, + measurer model.ExperimentMeasurer, + testStartTime time.Time, + reportID string, + inputIdx int, + input *model.OOAPIURLInfo, +) error { + runtimex.Assert(ctx != nil, "passed nil Context") + runtimex.Assert(args != nil, "passed nil ExperimentMainArgs") + runtimex.Assert(measurer != nil, "passed nil ExperimentMeasurer") + runtimex.Assert(reportID != "", "passed empty report ID") + sess := args.Session + + // Make sure we track this possibly-nil input into the database. + var ( + urlIdx sql.NullInt64 + urlInput string + ) + if input != nil { + index, err := args.Database.CreateOrUpdateURL( + input.URL, + input.CategoryCode, + input.CountryCode, + ) + if err != nil { + return err + } + urlIdx.Int64 = index + urlIdx.Valid = true + urlInput = input.URL + } + + // Create a measurement object inside of the database. + dbMeas, err := args.Database.CreateMeasurement( + sql.NullString{ + String: reportID, + Valid: true, + }, + measurer.ExperimentName(), + args.MeasurementDir, + inputIdx, + args.ResultID, + urlIdx, + ) + if err != nil { + return err + } + + // Create the measurement for this URL. + meas := model.NewMeasurement( + sess, + measurer, + reportID, + urlInput, // possibly the empty string + testStartTime, + args.Annotations, + ) + + // Perform the measurement proper. + err = measurer.Run(ctx, &model.ExperimentArgs{ + Callbacks: args.Callbacks, + Measurement: meas, + Session: sess, + }) + + // In case of error the measurement failed because of some + // fundamental issue, so we don't want to submit. + if err != nil { + return args.Database.Failed(dbMeas, err.Error()) + } + + // Extract the measurement summary and store it inside the database. + summary, err := measurer.GetSummaryKeys(meas) + if err != nil { + return err + } + + // Add summary to database. + err = args.Database.AddTestKeys(dbMeas, summary) + if err != nil { + return err + } + + // Attempt to submit the measurement. + err = SubmitOrStoreLocally(ctx, args, sess, meas, dbMeas) + if err != nil { + return err + } + + // Mark measurement as done. + return args.Database.Done(dbMeas) +} + +// SubmitOrStoreLocally submits the measurement or stores it locally. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - args contains the experiment's main arguments; +// +// - sess is the measurement session; +// +// - dbMeas is the database's view of the measurement. +// +// This function will return error only in case of fatal errors such as +// not being able to write onto the local disk. +func SubmitOrStoreLocally( + ctx context.Context, + args *model.ExperimentMainArgs, + sess model.ExperimentSession, + meas *model.Measurement, + dbMeas *model.DatabaseMeasurement, +) error { + runtimex.Assert(args != nil, "passed nil arguments") + runtimex.Assert(sess != nil, "passed nil Session") + runtimex.Assert(meas != nil, "passed nil measurement") + runtimex.Assert(dbMeas != nil, "passed nil dbMeas") + logger := sess.Logger() + + if !args.NoCollector { + // Submit the measurement to the OONI backend. + err := sess.SubmitMeasurementV2(ctx, meas) + if err == nil { + logger.Infof("Measurement: %s", ExplorerURL(meas)) + return args.Database.UploadSucceeded(dbMeas) + } + + // Handle the case where we could not submit the measurement. + failure := err.Error() + if err := args.Database.UploadFailed(dbMeas, failure); err != nil { + return err + } + + // Fallthrough and attempt to save measurement to disk + } + + // Serialize to JSON. + data, err := json.Marshal(meas) + if err != nil { + return err + } + + // Write the measurement and return result. + return os.WriteFile(dbMeas.MeasurementFilePath.String, data, 0600) +} + +// ExplorerURL returns the explorer URL associated with a measurement. +func ExplorerURL(meas *model.Measurement) string { + runtimex.Assert(meas != nil, "passed nil measurement") + runtimex.Assert(meas.ReportID != "", "passed empty report ID") + URL := &url.URL{ + Scheme: "https", + Host: "explorer.ooni.org", + Path: fmt.Sprintf("/measurement/%s", meas.ReportID), + } + if meas.Input != "" { + query := url.Values{} + query.Add("input", string(meas.Input)) + URL.RawQuery = query.Encode() + } + return URL.String() +} diff --git a/internal/experiment/webconnectivity/main.go b/internal/experiment/webconnectivity/main.go index 03805a9092..1c003ac0fe 100644 --- a/internal/experiment/webconnectivity/main.go +++ b/internal/experiment/webconnectivity/main.go @@ -2,12 +2,11 @@ package webconnectivity import ( "context" - "database/sql" - "encoding/json" "errors" "os" "time" + "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" ) @@ -16,10 +15,7 @@ import ( var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") // Main is the main function of the experiment. -func Main( - ctx context.Context, - args *model.ExperimentMainArgs, -) error { +func Main(ctx context.Context, args *model.ExperimentMainArgs) error { sess := args.Session logger := sess.Logger() @@ -35,19 +31,7 @@ func Main( // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := sess.CheckIn(ctx, &model.OOAPICheckInConfig{ - Charging: args.Charging, - OnWiFi: args.OnWiFi, - Platform: sess.Platform(), - ProbeASN: sess.ProbeASNString(), - ProbeCC: sess.ProbeCC(), - RunType: args.RunType, - SoftwareName: sess.SoftwareName(), - SoftwareVersion: sess.SoftwareVersion(), - WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ - CategoryCodes: args.CategoryCodes, - }, - }) + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -89,7 +73,7 @@ func Main( progresser(inputIdx, input.URL) // Measure the current URL. - err := measureSingleURL( + err := experiment.MeasurePossiblyNilInput( ctx, args, measurer, @@ -163,163 +147,3 @@ func newProgressEmitter( args.Callbacks.OnProgress(percentage, URL) } } - -// measureSingleURL measures a single URL. -// -// Arguments: -// -// - ctx is the context for deadline/cancellation/timeout; -// -// - measurer is the measurer; -// -// - testStartTime is when the nettest started; -// -// - inputIdx is the input URL's index; -// -// - input is the current input; -// -// - reportID is the reportID to use. -func measureSingleURL( - ctx context.Context, - args *model.ExperimentMainArgs, - measurer *Measurer, - testStartTime time.Time, - reportID string, - inputIdx int, - input *model.OOAPIURLInfo, -) error { - sess := args.Session - - // Make sure we track this URL into the database. - urlIdx, err := args.Database.CreateOrUpdateURL( - input.URL, - input.CategoryCode, - input.CountryCode, - ) - if err != nil { - return err - } - - // Create a measurement object inside of the database. - dbMeas, err := args.Database.CreateMeasurement( - sql.NullString{ - String: reportID, - Valid: true, - }, - "web_connectivity", - args.MeasurementDir, - inputIdx, - args.ResultID, - sql.NullInt64{ - Int64: urlIdx, - Valid: true, - }, - ) - if err != nil { - return err - } - - // Create the measurement for this URL. - meas := model.NewMeasurement( - sess, - measurer, - reportID, - input.URL, - testStartTime, - args.Annotations, - ) - - // Perform the measurement proper. - err = measurer.Run(ctx, &model.ExperimentArgs{ - Callbacks: args.Callbacks, - Measurement: meas, - Session: sess, - }) - - // In case of error the measurement failed because of some - // fundamental issue, so we don't want to submit. - if err != nil { - failure := err.Error() - return args.Database.Failed(dbMeas, failure) - } - - // Extract the measurement summary and store it inside the database. - summary, err := measurer.GetSummaryKeys(meas) - if err != nil { - return err - } - - err = args.Database.AddTestKeys(dbMeas, summary) - if err != nil { - return err - } - - // Attempt to submit the measurement. - err = submitOrStoreLocally(ctx, args, sess, meas, reportID, input.URL, dbMeas) - if err != nil { - return err - } - - // Mark the measurement as done - return args.Database.Done(dbMeas) -} - -// submitOrStoreLocally submits the measurement or stores it locally. -// -// Arguments: -// -// - ctx is the context for deadline/cancellation/timeout; -// -// - args contains the experiment's main arguments; -// -// - sess is the measurement session; -// -// - reportID is the reportID; -// -// - input is the possibly-empty input; -// -// - dbMeas is the database's view of the measurement. -// -// This function will return error only in case of fatal errors such as -// not being able to write onto the local disk. -func submitOrStoreLocally( - ctx context.Context, - args *model.ExperimentMainArgs, - sess model.ExperimentSession, - meas *model.Measurement, - reportID string, - input string, - dbMeas *model.DatabaseMeasurement, -) error { - logger := sess.Logger() - - if !args.NoCollector { - // Submit the measurement to the OONI backend. - err := sess.SubmitMeasurementV2(ctx, meas) - if err == nil { - logger.Infof( - "Measurement: https://explorer.ooni.org/measurement/%s?input=%s", - reportID, - input, - ) - return args.Database.UploadSucceeded(dbMeas) - } - - // Handle the case where we could not submit the measurement. - failure := err.Error() - if err := args.Database.UploadFailed(dbMeas, failure); err != nil { - return err - } - - // Fallthrough and attempt to save measurement on disk - } - - // Serialize to JSON. - data, err := json.Marshal(meas) - if err != nil { - return err - } - - // Write the measurement and return result. - return os.WriteFile(dbMeas.MeasurementFilePath.String, data, 0600) -} From db2a68a5782594c18579d8a08823d5edef2c8123 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 17:29:37 +0100 Subject: [PATCH 06/11] more fixes --- internal/cmd/tinyooni/utils.go | 2 +- internal/experiment/experiment.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/cmd/tinyooni/utils.go b/internal/cmd/tinyooni/utils.go index be56510e2d..cd33adc77f 100644 --- a/internal/cmd/tinyooni/utils.go +++ b/internal/cmd/tinyooni/utils.go @@ -8,7 +8,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/runtimex" ) -// getoonidir returns the $HOME directory. +// getHomeDir returns the $HOME directory. func getHomeDir() (string, string) { // See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da if runtime.GOOS == "windows" { diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go index 66ca591cdd..689840f203 100644 --- a/internal/experiment/experiment.go +++ b/internal/experiment/experiment.go @@ -1,6 +1,10 @@ // Package experiment contains common code for implementing experiments. package experiment +// +// Common code for implementing experiments +// + import ( "context" "database/sql" From 3d27c996c599904729af187ada49913456c61f27 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Dec 2022 17:40:41 +0100 Subject: [PATCH 07/11] make sure we can use ooniprobe to read --- .gitignore | 1 + cmd/ooniprobe/internal/log/handlers/cli/results.go | 6 +++++- internal/cmd/tinyooni/telegram.go | 8 +++++--- internal/cmd/tinyooni/utils.go | 5 +++++ internal/cmd/tinyooni/webconnectivity.go | 8 +++++--- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index ba35acb3d0..f0e54a6672 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ /*.tar.gz /testdata/gotmp /tmp-* +/tinyooni /*.zip diff --git a/cmd/ooniprobe/internal/log/handlers/cli/results.go b/cmd/ooniprobe/internal/log/handlers/cli/results.go index 9dd1deb2c3..49c99df971 100644 --- a/cmd/ooniprobe/internal/log/handlers/cli/results.go +++ b/cmd/ooniprobe/internal/log/handlers/cli/results.go @@ -86,7 +86,11 @@ var summarizers = map[string]func(uint64, uint64, string) []string{ } func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string { - return summarizers[name](totalCount, anomalyCount, ss) + summarizer, ok := summarizers[name] + if !ok { + return []string{"", "", ""} + } + return summarizer(totalCount, anomalyCount, ss) } func logResultItem(w io.Writer, f log.Fields) error { diff --git a/internal/cmd/tinyooni/telegram.go b/internal/cmd/tinyooni/telegram.go index 5e6bf85b2a..b5f9477fbe 100644 --- a/internal/cmd/tinyooni/telegram.go +++ b/internal/cmd/tinyooni/telegram.go @@ -43,6 +43,8 @@ func registerTelegram(rootCmd *cobra.Command, globalOptions *GlobalOptions) { func telegramMain(globalOptions *GlobalOptions, options *telegramOptions) { ctx := context.Background() + ooniHome := maybeGetOONIDir(globalOptions.HomeDir) + // create a new measurement session sess, err := newSession(ctx, globalOptions) runtimex.PanicOnError(err, "newSession failed") @@ -50,13 +52,13 @@ func telegramMain(globalOptions *GlobalOptions, options *telegramOptions) { err = sess.MaybeLookupLocationContext(ctx) runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") - db, err := database.Open("database.sqlite3") + db, err := database.Open(databasePath(ooniHome)) runtimex.PanicOnError(err, "database.Open failed") networkDB, err := db.CreateNetwork(sess) runtimex.PanicOnError(err, "db.Create failed") - dbResult, err := db.CreateResult(".", "custom", networkDB.ID) + dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) runtimex.PanicOnError(err, "db.CreateResult failed") args := &model.ExperimentMainArgs{ @@ -67,7 +69,7 @@ func telegramMain(globalOptions *GlobalOptions, options *telegramOptions) { Database: db, Inputs: nil, MaxRuntime: 0, - MeasurementDir: "results.d", + MeasurementDir: dbResult.MeasurementDir, NoCollector: false, OnWiFi: true, ResultID: dbResult.ID, diff --git a/internal/cmd/tinyooni/utils.go b/internal/cmd/tinyooni/utils.go index cd33adc77f..cb7ec49fea 100644 --- a/internal/cmd/tinyooni/utils.go +++ b/internal/cmd/tinyooni/utils.go @@ -38,3 +38,8 @@ func maybeGetOONIDir(optionsHome string) string { runtimex.Assert(homeDir != "", "homeDir is empty") return filepath.Join(homeDir, dirName) } + +// databasePath returns the database path given the OONI_HOME. +func databasePath(ooniHome string) string { + return filepath.Join(ooniHome, "db", "main.sqlite3") +} diff --git a/internal/cmd/tinyooni/webconnectivity.go b/internal/cmd/tinyooni/webconnectivity.go index 7175251f31..4e4bf71cef 100644 --- a/internal/cmd/tinyooni/webconnectivity.go +++ b/internal/cmd/tinyooni/webconnectivity.go @@ -77,6 +77,8 @@ func registerWebConnectivity(rootCmd *cobra.Command, globalOptions *GlobalOption func webConnectivityMain(globalOptions *GlobalOptions, options *webConnectivityOptions) { ctx := context.Background() + ooniHome := maybeGetOONIDir(globalOptions.HomeDir) + // create a new measurement session sess, err := newSession(ctx, globalOptions) runtimex.PanicOnError(err, "newSession failed") @@ -84,13 +86,13 @@ func webConnectivityMain(globalOptions *GlobalOptions, options *webConnectivityO err = sess.MaybeLookupLocationContext(ctx) runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") - db, err := database.Open("database.sqlite3") + db, err := database.Open(databasePath(ooniHome)) runtimex.PanicOnError(err, "database.Open failed") networkDB, err := db.CreateNetwork(sess) runtimex.PanicOnError(err, "db.Create failed") - dbResult, err := db.CreateResult(".", "custom", networkDB.ID) + dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) runtimex.PanicOnError(err, "db.CreateResult failed") args := &model.ExperimentMainArgs{ @@ -101,7 +103,7 @@ func webConnectivityMain(globalOptions *GlobalOptions, options *webConnectivityO Database: db, Inputs: options.Inputs, MaxRuntime: options.MaxRuntime, - MeasurementDir: "results.d", + MeasurementDir: dbResult.MeasurementDir, NoCollector: false, OnWiFi: true, ResultID: dbResult.ID, From 8ce226631de08000bc2badfe1b0b3583f6052c3f Mon Sep 17 00:00:00 2001 From: DecFox Date: Sun, 15 Jan 2023 23:50:53 +0530 Subject: [PATCH 08/11] feat: introduce tinyooni --- .../cmd/tinyooni/{utils.go => database.go} | 23 ++ internal/cmd/tinyooni/groups.json | 68 +++++ internal/cmd/tinyooni/main.go | 5 +- internal/cmd/tinyooni/oonirun.go | 101 ++++++++ internal/cmd/tinyooni/run.go | 81 ++++++ internal/cmd/tinyooni/runx.go | 54 ++++ internal/cmd/tinyooni/telegram.go | 82 ------ internal/cmd/tinyooni/webconnectivity.go | 116 --------- internal/database/props.go | 18 ++ internal/engine/experiment/dash/main.go | 63 +++++ .../engine/experiment/fbmessenger/main.go | 63 +++++ internal/engine/experiment/hhfm/main.go | 149 +++++++++++ internal/engine/experiment/hirl/main.go | 72 +++++ internal/engine/experiment/ndt7/main.go | 67 +++++ internal/engine/experiment/psiphon/main.go | 63 +++++ internal/engine/experiment/riseupvpn/main.go | 63 +++++ internal/engine/experiment/signal/main.go | 63 +++++ internal/engine/experiment/telegram/main.go | 6 +- internal/engine/experiment/tor/main.go | 69 +++++ internal/engine/experiment/whatsapp/main.go | 63 +++++ internal/experiment/webconnectivity/main.go | 4 +- internal/model/ooapi.go | 153 ++++++++++- internal/oonirunx/link.go | 86 ++++++ internal/oonirunx/v1.go | 95 +++++++ internal/oonirunx/v2.go | 245 ++++++++++++++++++ internal/registryx/allexperiments.go | 12 + internal/registryx/dash.go | 89 +++++++ internal/registryx/factory.go | 196 ++++++++++++++ internal/registryx/fbmessenger.go | 89 +++++++ internal/registryx/hhfm.go | 126 +++++++++ internal/registryx/hirl.go | 89 +++++++ internal/registryx/ndt.go | 89 +++++++ internal/registryx/psiphon.go | 89 +++++++ internal/registryx/riseupvpn.go | 89 +++++++ internal/registryx/signal.go | 89 +++++++ internal/registryx/telegram.go | 89 +++++++ internal/registryx/tor.go | 89 +++++++ internal/registryx/utils.go | 30 +++ internal/registryx/webconnectivity.go | 126 +++++++++ internal/registryx/whatsapp.go | 89 +++++++ 40 files changed, 3041 insertions(+), 211 deletions(-) rename internal/cmd/tinyooni/{utils.go => database.go} (58%) create mode 100644 internal/cmd/tinyooni/groups.json create mode 100644 internal/cmd/tinyooni/oonirun.go create mode 100644 internal/cmd/tinyooni/run.go create mode 100644 internal/cmd/tinyooni/runx.go delete mode 100644 internal/cmd/tinyooni/telegram.go delete mode 100644 internal/cmd/tinyooni/webconnectivity.go create mode 100644 internal/database/props.go create mode 100644 internal/engine/experiment/dash/main.go create mode 100644 internal/engine/experiment/fbmessenger/main.go create mode 100644 internal/engine/experiment/hhfm/main.go create mode 100644 internal/engine/experiment/hirl/main.go create mode 100644 internal/engine/experiment/ndt7/main.go create mode 100644 internal/engine/experiment/psiphon/main.go create mode 100644 internal/engine/experiment/riseupvpn/main.go create mode 100644 internal/engine/experiment/signal/main.go create mode 100644 internal/engine/experiment/tor/main.go create mode 100644 internal/engine/experiment/whatsapp/main.go create mode 100644 internal/oonirunx/link.go create mode 100644 internal/oonirunx/v1.go create mode 100644 internal/oonirunx/v2.go create mode 100644 internal/registryx/allexperiments.go create mode 100644 internal/registryx/dash.go create mode 100644 internal/registryx/factory.go create mode 100644 internal/registryx/fbmessenger.go create mode 100644 internal/registryx/hhfm.go create mode 100644 internal/registryx/hirl.go create mode 100644 internal/registryx/ndt.go create mode 100644 internal/registryx/psiphon.go create mode 100644 internal/registryx/riseupvpn.go create mode 100644 internal/registryx/signal.go create mode 100644 internal/registryx/telegram.go create mode 100644 internal/registryx/tor.go create mode 100644 internal/registryx/utils.go create mode 100644 internal/registryx/webconnectivity.go create mode 100644 internal/registryx/whatsapp.go diff --git a/internal/cmd/tinyooni/utils.go b/internal/cmd/tinyooni/database.go similarity index 58% rename from internal/cmd/tinyooni/utils.go rename to internal/cmd/tinyooni/database.go index cb7ec49fea..331e17fdef 100644 --- a/internal/cmd/tinyooni/utils.go +++ b/internal/cmd/tinyooni/database.go @@ -1,13 +1,36 @@ package main import ( + "context" "os" "path/filepath" "runtime" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/runtimex" ) +// initDatabase initializes a database and returns the corresponding database properties. +func initDatabase(ctx context.Context, sess *engine.Session, globalOptions *GlobalOptions) *database.DatabaseProps { + ooniHome := maybeGetOONIDir(globalOptions.HomeDir) + + db, err := database.Open(databasePath(ooniHome)) + runtimex.PanicOnError(err, "database.Open failed") + + networkDB, err := db.CreateNetwork(sess) + runtimex.PanicOnError(err, "db.Create failed") + + dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) + runtimex.PanicOnError(err, "db.CreateResult failed") + + return &database.DatabaseProps{ + Database: db, + DatabaseNetwork: networkDB, + DatabaseResult: dbResult, + } +} + // getHomeDir returns the $HOME directory. func getHomeDir() (string, string) { // See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da diff --git a/internal/cmd/tinyooni/groups.json b/internal/cmd/tinyooni/groups.json new file mode 100644 index 0000000000..daf21035a4 --- /dev/null +++ b/internal/cmd/tinyooni/groups.json @@ -0,0 +1,68 @@ +{ + "websites": { + "name": "Websites", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "web_connectivity" + }] + }, + "performance": { + "name": "Performance", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "dash" + }, { + "test_name": "ndt" + }] + }, + "middlebox": { + "name": "Middlebox", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "hirl" + }, { + "test_name": "hhfm" + }] + }, + "im": { + "name": "Instant Messaging", + "description": "Runs the instant messaging experiments", + "author": "OONI", + "nettests": [{ + "test_name": "facebook_messenger" + }, { + "test_name": "signal" + }, { + "test_name": "telegram" + }, { + "test_name": "whatsapp" + }] + }, + "circumvention": { + "name": "Circumvention", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "psiphon" + }, { + "test_name": "tor" + }] + }, + "experimental": { + "name": "Experimental", + "description": "Experimental nettests", + "author": "OONI", + "nettests": [{ + "test_name": "dnscheck" + }, { + "test_name": "stun_reachability" + }, { + "test_name": "torsf" + }, { + "test_name": "vanilla_tor" + }] + } +} diff --git a/internal/cmd/tinyooni/main.go b/internal/cmd/tinyooni/main.go index 1ea67a5091..c34412e0e6 100644 --- a/internal/cmd/tinyooni/main.go +++ b/internal/cmd/tinyooni/main.go @@ -141,8 +141,9 @@ func main() { rootCmd.MarkFlagsMutuallyExclusive("proxy", "tunnel") - registerWebConnectivity(rootCmd, &globalOptions) - registerTelegram(rootCmd, &globalOptions) + registerRunExperiment(rootCmd, &globalOptions) + registerRunGroup(rootCmd, &globalOptions) + registerOoniRun(rootCmd, &globalOptions) if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/internal/cmd/tinyooni/oonirun.go b/internal/cmd/tinyooni/oonirun.go new file mode 100644 index 0000000000..2f82790b76 --- /dev/null +++ b/internal/cmd/tinyooni/oonirun.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + + "github.com/ooni/probe-cli/v3/internal/oonirun" + "github.com/ooni/probe-cli/v3/internal/oonirunx" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +type oonirunOptions struct { + Inputs []string + InputFilePaths []string +} + +func registerOoniRun(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + options := &oonirunOptions{} + + subCmd := &cobra.Command{ + Use: "run", + Short: "Runs a given experiment group", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + ooniRunMain(options, globalOptions) + }, + } + rootCmd.AddCommand(subCmd) + flags := subCmd.Flags() + + flags.StringSliceVarP( + &options.Inputs, + "input", + "i", + []string{}, + "URL of the OONI Run v2 descriptor to run (may be specified multiple times)", + ) + flags.StringSliceVarP( + &options.InputFilePaths, + "input-file", + "f", + []string{}, + "Path to the OONI Run v2 descriptor to run (may be specified multiple times)", + ) +} + +func ooniRunMain(options *oonirunOptions, globalOptions *GlobalOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, globalOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + // initialize database + dbProps := initDatabase(ctx, sess, globalOptions) + + logger := sess.Logger() + cfg := &oonirunx.LinkConfig{ + AcceptChanges: globalOptions.Yes, + KVStore: sess.KeyValueStore(), + NoCollector: globalOptions.NoCollector, + NoJSON: globalOptions.NoJSON, + ReportFile: globalOptions.ReportFile, + Session: sess, + DatabaseProps: dbProps, + } + for _, URL := range options.Inputs { + r := oonirunx.NewLinkRunner(cfg, URL) + if err := r.Run(ctx); err != nil { + if errors.Is(err, oonirun.ErrNeedToAcceptChanges) { + logger.Warnf("oonirun: to accept these changes, rerun adding `-y` to the command line") + logger.Warnf("oonirun: we'll show this error every time the upstream link changes") + panic("oonirun: need to accept changes using `-y`") + } + logger.Warnf("oonirun: running link failed: %s", err.Error()) + continue + } + } + for _, filename := range options.InputFilePaths { + data, err := os.ReadFile(filename) + if err != nil { + logger.Warnf("oonirun: reading OONI Run v2 descriptor failed: %s", err.Error()) + continue + } + var descr oonirunx.V2Descriptor + if err := json.Unmarshal(data, &descr); err != nil { + logger.Warnf("oonirun: parsing OONI Run v2 descriptor failed: %s", err.Error()) + continue + } + if err := oonirunx.V2MeasureDescriptor(ctx, cfg, &descr); err != nil { + logger.Warnf("oonirun: running link failed: %s", err.Error()) + continue + } + } +} diff --git a/internal/cmd/tinyooni/run.go b/internal/cmd/tinyooni/run.go new file mode 100644 index 0000000000..5025bee607 --- /dev/null +++ b/internal/cmd/tinyooni/run.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/ooni/probe-cli/v3/internal/oonirunx" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +var ( + // TODO: we should probably have groups.json as part of the default OONI + // config in $OONIDir + pathToGroups = "./internal/cmd/tinyooni/groups.json" + + // + groups map[string]json.RawMessage +) + +func registerRunGroup(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + subCmd := &cobra.Command{ + Use: "run", + Short: "Runs a given experiment group", + Args: cobra.NoArgs, + } + rootCmd.AddCommand(subCmd) + registerGroups(subCmd, globalOptions) +} + +func registerGroups(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + data, err := os.ReadFile(pathToGroups) + runtimex.PanicOnError(err, "registerGroups failed: could not read groups.json") + + err = json.Unmarshal(data, &groups) + runtimex.PanicOnError(err, "json.Unmarshal failed") + + for name := range groups { + subCmd := &cobra.Command{ + Use: name, + Short: fmt.Sprintf("Runs the %s experiment group", name), + Run: func(cmd *cobra.Command, args []string) { + runGroupMain(cmd.Use, globalOptions) + }, + } + rootCmd.AddCommand(subCmd) + } +} + +func runGroupMain(experimentName string, globalOptions *GlobalOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, globalOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + // initialize database + dbProps := initDatabase(ctx, sess, globalOptions) + + logger := sess.Logger() + cfg := &oonirunx.LinkConfig{ + AcceptChanges: globalOptions.Yes, + KVStore: sess.KeyValueStore(), + NoCollector: globalOptions.NoCollector, + NoJSON: globalOptions.NoJSON, + ReportFile: globalOptions.ReportFile, + Session: sess, + DatabaseProps: dbProps, + } + var descr oonirunx.V2Descriptor + err = json.Unmarshal(groups[experimentName], &descr) + runtimex.PanicOnError(err, "json.Unmarshal failed") + if err := oonirunx.V2MeasureDescriptor(ctx, cfg, &descr); err != nil { + logger.Warnf("oonirun: running link failed: %s", err.Error()) + } +} diff --git a/internal/cmd/tinyooni/runx.go b/internal/cmd/tinyooni/runx.go new file mode 100644 index 0000000000..4d3356ad4a --- /dev/null +++ b/internal/cmd/tinyooni/runx.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "fmt" + + "github.com/ooni/probe-cli/v3/internal/registryx" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +func registerRunExperiment(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + subCmd := &cobra.Command{ + Use: "runx", + Short: "Runs a given experiment", + Args: cobra.NoArgs, + } + rootCmd.AddCommand(subCmd) + registerAllExperiments(subCmd, globalOptions) +} + +func registerAllExperiments(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + for name, factory := range registryx.AllExperiments { + subCmd := &cobra.Command{ + Use: name, + Short: fmt.Sprintf("Runs the %s experiment", name), + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + runExperimentsMain(cmd.Use, globalOptions) + }, + } + rootCmd.AddCommand(subCmd) + + factory.BuildFlags(subCmd.Use, subCmd) + } +} + +func runExperimentsMain(experimentName string, currentOptions *GlobalOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, currentOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + // initialize database + dbProps := initDatabase(ctx, sess, currentOptions) + + factory := registryx.AllExperiments[experimentName] + err = factory.Main(ctx, sess, dbProps) + runtimex.PanicOnError(err, fmt.Sprintf("%s.Main failed", experimentName)) +} diff --git a/internal/cmd/tinyooni/telegram.go b/internal/cmd/tinyooni/telegram.go deleted file mode 100644 index b5f9477fbe..0000000000 --- a/internal/cmd/tinyooni/telegram.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "context" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/spf13/cobra" -) - -// telegramOptions contains options for web connectivity. -type telegramOptions struct { - Annotations []string -} - -func registerTelegram(rootCmd *cobra.Command, globalOptions *GlobalOptions) { - options := &telegramOptions{} - - subCmd := &cobra.Command{ - Use: "telegram", - Short: "Runs the telegram experiment", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - telegramMain(globalOptions, options) - }, - Aliases: []string{"webconnectivity"}, - } - rootCmd.AddCommand(subCmd) - flags := subCmd.Flags() - - flags.StringSliceVarP( - &options.Annotations, - "annotation", - "A", - []string{}, - "add KEY=VALUE annotation to the report (can be repeated multiple times)", - ) -} - -func telegramMain(globalOptions *GlobalOptions, options *telegramOptions) { - ctx := context.Background() - - ooniHome := maybeGetOONIDir(globalOptions.HomeDir) - - // create a new measurement session - sess, err := newSession(ctx, globalOptions) - runtimex.PanicOnError(err, "newSession failed") - - err = sess.MaybeLookupLocationContext(ctx) - runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") - - db, err := database.Open(databasePath(ooniHome)) - runtimex.PanicOnError(err, "database.Open failed") - - networkDB, err := db.CreateNetwork(sess) - runtimex.PanicOnError(err, "db.Create failed") - - dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) - runtimex.PanicOnError(err, "db.CreateResult failed") - - args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category - Charging: true, - Callbacks: model.NewPrinterCallbacks(log.Log), - Database: db, - Inputs: nil, - MaxRuntime: 0, - MeasurementDir: dbResult.MeasurementDir, - NoCollector: false, - OnWiFi: true, - ResultID: dbResult.ID, - RunType: model.RunTypeManual, - Session: sess, - } - - err = telegram.Main(ctx, args) - runtimex.PanicOnError(err, "telegram.Main failed") -} diff --git a/internal/cmd/tinyooni/webconnectivity.go b/internal/cmd/tinyooni/webconnectivity.go deleted file mode 100644 index 4e4bf71cef..0000000000 --- a/internal/cmd/tinyooni/webconnectivity.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "context" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/spf13/cobra" -) - -// webConnectivityOptions contains options for web connectivity. -type webConnectivityOptions struct { - Annotations []string - InputFilePaths []string - Inputs []string - MaxRuntime int64 - Random bool -} - -func registerWebConnectivity(rootCmd *cobra.Command, globalOptions *GlobalOptions) { - options := &webConnectivityOptions{} - - subCmd := &cobra.Command{ - Use: "web_connectivity", - Short: "Runs the webconnectivity experiment", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - webConnectivityMain(globalOptions, options) - }, - Aliases: []string{"webconnectivity"}, - } - rootCmd.AddCommand(subCmd) - flags := subCmd.Flags() - - flags.StringSliceVarP( - &options.Annotations, - "annotation", - "A", - []string{}, - "add KEY=VALUE annotation to the report (can be repeated multiple times)", - ) - - flags.StringSliceVarP( - &options.InputFilePaths, - "input-file", - "f", - []string{}, - "path to file to supply test dependent input (may be specified multiple times)", - ) - - flags.StringSliceVarP( - &options.Inputs, - "input", - "i", - []string{}, - "add test-dependent input (may be specified multiple times)", - ) - - flags.Int64Var( - &options.MaxRuntime, - "max-runtime", - 0, - "maximum runtime in seconds for the experiment (zero means infinite)", - ) - - flags.BoolVar( - &options.Random, - "random", - false, - "randomize the inputs list", - ) -} - -func webConnectivityMain(globalOptions *GlobalOptions, options *webConnectivityOptions) { - ctx := context.Background() - - ooniHome := maybeGetOONIDir(globalOptions.HomeDir) - - // create a new measurement session - sess, err := newSession(ctx, globalOptions) - runtimex.PanicOnError(err, "newSession failed") - - err = sess.MaybeLookupLocationContext(ctx) - runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") - - db, err := database.Open(databasePath(ooniHome)) - runtimex.PanicOnError(err, "database.Open failed") - - networkDB, err := db.CreateNetwork(sess) - runtimex.PanicOnError(err, "db.Create failed") - - dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) - runtimex.PanicOnError(err, "db.CreateResult failed") - - args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category - Charging: true, - Callbacks: model.NewPrinterCallbacks(log.Log), - Database: db, - Inputs: options.Inputs, - MaxRuntime: options.MaxRuntime, - MeasurementDir: dbResult.MeasurementDir, - NoCollector: false, - OnWiFi: true, - ResultID: dbResult.ID, - RunType: model.RunTypeManual, - Session: sess, - } - - err = webconnectivity.Main(ctx, args) - runtimex.PanicOnError(err, "webconnectivity.Main failed") -} diff --git a/internal/database/props.go b/internal/database/props.go new file mode 100644 index 0000000000..645a893389 --- /dev/null +++ b/internal/database/props.go @@ -0,0 +1,18 @@ +package database + +// Database properties retreived on initialization + +import ( + "github.com/ooni/probe-cli/v3/internal/model" +) + +type DatabaseProps struct { + // + Database *Database + + // + DatabaseNetwork *model.DatabaseNetwork + + // + DatabaseResult *model.DatabaseResult +} diff --git a/internal/engine/experiment/dash/main.go b/internal/engine/experiment/dash/main.go new file mode 100644 index 0000000000..db73c16f97 --- /dev/null +++ b/internal/engine/experiment/dash/main.go @@ -0,0 +1,63 @@ +package dash + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("dash: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Dash == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Dash.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/fbmessenger/main.go b/internal/engine/experiment/fbmessenger/main.go new file mode 100644 index 0000000000..6ed4265d2f --- /dev/null +++ b/internal/engine/experiment/fbmessenger/main.go @@ -0,0 +1,63 @@ +package fbmessenger + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("fbmessenger: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.FacebookMessenger == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.FacebookMessenger.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/hhfm/main.go b/internal/engine/experiment/hhfm/main.go new file mode 100644 index 0000000000..a1c5ed35a9 --- /dev/null +++ b/internal/engine/experiment/hhfm/main.go @@ -0,0 +1,149 @@ +package hhfm + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("http_header_field_manipulation: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.HHFM == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.HHFM.ReportID + logger.Infof("ReportID: %s", reportID) + + // Obtain experiment inputs. + inputs := getInputs(args, checkInResp) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + // Create suitable stop policy. + shouldStop := newStopPolicy(args, testStartTime) + + // Create suitable progress emitter. + progresser := newProgressEmitter(args, inputs, testStartTime) + + // Measure each URL in sequence. + for inputIdx, input := range inputs { + + // Honour max runtime. + if shouldStop() { + break + } + + // Emit progress. + progresser(inputIdx, input.URL) + + // Measure the current URL. + err := experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + inputIdx, + &input, + ) + + // An error here means stuff like "cannot write to disk". + if err != nil { + return err + } + } + + return nil +} + +// getInputs obtains inputs from either args or checkInResp giving +// priority to user supplied arguments inside args. +func getInputs(args *model.ExperimentMainArgs, checkInResp *model.OOAPICheckInNettests) []model.OOAPIURLInfo { + runtimex.Assert(checkInResp.WebConnectivity != nil, "getInputs passed invalid checkInResp") + inputs := args.Inputs + if len(inputs) < 1 { + return checkInResp.WebConnectivity.URLs + } + outputs := []model.OOAPIURLInfo{} + for _, input := range inputs { + outputs = append(outputs, model.OOAPIURLInfo{ + CategoryCode: "MISC", + CountryCode: "ZZ", + URL: input, + }) + } + return outputs +} + +// newStopPolicy creates a new stop policy depending on the +// arguments passed to the experiment in args. +func newStopPolicy(args *model.ExperimentMainArgs, testStartTime time.Time) func() bool { + if args.MaxRuntime <= 0 { + return func() bool { + return false + } + } + maxRuntime := time.Duration(args.MaxRuntime) * time.Second + return func() bool { + return time.Since(testStartTime) > maxRuntime + } +} + +func newProgressEmitter( + args *model.ExperimentMainArgs, + inputs []model.OOAPIURLInfo, + testStartTime time.Time, +) func(idx int, URL string) { + total := len(inputs) + if total <= 0 { + return func(idx int, URL string) {} // just in case + } + if args.MaxRuntime <= 0 { + return func(idx int, URL string) { + percentage := 100.0 * (float64(idx) / float64(total)) + args.Callbacks.OnProgress(percentage, URL) + } + } + maxRuntime := (time.Duration(args.MaxRuntime) * time.Second) + time.Nanosecond // avoid zero division + return func(idx int, URL string) { + elapsed := time.Since(testStartTime) + percentage := 100.0 * (float64(elapsed) / float64(maxRuntime)) + args.Callbacks.OnProgress(percentage, URL) + } +} diff --git a/internal/engine/experiment/hirl/main.go b/internal/engine/experiment/hirl/main.go new file mode 100644 index 0000000000..3d4aad6451 --- /dev/null +++ b/internal/engine/experiment/hirl/main.go @@ -0,0 +1,72 @@ +package hirl + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("http_invalid_request_line: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.HIRL == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.HIRL.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{ + Config: *config, + Methods: []Method{ + randomInvalidMethod{}, + randomInvalidFieldCount{}, + randomBigRequestMethod{}, + randomInvalidVersionNumber{}, + squidCacheManager{}, + }, + } + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/ndt7/main.go b/internal/engine/experiment/ndt7/main.go new file mode 100644 index 0000000000..65415063be --- /dev/null +++ b/internal/engine/experiment/ndt7/main.go @@ -0,0 +1,67 @@ +package ndt7 + +import ( + "context" + "encoding/json" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("ndt7: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.NDT == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.NDT.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{ + config: *config, + jsonUnmarshal: json.Unmarshal, + } + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/psiphon/main.go b/internal/engine/experiment/psiphon/main.go new file mode 100644 index 0000000000..fcdae3f244 --- /dev/null +++ b/internal/engine/experiment/psiphon/main.go @@ -0,0 +1,63 @@ +package psiphon + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("psiphon: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Psiphon == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Psiphon.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/riseupvpn/main.go b/internal/engine/experiment/riseupvpn/main.go new file mode 100644 index 0000000000..16a487d16c --- /dev/null +++ b/internal/engine/experiment/riseupvpn/main.go @@ -0,0 +1,63 @@ +package riseupvpn + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("riseupvpn: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.RiseupVPN == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.RiseupVPN.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/signal/main.go b/internal/engine/experiment/signal/main.go new file mode 100644 index 0000000000..1e50308762 --- /dev/null +++ b/internal/engine/experiment/signal/main.go @@ -0,0 +1,63 @@ +package signal + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("signal: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Signal == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Signal.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/telegram/main.go b/internal/engine/experiment/telegram/main.go index 3e46fbbeb0..fc3e57c627 100644 --- a/internal/engine/experiment/telegram/main.go +++ b/internal/engine/experiment/telegram/main.go @@ -11,10 +11,10 @@ import ( ) // ErrNoCheckInInfo indicates check-in returned no suitable info. -var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") +var ErrNoCheckInInfo = errors.New("telegram: returned no check-in info") // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs) error { +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { sess := args.Session logger := sess.Logger() @@ -46,7 +46,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs) error { logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: Config{}} + measurer := &Measurer{Config: *config} // Record when we started running this nettest. testStartTime := time.Now() diff --git a/internal/engine/experiment/tor/main.go b/internal/engine/experiment/tor/main.go new file mode 100644 index 0000000000..e82d68fed3 --- /dev/null +++ b/internal/engine/experiment/tor/main.go @@ -0,0 +1,69 @@ +package tor + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("tor: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Tor == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Tor.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{ + config: *config, + fetchTorTargets: func(ctx context.Context, sess model.ExperimentSession, + cc string) (map[string]model.OOAPITorTarget, error) { + return sess.FetchTorTargets(ctx, cc) + }, + } + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/engine/experiment/whatsapp/main.go b/internal/engine/experiment/whatsapp/main.go new file mode 100644 index 0000000000..9aa730979b --- /dev/null +++ b/internal/engine/experiment/whatsapp/main.go @@ -0,0 +1,63 @@ +package whatsapp + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("whatsapp: returned no check-in info") + +// Main is the main function of the experiment. +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { + sess := args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Whatsapp == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Whatsapp.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *config} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/webconnectivity/main.go b/internal/experiment/webconnectivity/main.go index 1c003ac0fe..ae1c153588 100644 --- a/internal/experiment/webconnectivity/main.go +++ b/internal/experiment/webconnectivity/main.go @@ -15,7 +15,7 @@ import ( var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs) error { +func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { sess := args.Session logger := sess.Logger() @@ -50,7 +50,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs) error { inputs := getInputs(args, checkInResp) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: &Config{}} + measurer := &Measurer{Config: config} // Record when we started running this nettest. testStartTime := time.Now() diff --git a/internal/model/ooapi.go b/internal/model/ooapi.go index 8baa0f5247..b7d609262a 100644 --- a/internal/model/ooapi.go +++ b/internal/model/ooapi.go @@ -45,6 +45,58 @@ type OOAPICheckInConfig struct { WebConnectivity OOAPICheckInConfigWebConnectivity `json:"web_connectivity"` } +// OOAPICheckInInfoWebConnectivity contains the WebConnectivity +// part of OOAPICheckInInfo. +type OOAPICheckInInfoWebConnectivity struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` + + // URLs contains the URL to measure. + URLs []OOAPIURLInfo `json:"urls"` +} + +// OOAPICheckInInfoNDT contains the NDT +// part of OOAPICheckInInfo. +type OOAPICheckInInfoNDT struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoDash contains the Dash +// part of OOAPICheckInInfo. +type OOAPICheckInInfoDash struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoHHFM contains the HHFM +// part of OOAPICheckInInfo. +type OOAPICheckInInfoHHFM struct { + // Report ID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoHIRL contains the HIRL +// part of OOAPICheckInInfo. +type OOAPICheckInInfoHIRL struct { + // Report ID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoFacebookMessenger contains the FBMessenger +// part of OOAPICheckInInfo. +type OOAPICheckInInfoFacebookMessenger struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoSignal contains the Signal +// part of OOAPICheckInInfo. +type OOAPICheckInInfoSignal struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + // OOAPICheckInInfoTelegram contains the Telegram // part of OOAPICheckInInfo. type OOAPICheckInInfoTelegram struct { @@ -52,23 +104,112 @@ type OOAPICheckInInfoTelegram struct { ReportID string `json:"report_id"` } -// OOAPICheckInInfoWebConnectivity contains the WebConnectivity +// OOAPICheckInInfoWhatsapp contains the Whatsapp // part of OOAPICheckInInfo. -type OOAPICheckInInfoWebConnectivity struct { +type OOAPICheckInInfoWhatsapp struct { // ReportID is the report ID the probe should use. ReportID string `json:"report_id"` +} - // URLs contains the URL to measure. - URLs []OOAPIURLInfo `json:"urls"` +// OOAPICheckInInfoPsiphon contains the Psiphon +// part of OOAPICheckInInfo. +type OOAPICheckInInfoPsiphon struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoTor contains the Tor +// part of OOAPICheckInInfo. +type OOAPICheckInInfoTor struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoDNSCheck contains the DNSCheck +// part of OOAPICheckInInfo. +type OOAPICheckInInfoDNSCheck struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoStunReachability contains the StunReachability +// part of OOAPICheckInInfo. +type OOAPICheckInInfoStunReachability struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoTorsf contains the Torsf +// part of OOAPICheckInInfo. +type OOAPICheckInInfoTorsf struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoVanillaTor contains the VanillaTor +// part of OOAPICheckInInfo. +type OOAPICheckInInfoVanillaTor struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoRiseupVPN contains the RiseupVPN +// part of OOAPICheckInInfo. +type OOAPICheckInInfoRiseupVPN struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` } // OOAPICheckInNettests contains nettest information returned by the checkin API call. type OOAPICheckInNettests struct { + // WebConnectivity contains WebConnectivity related information. + WebConnectivity *OOAPICheckInInfoWebConnectivity `json:"web_connectivity"` + + // Dash contains Dash related information. + Dash *OOAPICheckInInfoDash `json:"dash"` + + // NDT contains NDT related information. + NDT *OOAPICheckInInfoNDT `json:"ndt"` + + // HHFM contains the HHFM related information. + HHFM *OOAPICheckInInfoHHFM `json:"http_header_field_manipulation"` + + // HIRL contains the HIRL related information. + HIRL *OOAPICheckInInfoHIRL `json:"http_invalid_request_line"` + + // FacebookMessenger contaings Facebook Messenger related information. + FacebookMessenger *OOAPICheckInInfoFacebookMessenger `json:"facebook_messenger"` + + // Signal contains Signal related information. + // TODO: Add Signal to the check-in API response + Signal *OOAPICheckInInfoSignal `json:"signal"` + // Telegram contains Telegram related information. Telegram *OOAPICheckInInfoTelegram `json:"telegram"` - // WebConnectivity contains WebConnectivity related information. - WebConnectivity *OOAPICheckInInfoWebConnectivity `json:"web_connectivity"` + // Whatsapp contains Whatsapp related information. + Whatsapp *OOAPICheckInInfoWhatsapp `json:"whatsapp"` + + // Psiphon contains Psiphon related information. + Psiphon *OOAPICheckInInfoPsiphon `json:"psiphon"` + + // Tor contains Tor related information. + Tor *OOAPICheckInInfoTor `json:"tor"` + + // DNSCheck contains DNSCheck related information. + DNSChck *OOAPICheckInInfoDNSCheck `json:"dnscheck"` + + // StunReachability contains StunReachability related information. + StunReachability *OOAPICheckInInfoStunReachability `json:"stun_reachability"` + + // Torsf contains Torsf related information. + Torsf *OOAPICheckInInfoTorsf `json:"torsf"` + + // VanillaTor contains VanillaTor related information. + VanillaTor *OOAPICheckInInfoVanillaTor `json:"vanilla_tor"` + + // RiseupVPN contains RiseupVPN related information. + RiseupVPN *OOAPICheckInInfoRiseupVPN `json:"riseupvpn"` } // OOAPICheckInResult is the result returned by the checkin API. diff --git a/internal/oonirunx/link.go b/internal/oonirunx/link.go new file mode 100644 index 0000000000..f0329f872b --- /dev/null +++ b/internal/oonirunx/link.go @@ -0,0 +1,86 @@ +package oonirunx + +// +// OONI Run v1 and v2 links +// + +import ( + "context" + "strings" + + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// LinkConfig contains config for an OONI Run link. You MUST fill all the fields that +// are marked as MANDATORY, or the LinkConfig would cause crashes. +type LinkConfig struct { + // AcceptChanges is OPTIONAL and tells this library that the user is + // okay with running a new or modified OONI Run link without previously + // reviewing what it contains or what has changed. + AcceptChanges bool + + // KVStore is the MANDATORY key-value store to use to keep track of + // OONI Run links and know when they are new or modified. + KVStore model.KeyValueStore + + // NoCollector OPTIONALLY indicates we should not be using any collector. + NoCollector bool + + // NoJSON OPTIONALLY indicates we don't want to save measurements to a JSON file. + NoJSON bool + + // ReportFile is the MANDATORY file in which to save reports, which is only + // used when noJSON is set to false. + ReportFile string + + // Session is the MANDATORY Session to use. + Session *engine.Session + + // DatabaseProps is the MANDATORY database properties to use + DatabaseProps *database.DatabaseProps +} + +// LinkRunner knows how to run an OONI Run v1 or v2 link. +type LinkRunner interface { + Run(ctx context.Context) error +} + +// linkRunner implements LinkRunner. +type linkRunner struct { + config *LinkConfig + f func(ctx context.Context, config *LinkConfig, URL string) error + url string +} + +// Run implements LinkRunner.Run. +func (lr *linkRunner) Run(ctx context.Context) error { + return lr.f(ctx, lr.config, lr.url) +} + +// NewLinkRunner creates a suitable link runner for the current config +// and the given URL, which is one of the following: +// +// 1. OONI Run v1 link with https scheme (e.g., https://run.ooni.io/nettest?...) +// +// 2. OONI Run v1 link with ooni scheme (e.g., ooni://nettest?...) +// +// 3. arbitrary URL of the OONI Run v2 descriptor. +func NewLinkRunner(c *LinkConfig, URL string) LinkRunner { + // TODO(bassosimone): add support for v2 deeplinks. + out := &linkRunner{ + config: c, + f: nil, + url: URL, + } + switch { + case strings.HasPrefix(URL, "https://run.ooni.io/nettest"): + out.f = v1Measure + case strings.HasPrefix(URL, "ooni://nettest"): + out.f = v1Measure + default: + out.f = v2MeasureHTTPS + } + return out +} diff --git a/internal/oonirunx/v1.go b/internal/oonirunx/v1.go new file mode 100644 index 0000000000..fe9ff00546 --- /dev/null +++ b/internal/oonirunx/v1.go @@ -0,0 +1,95 @@ +package oonirunx + +// +// OONI Run v1 implementation +// + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/registryx" +) + +var ( + // ErrInvalidV1URLScheme indicates a v1 OONI Run URL has an invalid scheme. + ErrInvalidV1URLScheme = errors.New("oonirun: invalid v1 URL scheme") + + // ErrInvalidV1URLHost indicates a v1 OONI Run URL has an invalid host. + ErrInvalidV1URLHost = errors.New("oonirun: invalid v1 URL host") + + // ErrInvalidV1URLPath indicates a v1 OONI Run URL has an invalid path. + ErrInvalidV1URLPath = errors.New("oonirun: invalid v1 URL path") + + // ErrInvalidV1URLQueryArgument indicates a v1 OONI Run URL query argument is invalid. + ErrInvalidV1URLQueryArgument = errors.New("oonirun: invalid v1 URL query argument") +) + +// v1Arguments contains arguments for a v1 OONI Run URL. These arguments are +// always encoded inside of the "ta" field, which is optional. +type v1Arguments struct { + URLs []string `json:"urls"` +} + +// v1Measure performs a measurement using the given v1 OONI Run URL. +func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { + config.Session.Logger().Infof("oonirun/v1: running %s", URL) + pu, err := url.Parse(URL) + if err != nil { + return err + } + switch pu.Scheme { + case "https": + if pu.Host != "run.ooni.io" { + return ErrInvalidV1URLHost + } + if pu.Path != "/nettest" { + return ErrInvalidV1URLPath + } + case "ooni": + if pu.Host != "nettest" { + return ErrInvalidV1URLHost + } + if pu.Path != "" && pu.Path != "/" { + return ErrInvalidV1URLPath + } + default: + return ErrInvalidV1URLScheme + } + name := pu.Query().Get("tn") + if name == "" { + return fmt.Errorf("%w: empty test name", ErrInvalidV1URLQueryArgument) + } + var inputs []string + if ta := pu.Query().Get("ta"); ta != "" { + inputs, err = v1ParseArguments(ta) + if err != nil { + return err + } + } + if mv := pu.Query().Get("mv"); mv != "1.2.0" { + return fmt.Errorf("%w: unknown minimum version", ErrInvalidV1URLQueryArgument) + } + factory := registryx.AllExperiments[name] + args := make(map[string]any) + extraOptions := make(map[string]any) // the v1 spec does not allow users to pass experiment options + return factory.Oonirun(ctx, config.Session, inputs, args, extraOptions, config.DatabaseProps) +} + +// v1ParseArguments parses the `ta` field of the query string. +func v1ParseArguments(ta string) ([]string, error) { + var inputs []string + pa, err := url.QueryUnescape(ta) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidV1URLQueryArgument, err.Error()) + } + var arguments v1Arguments + if err := json.Unmarshal([]byte(pa), &arguments); err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidV1URLQueryArgument, err.Error()) + } + inputs = arguments.URLs + return inputs, nil +} diff --git a/internal/oonirunx/v2.go b/internal/oonirunx/v2.go new file mode 100644 index 0000000000..3538c4e651 --- /dev/null +++ b/internal/oonirunx/v2.go @@ -0,0 +1,245 @@ +package oonirunx + +// +// OONI Run v2 implementation +// + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/hexops/gotextdiff" + "github.com/hexops/gotextdiff/myers" + "github.com/hexops/gotextdiff/span" + "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/httpx" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/registryx" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var ( + // v2CountEmptyNettestNames counts the number of cases in which we have been + // given an empty nettest name, which is useful for testing. + v2CountEmptyNettestNames = &atomicx.Int64{} + + // v2CountFailedExperiments countes the number of failed experiments + // and is useful when testing this package + v2CountFailedExperiments = &atomicx.Int64{} +) + +// V2Descriptor describes a list of nettests to run together. +type V2Descriptor struct { + // Name is the name of this descriptor. + Name string `json:"name"` + + // Description contains a long description. + Description string `json:"description"` + + // Author contains the author's name. + Author string `json:"author"` + + // Nettests contains the list of nettests to run. + Nettests []V2Nettest `json:"nettests"` +} + +// V2Nettest specifies how a nettest should run. +type V2Nettest struct { + // Inputs contains inputs for the experiment. + Inputs []string `json:"inputs"` + + // + Args map[string]any `json:"args"` + + // Options contains the experiment options. Any option name starting with + // `Safe` will be available for the experiment run, but omitted from + // the serialized Measurement that the experiment builder will submit + // to the OONI backend. + Options map[string]any `json:"options"` + + // TestName contains the nettest name. + TestName string `json:"test_name"` +} + +// ErrHTTPRequestFailed indicates that an HTTP request failed. +var ErrHTTPRequestFailed = errors.New("oonirun: HTTP request failed") + +// getV2DescriptorFromHTTPSURL GETs a v2Descriptor instance from +// a static URL (e.g., from a GitHub repo or from a Gist). +func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient, + logger model.Logger, URL string) (*V2Descriptor, error) { + template := httpx.APIClientTemplate{ + Accept: "", + Authorization: "", + BaseURL: URL, + HTTPClient: client, + Host: "", + LogBody: true, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + } + var desc V2Descriptor + if err := template.Build().GetJSON(ctx, "", &desc); err != nil { + return nil, err + } + return &desc, nil +} + +// v2DescriptorCache contains all the known v2Descriptor entries. +type v2DescriptorCache struct { + // Entries contains all the cached descriptors. + Entries map[string]*V2Descriptor +} + +// v2DescriptorCacheKey is the name of the kvstore2 entry keeping +// information about already known v2Descriptor instances. +const v2DescriptorCacheKey = "oonirun-v2.state" + +// v2DescriptorCacheLoad loads the v2DescriptorCache. +func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, error) { + data, err := fsstore.Get(v2DescriptorCacheKey) + if err != nil { + if errors.Is(err, kvstore.ErrNoSuchKey) { + cache := &v2DescriptorCache{ + Entries: make(map[string]*V2Descriptor), + } + return cache, nil + } + return nil, err + } + var cache v2DescriptorCache + if err := json.Unmarshal(data, &cache); err != nil { + return nil, err + } + if cache.Entries == nil { + cache.Entries = make(map[string]*V2Descriptor) + } + return &cache, nil +} + +// PullChangesWithoutSideEffects fetches v2Descriptor changes. +// +// This function DOES NOT change the state of the cache. It just returns to +// the caller what changed for a given entry. It is up-to-the-caller to choose +// what to do in case there are changes depending on the CLI flags. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation; +// +// - client is the HTTPClient to use; +// +// - URL is the URL from which to download/update the OONIRun v2Descriptor. +// +// Return values: +// +// - oldValue is the old v2Descriptor, which may be nil; +// +// - newValue is the new v2Descriptor, which may be nil; +// +// - err is the error that occurred, or nil in case of success. +func (cache *v2DescriptorCache) PullChangesWithoutSideEffects( + ctx context.Context, client model.HTTPClient, logger model.Logger, + URL string) (oldValue, newValue *V2Descriptor, err error) { + oldValue = cache.Entries[URL] + newValue, err = getV2DescriptorFromHTTPSURL(ctx, client, logger, URL) + return +} + +// Update updates the given cache entry and writes back onto the disk. +// +// Note: this method modifies cache and is not safe for concurrent usage. +func (cache *v2DescriptorCache) Update( + fsstore model.KeyValueStore, URL string, entry *V2Descriptor) error { + cache.Entries[URL] = entry + data, err := json.Marshal(cache) + runtimex.PanicOnError(err, "json.Marshal failed") + return fsstore.Set(v2DescriptorCacheKey, data) +} + +// ErrNilDescriptor indicates that we have been passed a descriptor that is nil. +var ErrNilDescriptor = errors.New("oonirun: descriptor is nil") + +// V2MeasureDescriptor performs the measurement or measurements +// described by the given list of v2Descriptor. +func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descriptor) error { + if desc == nil { + // Note: we have a test checking that we can handle a nil + // descriptor, yet adding also this extra safety net feels + // more robust in terms of the implementation. + return ErrNilDescriptor + } + logger := config.Session.Logger() + for _, nettest := range desc.Nettests { + if nettest.TestName == "" { + logger.Warn("oonirun: nettest name cannot be empty") + v2CountEmptyNettestNames.Add(1) + continue + } + factory := registryx.AllExperiments[nettest.TestName] + err := factory.Oonirun(ctx, config.Session, nettest.Inputs, nettest.Args, nettest.Options, config.DatabaseProps) + if err != nil { + logger.Warnf("cannot run experiment: %s", err.Error()) + v2CountFailedExperiments.Add(1) + continue + } + } + return nil +} + +// ErrNeedToAcceptChanges indicates that the user needs to accept +// changes (i.e., a new or modified set of descriptors) before +// we can actually run this set of descriptors. +var ErrNeedToAcceptChanges = errors.New("oonirun: need to accept changes") + +// v2DescriptorDiff shows what changed between the old and the new descriptors. +func v2DescriptorDiff(oldValue, newValue *V2Descriptor, URL string) string { + oldData, err := json.MarshalIndent(oldValue, "", " ") + runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly") + newData, err := json.MarshalIndent(newValue, "", " ") + runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly") + oldString, newString := string(oldData)+"\n", string(newData)+"\n" + oldFile := "OLD " + URL + newFile := "NEW " + URL + edits := myers.ComputeEdits(span.URIFromPath(oldFile), oldString, newString) + return fmt.Sprint(gotextdiff.ToUnified(oldFile, newFile, oldString, edits)) +} + +// v2MeasureHTTPS performs a measurement using an HTTPS v2 OONI Run URL +// and returns whether performing this measurement failed. +// +// This function maintains an on-disk cache that tracks the status of +// OONI Run v2 links. If there are any changes and the user has not +// provided config.AcceptChanges, this function will log what has changed +// and will return with an ErrNeedToAcceptChanges error. +// +// In such a case, the caller SHOULD print additional information +// explaining how to accept changes and then SHOULD exit 1 or similar. +func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error { + logger := config.Session.Logger() + logger.Infof("oonirun/v2: running %s", URL) + cache, err := v2DescriptorCacheLoad(config.KVStore) + if err != nil { + return err + } + clnt := config.Session.DefaultHTTPClient() + oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL) + if err != nil { + return err + } + diff := v2DescriptorDiff(oldValue, newValue, URL) + if !config.AcceptChanges && diff != "" { + logger.Warnf("oonirun: %s changed as follows:\n\n%s", URL, diff) + logger.Warnf("oonirun: we are not going to run this link until you accept changes") + return ErrNeedToAcceptChanges + } + if diff != "" { + if err := cache.Update(config.KVStore, URL, newValue); err != nil { + return err + } + } + return V2MeasureDescriptor(ctx, config, newValue) // handles nil newValue gracefully +} diff --git a/internal/registryx/allexperiments.go b/internal/registryx/allexperiments.go new file mode 100644 index 0000000000..56216b62b3 --- /dev/null +++ b/internal/registryx/allexperiments.go @@ -0,0 +1,12 @@ +package registryx + +// Where we register all the available experiments. +var AllExperiments = map[string]*Factory{} + +// ExperimentNames returns the name of all experiments +func ExperimentNames() (names []string) { + for key := range AllExperiments { + names = append(names, key) + } + return +} diff --git a/internal/registryx/dash.go b/internal/registryx/dash.go new file mode 100644 index 0000000000..50f5212ede --- /dev/null +++ b/internal/registryx/dash.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dash" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type dashOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &dashOptions{} + AllExperiments["dash"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &dash.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return dashMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &dashOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &dash.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return dashMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + dashBuildFlags(experimentName, rootCmd, options, &dash.Config{}) + }, + } +} + +func dashMain(ctx context.Context, sess model.ExperimentSession, options *dashOptions, + config *dash.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return dash.Main(ctx, args, config) +} + +func dashBuildFlags(experimentName string, rootCmd *cobra.Command, + options *dashOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/factory.go b/internal/registryx/factory.go new file mode 100644 index 0000000000..2e18463639 --- /dev/null +++ b/internal/registryx/factory.go @@ -0,0 +1,196 @@ +package registryx + +import ( + "context" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +// Factory is a forwarder for the respective experiment's main +type Factory struct { + // Main calls the experiment.Main functions + Main func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error + + // + Oonirun func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error + + // BuildFlags initializes the experiment specific flags + BuildFlags func(experimentName string, rootCmd *cobra.Command) +} + +var ( + // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. + ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") + + // ErrNoSuchField indicates there's no field with the given name. + ErrNoSuchField = errors.New("no such field") + + // ErrCannotSetIntegerOption means SetOptionAny couldn't set an integer option. + ErrCannotSetIntegerOption = errors.New("cannot set integer option") + + // ErrInvalidStringRepresentationOfBool indicates the string you passed + // to SetOptionaAny is not a valid string representation of a bool. + ErrInvalidStringRepresentationOfBool = errors.New("invalid string representation of bool") + + // ErrCannotSetBoolOption means SetOptionAny couldn't set a bool option. + ErrCannotSetBoolOption = errors.New("cannot set bool option") + + // ErrCannotSetStringOption means SetOptionAny couldn't set a string option. + ErrCannotSetStringOption = errors.New("cannot set string option") + + // ErrUnsupportedOptionType means we don't support the type passed to + // the SetOptionAny method as an opaque any type. + ErrUnsupportedOptionType = errors.New("unsupported option type") +) + +// options returns the options exposed by this experiment. +func options(config any) (map[string]model.ExperimentOptionInfo, error) { + result := make(map[string]model.ExperimentOptionInfo) + ptrinfo := reflect.ValueOf(config) + if ptrinfo.Kind() != reflect.Ptr { + return nil, ErrConfigIsNotAStructPointer + } + structinfo := ptrinfo.Elem().Type() + if structinfo.Kind() != reflect.Struct { + return nil, ErrConfigIsNotAStructPointer + } + for i := 0; i < structinfo.NumField(); i++ { + field := structinfo.Field(i) + result[field.Name] = model.ExperimentOptionInfo{ + Doc: field.Tag.Get("ooni"), + Type: field.Type.String(), + } + } + return result, nil +} + +// setOptionBool sets a bool option. +func setOptionBool(field reflect.Value, value any) error { + switch v := value.(type) { + case bool: + field.SetBool(v) + return nil + case string: + if v != "true" && v != "false" { + return fmt.Errorf("%w: %s", ErrInvalidStringRepresentationOfBool, v) + } + field.SetBool(v == "true") + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetBoolOption, value) + } +} + +// setOptionInt sets an int option +func setOptionInt(field reflect.Value, value any) error { + switch v := value.(type) { + case int64: + field.SetInt(v) + return nil + case int32: + field.SetInt(int64(v)) + return nil + case int16: + field.SetInt(int64(v)) + return nil + case int8: + field.SetInt(int64(v)) + return nil + case int: + field.SetInt(int64(v)) + return nil + case string: + number, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("%w: %s", ErrCannotSetIntegerOption, err.Error()) + } + field.SetInt(number) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetIntegerOption, value) + } +} + +// setOptionString sets a string option +func setOptionString(field reflect.Value, value any) error { + switch v := value.(type) { + case string: + field.SetString(v) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetStringOption, value) + } +} + +// setOptionAny sets an option given any value. +func setOptionAny(config any, key string, value any) error { + field, err := fieldbyname(config, key) + if err != nil { + return err + } + switch field.Kind() { + case reflect.Int64: + return setOptionInt(field, value) + case reflect.Bool: + return setOptionBool(field, value) + case reflect.String: + return setOptionString(field, value) + default: + return fmt.Errorf("%w: %T", ErrUnsupportedOptionType, value) + } +} + +// SetOptionsAny calls SetOptionAny for each entry inside [options]. +func setOptionsAny(config any, options map[string]any) error { + for key, value := range options { + if err := setOptionAny(config, key, value); err != nil { + return err + } + } + return nil +} + +// fieldbyname return v's field whose name is equal to the given key. +func fieldbyname(v interface{}, key string) (reflect.Value, error) { + // See https://stackoverflow.com/a/6396678/4354461 + ptrinfo := reflect.ValueOf(v) + if ptrinfo.Kind() != reflect.Ptr { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + structinfo := ptrinfo.Elem() + if structinfo.Kind() != reflect.Struct { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + field := structinfo.FieldByName(key) + if !field.IsValid() || !field.CanSet() { + return reflect.Value{}, fmt.Errorf("%w: %s", ErrNoSuchField, key) + } + return field, nil +} + +func documentationForOptions(name string, config any) string { + var sb strings.Builder + options, err := options(config) + if err != nil || len(options) < 1 { + return "" + } + fmt.Fprint(&sb, "Pass KEY=VALUE options to the experiment. Available options:\n") + for name, info := range options { + if info.Doc == "" { + continue + } + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, " -O, --option %s=<%s>\n", name, info.Type) + fmt.Fprintf(&sb, " %s\n", info.Doc) + } + return sb.String() +} diff --git a/internal/registryx/fbmessenger.go b/internal/registryx/fbmessenger.go new file mode 100644 index 0000000000..fac2d49b7c --- /dev/null +++ b/internal/registryx/fbmessenger.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type fbmessengerOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &fbmessengerOptions{} + AllExperiments["facebook_messenger"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &fbmessenger.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return fbmessengerMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &fbmessengerOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &fbmessenger.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return fbmessengerMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + fbmessengerBuildFlags(experimentName, rootCmd, options, &fbmessenger.Config{}) + }, + } +} + +func fbmessengerMain(ctx context.Context, sess model.ExperimentSession, options *fbmessengerOptions, + config *fbmessenger.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return fbmessenger.Main(ctx, args, config) +} + +func fbmessengerBuildFlags(experimentName string, rootCmd *cobra.Command, + options *fbmessengerOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/hhfm.go b/internal/registryx/hhfm.go new file mode 100644 index 0000000000..cc83500c79 --- /dev/null +++ b/internal/registryx/hhfm.go @@ -0,0 +1,126 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +// hhfmOptions contains options for web connectivity. +type hhfmOptions struct { + Annotations []string + InputFilePaths []string + Inputs []string + MaxRuntime int64 + Random bool + ConfigOptions []string +} + +func init() { + options := &hhfmOptions{} + AllExperiments["hhfm"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &hhfm.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return hhfmMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &hhfmOptions{} + options.Inputs = inputs + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &hhfm.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return hhfmMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + hhfmBuildFlags(experimentName, rootCmd, options, &hhfm.Config{}) + }, + } +} + +func hhfmMain(ctx context.Context, sess model.ExperimentSession, options *hhfmOptions, + config *hhfm.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: options.Inputs, + MaxRuntime: options.MaxRuntime, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + + return hhfm.Main(ctx, args, config) +} + +func hhfmBuildFlags(experimentName string, rootCmd *cobra.Command, + options *hhfmOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + flags.StringSliceVarP( + &options.InputFilePaths, + "input-file", + "f", + []string{}, + "path to file to supply test dependent input (may be specified multiple times)", + ) + + flags.StringSliceVarP( + &options.Inputs, + "input", + "i", + []string{}, + "add test-dependent input (may be specified multiple times)", + ) + + flags.Int64Var( + &options.MaxRuntime, + "max-runtime", + 0, + "maximum runtime in seconds for the experiment (zero means infinite)", + ) + + flags.BoolVar( + &options.Random, + "random", + false, + "randomize the inputs list", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/hirl.go b/internal/registryx/hirl.go new file mode 100644 index 0000000000..2964679e0d --- /dev/null +++ b/internal/registryx/hirl.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type hirlOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &hirlOptions{} + AllExperiments["hirl"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &hirl.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return hirlMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &hirlOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &hirl.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return hirlMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + hirlBuildFlags(experimentName, rootCmd, options, &hirl.Config{}) + }, + } +} + +func hirlMain(ctx context.Context, sess model.ExperimentSession, options *hirlOptions, + config *hirl.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return hirl.Main(ctx, args, config) +} + +func hirlBuildFlags(experimentName string, rootCmd *cobra.Command, + options *hirlOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/ndt.go b/internal/registryx/ndt.go new file mode 100644 index 0000000000..e632da2756 --- /dev/null +++ b/internal/registryx/ndt.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type ndtOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &ndtOptions{} + AllExperiments["ndt"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &ndt7.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return ndtMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &ndtOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &ndt7.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return ndtMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + ndtBuildFlags(experimentName, rootCmd, options, &ndt7.Config{}) + }, + } +} + +func ndtMain(ctx context.Context, sess model.ExperimentSession, options *ndtOptions, + config *ndt7.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return ndt7.Main(ctx, args, config) +} + +func ndtBuildFlags(experimentName string, rootCmd *cobra.Command, + options *ndtOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/psiphon.go b/internal/registryx/psiphon.go new file mode 100644 index 0000000000..dd9cbcf82d --- /dev/null +++ b/internal/registryx/psiphon.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type psiphonOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &psiphonOptions{} + AllExperiments["psiphon"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &psiphon.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return psiphonMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &psiphonOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &psiphon.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return psiphonMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + psiphonBuildFlags(experimentName, rootCmd, options, &psiphon.Config{}) + }, + } +} + +func psiphonMain(ctx context.Context, sess model.ExperimentSession, options *psiphonOptions, + config *psiphon.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return psiphon.Main(ctx, args, config) +} + +func psiphonBuildFlags(experimentName string, rootCmd *cobra.Command, + options *psiphonOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/riseupvpn.go b/internal/registryx/riseupvpn.go new file mode 100644 index 0000000000..f6fb45c49b --- /dev/null +++ b/internal/registryx/riseupvpn.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type riseupvpnOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &riseupvpnOptions{} + AllExperiments["riseupvpn"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &riseupvpn.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return riseupvpnMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &riseupvpnOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &riseupvpn.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return riseupvpnMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + riseupvpnBuildFlags(experimentName, rootCmd, options, &riseupvpn.Config{}) + }, + } +} + +func riseupvpnMain(ctx context.Context, sess model.ExperimentSession, options *riseupvpnOptions, + config *riseupvpn.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return riseupvpn.Main(ctx, args, config) +} + +func riseupvpnBuildFlags(experimentName string, rootCmd *cobra.Command, + options *riseupvpnOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/signal.go b/internal/registryx/signal.go new file mode 100644 index 0000000000..2991b92d65 --- /dev/null +++ b/internal/registryx/signal.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/signal" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type signalOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &signalOptions{} + AllExperiments["signal"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &signal.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return signalMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &signalOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &signal.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return signalMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + signalBuildFlags(experimentName, rootCmd, options, &signal.Config{}) + }, + } +} + +func signalMain(ctx context.Context, sess model.ExperimentSession, options *signalOptions, + config *signal.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return signal.Main(ctx, args, config) +} + +func signalBuildFlags(experimentName string, rootCmd *cobra.Command, + options *signalOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/telegram.go b/internal/registryx/telegram.go new file mode 100644 index 0000000000..77de5c9dfd --- /dev/null +++ b/internal/registryx/telegram.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type telegramOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &telegramOptions{} + AllExperiments["telegram"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &telegram.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return telegramMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &telegramOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &telegram.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return telegramMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + telegramBuildFlags(experimentName, rootCmd, options, &telegram.Config{}) + }, + } +} + +func telegramMain(ctx context.Context, sess model.ExperimentSession, options *telegramOptions, + config *telegram.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return telegram.Main(ctx, args, config) +} + +func telegramBuildFlags(experimentName string, rootCmd *cobra.Command, + options *telegramOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/tor.go b/internal/registryx/tor.go new file mode 100644 index 0000000000..5c64640360 --- /dev/null +++ b/internal/registryx/tor.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/tor" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type torOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &torOptions{} + AllExperiments["tor"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &tor.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return torMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &torOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &tor.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return torMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + torBuildFlags(experimentName, rootCmd, options, &tor.Config{}) + }, + } +} + +func torMain(ctx context.Context, sess model.ExperimentSession, options *torOptions, + config *tor.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return tor.Main(ctx, args, config) +} + +func torBuildFlags(experimentName string, rootCmd *cobra.Command, + options *torOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/utils.go b/internal/registryx/utils.go new file mode 100644 index 0000000000..cc8bdeaadf --- /dev/null +++ b/internal/registryx/utils.go @@ -0,0 +1,30 @@ +package registryx + +import ( + "errors" + "strings" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// splitPair takes in input a string in the form KEY=VALUE and splits it. This +// function returns an error if it cannot find the = character to split the string. +func splitPair(s string) (string, string, error) { + v := strings.SplitN(s, "=", 2) + if len(v) != 2 { + return "", "", errors.New("invalid key-value pair") + } + return v[0], v[1], nil +} + +// mustMakeMapStringAny makes a map from string to any using as input a list +// of key-value pairs used to initialize the map, or panics on error +func mustMakeMapStringAny(input []string) (output map[string]any) { + output = make(map[string]any) + for _, opt := range input { + key, value, err := splitPair(opt) + runtimex.PanicOnError(err, "cannot split key-value pair") + output[key] = value + } + return +} diff --git a/internal/registryx/webconnectivity.go b/internal/registryx/webconnectivity.go new file mode 100644 index 0000000000..9c1d4beee0 --- /dev/null +++ b/internal/registryx/webconnectivity.go @@ -0,0 +1,126 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +// webConnectivityOptions contains options for web connectivity. +type webConnectivityOptions struct { + Annotations []string + InputFilePaths []string + Inputs []string + MaxRuntime int64 + Random bool + ConfigOptions []string +} + +func init() { + options := &webConnectivityOptions{} + AllExperiments["web_connectivity"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &webconnectivity.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return webconnectivityMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &webConnectivityOptions{} + options.Inputs = inputs + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &webconnectivity.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return webconnectivityMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + webconnectivityBuildFlags(experimentName, rootCmd, options, &webconnectivity.Config{}) + }, + } +} + +func webconnectivityMain(ctx context.Context, sess model.ExperimentSession, options *webConnectivityOptions, + config *webconnectivity.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: options.Inputs, + MaxRuntime: options.MaxRuntime, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + + return webconnectivity.Main(ctx, args, config) +} + +func webconnectivityBuildFlags(experimentName string, rootCmd *cobra.Command, + options *webConnectivityOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + flags.StringSliceVarP( + &options.InputFilePaths, + "input-file", + "f", + []string{}, + "path to file to supply test dependent input (may be specified multiple times)", + ) + + flags.StringSliceVarP( + &options.Inputs, + "input", + "i", + []string{}, + "add test-dependent input (may be specified multiple times)", + ) + + flags.Int64Var( + &options.MaxRuntime, + "max-runtime", + 0, + "maximum runtime in seconds for the experiment (zero means infinite)", + ) + + flags.BoolVar( + &options.Random, + "random", + false, + "randomize the inputs list", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/whatsapp.go b/internal/registryx/whatsapp.go new file mode 100644 index 0000000000..f9311eabad --- /dev/null +++ b/internal/registryx/whatsapp.go @@ -0,0 +1,89 @@ +package registryx + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +type whatsappOptions struct { + Annotations []string + ConfigOptions []string +} + +func init() { + options := &whatsappOptions{} + AllExperiments["whatsapp"] = &Factory{ + Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { + config := &whatsapp.Config{} + configMap := mustMakeMapStringAny(options.ConfigOptions) + if err := setOptionsAny(config, configMap); err != nil { + return err + } + return whatsappMain(ctx, sess, options, config, db) + }, + Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, + args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { + options := &whatsappOptions{} + if err := setOptionsAny(options, args); err != nil { + return err + } + config := &whatsapp.Config{} + if err := setOptionsAny(config, extraOptions); err != nil { + return err + } + return whatsappMain(ctx, sess, options, config, db) + }, + BuildFlags: func(experimentName string, rootCmd *cobra.Command) { + whatsappBuildFlags(experimentName, rootCmd, options, &whatsapp.Config{}) + }, + } +} + +func whatsappMain(ctx context.Context, sess model.ExperimentSession, options *whatsappOptions, + config *whatsapp.Config, db *database.DatabaseProps) error { + args := &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } + return whatsapp.Main(ctx, args, config) +} + +func whatsappBuildFlags(experimentName string, rootCmd *cobra.Command, + options *whatsappOptions, config any) { + flags := rootCmd.Flags() + + flags.StringSliceVarP( + &options.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := documentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &options.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} From d7c3a13d4965d4423294846082d5a7e72368ba1d Mon Sep 17 00:00:00 2001 From: DecFox Date: Mon, 16 Jan 2023 08:42:21 +0530 Subject: [PATCH 09/11] handle experiment annotations --- internal/registryx/dash.go | 5 +++-- internal/registryx/fbmessenger.go | 5 +++-- internal/registryx/hhfm.go | 5 +++-- internal/registryx/hirl.go | 5 +++-- internal/registryx/ndt.go | 5 +++-- internal/registryx/psiphon.go | 5 +++-- internal/registryx/riseupvpn.go | 5 +++-- internal/registryx/signal.go | 5 +++-- internal/registryx/telegram.go | 5 +++-- internal/registryx/tor.go | 5 +++-- internal/registryx/utils.go | 12 ++++++++++++ 11 files changed, 42 insertions(+), 20 deletions(-) diff --git a/internal/registryx/dash.go b/internal/registryx/dash.go index 50f5212ede..679a5f6a1f 100644 --- a/internal/registryx/dash.go +++ b/internal/registryx/dash.go @@ -47,9 +47,10 @@ func init() { func dashMain(ctx context.Context, sess model.ExperimentSession, options *dashOptions, config *dash.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/fbmessenger.go b/internal/registryx/fbmessenger.go index fac2d49b7c..d8bc4e29e9 100644 --- a/internal/registryx/fbmessenger.go +++ b/internal/registryx/fbmessenger.go @@ -47,9 +47,10 @@ func init() { func fbmessengerMain(ctx context.Context, sess model.ExperimentSession, options *fbmessengerOptions, config *fbmessenger.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/hhfm.go b/internal/registryx/hhfm.go index cc83500c79..267cd4fad6 100644 --- a/internal/registryx/hhfm.go +++ b/internal/registryx/hhfm.go @@ -53,9 +53,10 @@ func init() { func hhfmMain(ctx context.Context, sess model.ExperimentSession, options *hhfmOptions, config *hhfm.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/hirl.go b/internal/registryx/hirl.go index 2964679e0d..4548eea6cf 100644 --- a/internal/registryx/hirl.go +++ b/internal/registryx/hirl.go @@ -47,9 +47,10 @@ func init() { func hirlMain(ctx context.Context, sess model.ExperimentSession, options *hirlOptions, config *hirl.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/ndt.go b/internal/registryx/ndt.go index e632da2756..c70d7604f5 100644 --- a/internal/registryx/ndt.go +++ b/internal/registryx/ndt.go @@ -47,9 +47,10 @@ func init() { func ndtMain(ctx context.Context, sess model.ExperimentSession, options *ndtOptions, config *ndt7.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/psiphon.go b/internal/registryx/psiphon.go index dd9cbcf82d..d34c339638 100644 --- a/internal/registryx/psiphon.go +++ b/internal/registryx/psiphon.go @@ -47,9 +47,10 @@ func init() { func psiphonMain(ctx context.Context, sess model.ExperimentSession, options *psiphonOptions, config *psiphon.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/riseupvpn.go b/internal/registryx/riseupvpn.go index f6fb45c49b..0f93000ba4 100644 --- a/internal/registryx/riseupvpn.go +++ b/internal/registryx/riseupvpn.go @@ -47,9 +47,10 @@ func init() { func riseupvpnMain(ctx context.Context, sess model.ExperimentSession, options *riseupvpnOptions, config *riseupvpn.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/signal.go b/internal/registryx/signal.go index 2991b92d65..5d5128c5d3 100644 --- a/internal/registryx/signal.go +++ b/internal/registryx/signal.go @@ -47,9 +47,10 @@ func init() { func signalMain(ctx context.Context, sess model.ExperimentSession, options *signalOptions, config *signal.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/telegram.go b/internal/registryx/telegram.go index 77de5c9dfd..0bda1d86fd 100644 --- a/internal/registryx/telegram.go +++ b/internal/registryx/telegram.go @@ -47,9 +47,10 @@ func init() { func telegramMain(ctx context.Context, sess model.ExperimentSession, options *telegramOptions, config *telegram.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/tor.go b/internal/registryx/tor.go index 5c64640360..00e502fd49 100644 --- a/internal/registryx/tor.go +++ b/internal/registryx/tor.go @@ -47,9 +47,10 @@ func init() { func torMain(ctx context.Context, sess model.ExperimentSession, options *torOptions, config *tor.Config, db *database.DatabaseProps) error { + annotations := mustMakeMapStringString(options.Annotations) args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, diff --git a/internal/registryx/utils.go b/internal/registryx/utils.go index cc8bdeaadf..3e7703f2dd 100644 --- a/internal/registryx/utils.go +++ b/internal/registryx/utils.go @@ -28,3 +28,15 @@ func mustMakeMapStringAny(input []string) (output map[string]any) { } return } + +// mustMakeMapStringString makes a map from string to string using as input a list +// of key-value pairs used to initialize the map, or panics on error +func mustMakeMapStringString(input []string) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + key, value, err := splitPair(opt) + runtimex.PanicOnError(err, "cannot split key-value pair") + output[key] = value + } + return +} From 8e003936048b936159abf3c507bbdd38ba506d0b Mon Sep 17 00:00:00 2001 From: DecFox Date: Thu, 2 Feb 2023 09:56:43 +0530 Subject: [PATCH 10/11] resolve merge conflicts --- internal/experiment/experiment.go | 2 +- internal/experiment/webconnectivity/main.go | 4 ++-- internal/legacy/mockable/mockable.go | 2 +- internal/model/experiment.go | 2 +- internal/oonirunx/v2.go | 6 +++--- internal/registryx/dash.go | 2 +- internal/registryx/fbmessenger.go | 2 +- internal/registryx/hhfm.go | 2 +- internal/registryx/hirl.go | 2 +- internal/registryx/ndt.go | 2 +- internal/registryx/psiphon.go | 2 +- internal/registryx/riseupvpn.go | 2 +- internal/registryx/signal.go | 2 +- internal/registryx/telegram.go | 2 +- internal/registryx/tor.go | 2 +- internal/registryx/whatsapp.go | 2 +- 16 files changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go index 689840f203..410199fea4 100644 --- a/internal/experiment/experiment.go +++ b/internal/experiment/experiment.go @@ -24,7 +24,7 @@ func CallCheckIn( ctx context.Context, args *model.ExperimentMainArgs, sess model.ExperimentSession, -) (*model.OOAPICheckInNettests, error) { +) (*model.OOAPICheckInResultNettests, error) { return sess.CheckIn(ctx, &model.OOAPICheckInConfig{ Charging: args.Charging, OnWiFi: args.OnWiFi, diff --git a/internal/experiment/webconnectivity/main.go b/internal/experiment/webconnectivity/main.go index ae1c153588..0a20251dd2 100644 --- a/internal/experiment/webconnectivity/main.go +++ b/internal/experiment/webconnectivity/main.go @@ -50,7 +50,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e inputs := getInputs(args, checkInResp) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: config} + measurer := &Measurer{Config: *config} // Record when we started running this nettest. testStartTime := time.Now() @@ -94,7 +94,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e // getInputs obtains inputs from either args or checkInResp giving // priority to user supplied arguments inside args. -func getInputs(args *model.ExperimentMainArgs, checkInResp *model.OOAPICheckInNettests) []model.OOAPIURLInfo { +func getInputs(args *model.ExperimentMainArgs, checkInResp *model.OOAPICheckInResultNettests) []model.OOAPIURLInfo { runtimex.Assert(checkInResp.WebConnectivity != nil, "getInputs passed invalid checkInResp") inputs := args.Inputs if len(inputs) < 1 { diff --git a/internal/legacy/mockable/mockable.go b/internal/legacy/mockable/mockable.go index d415390eb8..004b7e667f 100644 --- a/internal/legacy/mockable/mockable.go +++ b/internal/legacy/mockable/mockable.go @@ -145,7 +145,7 @@ func (sess *Session) UserAgent() string { return sess.MockableUserAgent } -func (sess *Session) CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInNettests, error) { +func (sess *Session) CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { panic("not implemented") } diff --git a/internal/model/experiment.go b/internal/model/experiment.go index 1ab02cc784..813265851e 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -12,7 +12,7 @@ import ( // ExperimentSession is the experiment's view of a session. type ExperimentSession interface { // CheckIn invokes the check-in API. - CheckIn(ctx context.Context, config *OOAPICheckInConfig) (*OOAPICheckInNettests, error) + CheckIn(ctx context.Context, config *OOAPICheckInConfig) (*OOAPICheckInResultNettests, error) // DefaultHTTPClient returns the default HTTPClient used by the session. DefaultHTTPClient() HTTPClient diff --git a/internal/oonirunx/v2.go b/internal/oonirunx/v2.go index 3538c4e651..72c560e8a2 100644 --- a/internal/oonirunx/v2.go +++ b/internal/oonirunx/v2.go @@ -9,11 +9,11 @@ import ( "encoding/json" "errors" "fmt" + "sync/atomic" "github.com/hexops/gotextdiff" "github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/span" - "github.com/ooni/probe-cli/v3/internal/atomicx" "github.com/ooni/probe-cli/v3/internal/httpx" "github.com/ooni/probe-cli/v3/internal/kvstore" "github.com/ooni/probe-cli/v3/internal/model" @@ -24,11 +24,11 @@ import ( var ( // v2CountEmptyNettestNames counts the number of cases in which we have been // given an empty nettest name, which is useful for testing. - v2CountEmptyNettestNames = &atomicx.Int64{} + v2CountEmptyNettestNames = &atomic.Int64{} // v2CountFailedExperiments countes the number of failed experiments // and is useful when testing this package - v2CountFailedExperiments = &atomicx.Int64{} + v2CountFailedExperiments = &atomic.Int64{} ) // V2Descriptor describes a list of nettests to run together. diff --git a/internal/registryx/dash.go b/internal/registryx/dash.go index 679a5f6a1f..84b7bac140 100644 --- a/internal/registryx/dash.go +++ b/internal/registryx/dash.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/dash" + "github.com/ooni/probe-cli/v3/internal/experiment/dash" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/fbmessenger.go b/internal/registryx/fbmessenger.go index d8bc4e29e9..b24333964f 100644 --- a/internal/registryx/fbmessenger.go +++ b/internal/registryx/fbmessenger.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger" + "github.com/ooni/probe-cli/v3/internal/experiment/fbmessenger" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/hhfm.go b/internal/registryx/hhfm.go index 267cd4fad6..7f62504330 100644 --- a/internal/registryx/hhfm.go +++ b/internal/registryx/hhfm.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm" + "github.com/ooni/probe-cli/v3/internal/experiment/hhfm" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/hirl.go b/internal/registryx/hirl.go index 4548eea6cf..4aa5a5117b 100644 --- a/internal/registryx/hirl.go +++ b/internal/registryx/hirl.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl" + "github.com/ooni/probe-cli/v3/internal/experiment/hirl" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/ndt.go b/internal/registryx/ndt.go index c70d7604f5..1f2e6c2388 100644 --- a/internal/registryx/ndt.go +++ b/internal/registryx/ndt.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7" + "github.com/ooni/probe-cli/v3/internal/experiment/ndt7" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/psiphon.go b/internal/registryx/psiphon.go index d34c339638..cb6a337a35 100644 --- a/internal/registryx/psiphon.go +++ b/internal/registryx/psiphon.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon" + "github.com/ooni/probe-cli/v3/internal/experiment/psiphon" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/riseupvpn.go b/internal/registryx/riseupvpn.go index 0f93000ba4..fd982d3c4b 100644 --- a/internal/registryx/riseupvpn.go +++ b/internal/registryx/riseupvpn.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn" + "github.com/ooni/probe-cli/v3/internal/experiment/riseupvpn" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/signal.go b/internal/registryx/signal.go index 5d5128c5d3..76858638fb 100644 --- a/internal/registryx/signal.go +++ b/internal/registryx/signal.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/signal" + "github.com/ooni/probe-cli/v3/internal/experiment/signal" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/telegram.go b/internal/registryx/telegram.go index 0bda1d86fd..4893af6e81 100644 --- a/internal/registryx/telegram.go +++ b/internal/registryx/telegram.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/experiment/telegram" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/tor.go b/internal/registryx/tor.go index 00e502fd49..bfff71ec3f 100644 --- a/internal/registryx/tor.go +++ b/internal/registryx/tor.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/tor" + "github.com/ooni/probe-cli/v3/internal/experiment/tor" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) diff --git a/internal/registryx/whatsapp.go b/internal/registryx/whatsapp.go index f9311eabad..61650a3f47 100644 --- a/internal/registryx/whatsapp.go +++ b/internal/registryx/whatsapp.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp" + "github.com/ooni/probe-cli/v3/internal/experiment/whatsapp" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) From d4580b50e3dd072aaf1f4ab57fb50bbbf3fd959b Mon Sep 17 00:00:00 2001 From: DecFox Date: Sun, 5 Feb 2023 22:51:23 +0530 Subject: [PATCH 11/11] make experiment-specific the global truth --- internal/cmd/tinyooni/database.go | 5 +- internal/cmd/tinyooni/runx.go | 8 +- internal/experiment/dash/main.go | 38 +++- internal/experiment/fbmessenger/main.go | 39 +++- internal/experiment/hhfm/main.go | 45 ++++- internal/experiment/hirl/main.go | 39 +++- internal/experiment/ndt7/main.go | 39 +++- internal/experiment/psiphon/main.go | 39 +++- internal/experiment/riseupvpn/main.go | 39 +++- internal/experiment/signal/main.go | 39 +++- internal/experiment/telegram/main.go | 39 +++- internal/experiment/tor/main.go | 39 +++- internal/experiment/webconnectivity/main.go | 45 ++++- internal/experiment/whatsapp/main.go | 39 +++- internal/model/database.go | 12 ++ internal/model/experiment.go | 17 ++ internal/ooapi/checkin.go | 2 +- internal/oonirunx/link.go | 3 +- internal/oonirunx/v1.go | 7 +- internal/oonirunx/v2.go | 7 +- internal/registryx/allexperiments.go | 6 +- internal/registryx/dash.go | 68 +++---- internal/registryx/factory.go | 192 ++------------------ internal/registryx/fbmessenger.go | 72 ++++---- internal/registryx/hhfm.go | 80 ++++---- internal/registryx/hirl.go | 68 +++---- internal/registryx/ndt.go | 68 +++---- internal/registryx/psiphon.go | 68 +++---- internal/registryx/riseupvpn.go | 68 +++---- internal/registryx/signal.go | 72 ++++---- internal/registryx/telegram.go | 68 +++---- internal/registryx/tor.go | 66 +++---- internal/registryx/webconnectivity.go | 75 +++----- internal/registryx/whatsapp.go | 71 ++++---- internal/setter/setter.go | 180 ++++++++++++++++++ internal/setter/setter_test.go | 7 + 36 files changed, 999 insertions(+), 770 deletions(-) create mode 100644 internal/setter/setter.go create mode 100644 internal/setter/setter_test.go diff --git a/internal/cmd/tinyooni/database.go b/internal/cmd/tinyooni/database.go index 331e17fdef..c3744aff3a 100644 --- a/internal/cmd/tinyooni/database.go +++ b/internal/cmd/tinyooni/database.go @@ -8,11 +8,12 @@ import ( "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" ) // initDatabase initializes a database and returns the corresponding database properties. -func initDatabase(ctx context.Context, sess *engine.Session, globalOptions *GlobalOptions) *database.DatabaseProps { +func initDatabase(ctx context.Context, sess *engine.Session, globalOptions *GlobalOptions) *model.DatabaseProps { ooniHome := maybeGetOONIDir(globalOptions.HomeDir) db, err := database.Open(databasePath(ooniHome)) @@ -24,7 +25,7 @@ func initDatabase(ctx context.Context, sess *engine.Session, globalOptions *Glob dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) runtimex.PanicOnError(err, "db.CreateResult failed") - return &database.DatabaseProps{ + return &model.DatabaseProps{ Database: db, DatabaseNetwork: networkDB, DatabaseResult: dbResult, diff --git a/internal/cmd/tinyooni/runx.go b/internal/cmd/tinyooni/runx.go index 4d3356ad4a..9f8c06ba2a 100644 --- a/internal/cmd/tinyooni/runx.go +++ b/internal/cmd/tinyooni/runx.go @@ -31,7 +31,10 @@ func registerAllExperiments(rootCmd *cobra.Command, globalOptions *GlobalOptions } rootCmd.AddCommand(subCmd) - factory.BuildFlags(subCmd.Use, subCmd) + // build experiment specific flags here + options := registryx.AllExperimentOptions[subCmd.Use] + options.BuildFlags(subCmd.Use, subCmd) + factory.SetOptions(options) } } @@ -49,6 +52,7 @@ func runExperimentsMain(experimentName string, currentOptions *GlobalOptions) { dbProps := initDatabase(ctx, sess, currentOptions) factory := registryx.AllExperiments[experimentName] - err = factory.Main(ctx, sess, dbProps) + factory.SetArguments(sess, dbProps, nil) + err = factory.Main(ctx) runtimex.PanicOnError(err, fmt.Sprintf("%s.Main failed", experimentName)) } diff --git a/internal/experiment/dash/main.go b/internal/experiment/dash/main.go index db73c16f97..55854d866f 100644 --- a/internal/experiment/dash/main.go +++ b/internal/experiment/dash/main.go @@ -8,29 +8,53 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("dash: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +70,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{config: *config} + measurer := &Measurer{config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/fbmessenger/main.go b/internal/experiment/fbmessenger/main.go index 6ed4265d2f..57195528aa 100644 --- a/internal/experiment/fbmessenger/main.go +++ b/internal/experiment/fbmessenger/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("fbmessenger: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +71,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/hhfm/main.go b/internal/experiment/hhfm/main.go index 971257b54a..79163371a7 100644 --- a/internal/experiment/hhfm/main.go +++ b/internal/experiment/hhfm/main.go @@ -9,29 +9,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("http_header_field_manipulation: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -47,19 +72,19 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Obtain experiment inputs. - inputs := getInputs(args, checkInResp) + inputs := getInputs(em.args, checkInResp) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() // Create suitable stop policy. - shouldStop := newStopPolicy(args, testStartTime) + shouldStop := newStopPolicy(em.args, testStartTime) // Create suitable progress emitter. - progresser := newProgressEmitter(args, inputs, testStartTime) + progresser := newProgressEmitter(em.args, inputs, testStartTime) // Measure each URL in sequence. for inputIdx, input := range inputs { @@ -75,7 +100,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e // Measure the current URL. err := experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/hirl/main.go b/internal/experiment/hirl/main.go index 3d4aad6451..41a879c898 100644 --- a/internal/experiment/hirl/main.go +++ b/internal/experiment/hirl/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("http_invalid_request_line: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -47,7 +72,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e // Create an instance of the experiment's measurer. measurer := &Measurer{ - Config: *config, + Config: *em.configArgs, Methods: []Method{ randomInvalidMethod{}, randomInvalidFieldCount{}, @@ -62,7 +87,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/ndt7/main.go b/internal/experiment/ndt7/main.go index 65415063be..025185c378 100644 --- a/internal/experiment/ndt7/main.go +++ b/internal/experiment/ndt7/main.go @@ -9,29 +9,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("ndt7: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -48,7 +73,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e // Create an instance of the experiment's measurer. measurer := &Measurer{ - config: *config, + config: *em.configArgs, jsonUnmarshal: json.Unmarshal, } @@ -57,7 +82,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/psiphon/main.go b/internal/experiment/psiphon/main.go index fcdae3f244..41a90b4d30 100644 --- a/internal/experiment/psiphon/main.go +++ b/internal/experiment/psiphon/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("psiphon: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +71,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/riseupvpn/main.go b/internal/experiment/riseupvpn/main.go index 16a487d16c..301af2e250 100644 --- a/internal/experiment/riseupvpn/main.go +++ b/internal/experiment/riseupvpn/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("riseupvpn: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +71,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/signal/main.go b/internal/experiment/signal/main.go index 1e50308762..9656e05885 100644 --- a/internal/experiment/signal/main.go +++ b/internal/experiment/signal/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("signal: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +71,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/telegram/main.go b/internal/experiment/telegram/main.go index fc3e57c627..0649c808e5 100644 --- a/internal/experiment/telegram/main.go +++ b/internal/experiment/telegram/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("telegram: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +71,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/tor/main.go b/internal/experiment/tor/main.go index e82d68fed3..48b587b42a 100644 --- a/internal/experiment/tor/main.go +++ b/internal/experiment/tor/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("tor: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -47,7 +72,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e // Create an instance of the experiment's measurer. measurer := &Measurer{ - config: *config, + config: *em.configArgs, fetchTorTargets: func(ctx context.Context, sess model.ExperimentSession, cc string) (map[string]model.OOAPITorTarget, error) { return sess.FetchTorTargets(ctx, cc) @@ -59,7 +84,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/webconnectivity/main.go b/internal/experiment/webconnectivity/main.go index 0a20251dd2..3c6baa3940 100644 --- a/internal/experiment/webconnectivity/main.go +++ b/internal/experiment/webconnectivity/main.go @@ -9,29 +9,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -47,19 +72,19 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Obtain experiment inputs. - inputs := getInputs(args, checkInResp) + inputs := getInputs(em.args, checkInResp) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() // Create suitable stop policy. - shouldStop := newStopPolicy(args, testStartTime) + shouldStop := newStopPolicy(em.args, testStartTime) // Create suitable progress emitter. - progresser := newProgressEmitter(args, inputs, testStartTime) + progresser := newProgressEmitter(em.args, inputs, testStartTime) // Measure each URL in sequence. for inputIdx, input := range inputs { @@ -75,7 +100,7 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e // Measure the current URL. err := experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/experiment/whatsapp/main.go b/internal/experiment/whatsapp/main.go index 9aa730979b..77c5e4738d 100644 --- a/internal/experiment/whatsapp/main.go +++ b/internal/experiment/whatsapp/main.go @@ -8,29 +8,54 @@ import ( "github.com/ooni/probe-cli/v3/internal/experiment" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" ) // ErrNoCheckInInfo indicates check-in returned no suitable info. var ErrNoCheckInInfo = errors.New("whatsapp: returned no check-in info") +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + // Main is the main function of the experiment. -func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) error { - sess := args.Session +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session logger := sess.Logger() // Create the directory where to store results unless it already exists - if err := os.MkdirAll(args.MeasurementDir, 0700); err != nil { + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { return err } // Attempt to remove the results directory when done unless it // contains files, in which case we should keep it. - defer os.Remove(args.MeasurementDir) + defer os.Remove(em.args.MeasurementDir) // Call the check-in API to obtain configuration. Note that the value // returned here MAY have been cached by the engine. logger.Infof("calling check-in API...") - checkInResp, err := experiment.CallCheckIn(ctx, args, sess) + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) // Bail if either the check-in API failed or we don't have a reportID // with which to submit Web Connectivity measurements results. @@ -46,14 +71,14 @@ func Main(ctx context.Context, args *model.ExperimentMainArgs, config *Config) e logger.Infof("ReportID: %s", reportID) // Create an instance of the experiment's measurer. - measurer := &Measurer{Config: *config} + measurer := &Measurer{Config: *em.configArgs} // Record when we started running this nettest. testStartTime := time.Now() return experiment.MeasurePossiblyNilInput( ctx, - args, + em.args, measurer, testStartTime, reportID, diff --git a/internal/model/database.go b/internal/model/database.go index e28a48cf95..374e6cb4da 100644 --- a/internal/model/database.go +++ b/internal/model/database.go @@ -265,3 +265,15 @@ type PerformanceTestKeys struct { Ping float64 `json:"ping"` Bitrate float64 `json:"median_bitrate"` } + +// DatabaseProps contains the database properties for a database instance +type DatabaseProps struct { + // + Database WritableDatabase + + // + DatabaseNetwork *DatabaseNetwork + + // + DatabaseResult *DatabaseResult +} diff --git a/internal/model/experiment.go b/internal/model/experiment.go index 813265851e..b7026c9926 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -7,6 +7,8 @@ package model import ( "context" + + "github.com/spf13/cobra" ) // ExperimentSession is the experiment's view of a session. @@ -382,3 +384,18 @@ type Saver interface { type ExperimentInputProcessor interface { Run(ctx context.Context) error } + +// ExperimentOptions +type ExperimentOptions interface { + // SetArguments + SetArguments(sess ExperimentSession, db *DatabaseProps) *ExperimentMainArgs + + // ExtraOptions + ExtraOptions() map[string]any + + // BuildWithOONIRun + BuildWithOONIRun(inputs []string, args map[string]any) error + + // BuildFlags + BuildFlags(experimentName string, rootCmd *cobra.Command) +} diff --git a/internal/ooapi/checkin.go b/internal/ooapi/checkin.go index 89753280fe..0f9ac9316a 100644 --- a/internal/ooapi/checkin.go +++ b/internal/ooapi/checkin.go @@ -25,7 +25,7 @@ func NewDescriptorCheckIn( AcceptEncodingGzip: true, // we want a small response Authorization: "", ContentType: httpapi.ApplicationJSON, - LogBody: false, // we don't want to log psiphon config + LogBody: true, // we don't want to log psiphon config MaxBodySize: 0, Method: http.MethodPost, Request: &httpapi.RequestDescriptor[*model.OOAPICheckInConfig]{ diff --git a/internal/oonirunx/link.go b/internal/oonirunx/link.go index f0329f872b..989f47f2f0 100644 --- a/internal/oonirunx/link.go +++ b/internal/oonirunx/link.go @@ -8,7 +8,6 @@ import ( "context" "strings" - "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/model" ) @@ -39,7 +38,7 @@ type LinkConfig struct { Session *engine.Session // DatabaseProps is the MANDATORY database properties to use - DatabaseProps *database.DatabaseProps + DatabaseProps *model.DatabaseProps } // LinkRunner knows how to run an OONI Run v1 or v2 link. diff --git a/internal/oonirunx/v1.go b/internal/oonirunx/v1.go index fe9ff00546..dc729b742a 100644 --- a/internal/oonirunx/v1.go +++ b/internal/oonirunx/v1.go @@ -73,10 +73,13 @@ func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { if mv := pu.Query().Get("mv"); mv != "1.2.0" { return fmt.Errorf("%w: unknown minimum version", ErrInvalidV1URLQueryArgument) } - factory := registryx.AllExperiments[name] args := make(map[string]any) + options := registryx.AllExperimentOptions[name] + options.BuildWithOONIRun(inputs, args) extraOptions := make(map[string]any) // the v1 spec does not allow users to pass experiment options - return factory.Oonirun(ctx, config.Session, inputs, args, extraOptions, config.DatabaseProps) + factory := registryx.AllExperiments[name] + factory.SetArguments(config.Session, config.DatabaseProps, extraOptions) + return factory.Main(ctx) } // v1ParseArguments parses the `ta` field of the query string. diff --git a/internal/oonirunx/v2.go b/internal/oonirunx/v2.go index 72c560e8a2..7a3864b245 100644 --- a/internal/oonirunx/v2.go +++ b/internal/oonirunx/v2.go @@ -179,8 +179,13 @@ func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descri v2CountEmptyNettestNames.Add(1) continue } + // populate options using the V2Nettest struct + options := registryx.AllExperimentOptions[nettest.TestName] + options.BuildWithOONIRun(nettest.Inputs, nettest.Args) + // pass the set options to the experiment factory factory := registryx.AllExperiments[nettest.TestName] - err := factory.Oonirun(ctx, config.Session, nettest.Inputs, nettest.Args, nettest.Options, config.DatabaseProps) + factory.SetArguments(config.Session, config.DatabaseProps, nettest.Options) + err := factory.Main(ctx) if err != nil { logger.Warnf("cannot run experiment: %s", err.Error()) v2CountFailedExperiments.Add(1) diff --git a/internal/registryx/allexperiments.go b/internal/registryx/allexperiments.go index 56216b62b3..718e4f5b3d 100644 --- a/internal/registryx/allexperiments.go +++ b/internal/registryx/allexperiments.go @@ -1,7 +1,9 @@ package registryx +import "github.com/ooni/probe-cli/v3/internal/model" + // Where we register all the available experiments. -var AllExperiments = map[string]*Factory{} +var AllExperiments = map[string]Experiment{} // ExperimentNames returns the name of all experiments func ExperimentNames() (names []string) { @@ -10,3 +12,5 @@ func ExperimentNames() (names []string) { } return } + +var AllExperimentOptions = map[string]model.ExperimentOptions{} diff --git a/internal/registryx/dash.go b/internal/registryx/dash.go index 84b7bac140..c0b6ba147e 100644 --- a/internal/registryx/dash.go +++ b/internal/registryx/dash.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/dash" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,19 @@ type dashOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &dashOptions{} + func init() { options := &dashOptions{} - AllExperiments["dash"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &dash.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return dashMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &dashOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &dash.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return dashMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - dashBuildFlags(experimentName, rootCmd, options, &dash.Config{}) - }, - } + AllExperimentOptions["dash"] = options + AllExperiments["dash"] = &dash.ExperimentMain{} } -func dashMain(ctx context.Context, sess model.ExperimentSession, options *dashOptions, - config *dash.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +// SetArguments +func (do *dashOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(do.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +40,37 @@ func dashMain(ctx context.Context, sess model.ExperimentSession, options *dashOp RunType: model.RunTypeManual, Session: sess, } - return dash.Main(ctx, args, config) } -func dashBuildFlags(experimentName string, rootCmd *cobra.Command, - options *dashOptions, config any) { +// ExtraOptions +func (do *dashOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(do.ConfigOptions) +} + +// BuildWithOONIRun +func (do *dashOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(do, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (do *dashOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := dash.Config{} flags.StringSliceVarP( - &options.Annotations, + &do.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &do.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/factory.go b/internal/registryx/factory.go index 2e18463639..d9f307679d 100644 --- a/internal/registryx/factory.go +++ b/internal/registryx/factory.go @@ -2,195 +2,25 @@ package registryx import ( "context" - "errors" - "fmt" - "reflect" - "strconv" - "strings" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/model" "github.com/spf13/cobra" ) -// Factory is a forwarder for the respective experiment's main -type Factory struct { - // Main calls the experiment.Main functions - Main func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error - +// Experiment +type Experiment interface { // - Oonirun func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error - - // BuildFlags initializes the experiment specific flags - BuildFlags func(experimentName string, rootCmd *cobra.Command) -} - -var ( - // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. - ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") - - // ErrNoSuchField indicates there's no field with the given name. - ErrNoSuchField = errors.New("no such field") - - // ErrCannotSetIntegerOption means SetOptionAny couldn't set an integer option. - ErrCannotSetIntegerOption = errors.New("cannot set integer option") - - // ErrInvalidStringRepresentationOfBool indicates the string you passed - // to SetOptionaAny is not a valid string representation of a bool. - ErrInvalidStringRepresentationOfBool = errors.New("invalid string representation of bool") - - // ErrCannotSetBoolOption means SetOptionAny couldn't set a bool option. - ErrCannotSetBoolOption = errors.New("cannot set bool option") - - // ErrCannotSetStringOption means SetOptionAny couldn't set a string option. - ErrCannotSetStringOption = errors.New("cannot set string option") - - // ErrUnsupportedOptionType means we don't support the type passed to - // the SetOptionAny method as an opaque any type. - ErrUnsupportedOptionType = errors.New("unsupported option type") -) - -// options returns the options exposed by this experiment. -func options(config any) (map[string]model.ExperimentOptionInfo, error) { - result := make(map[string]model.ExperimentOptionInfo) - ptrinfo := reflect.ValueOf(config) - if ptrinfo.Kind() != reflect.Ptr { - return nil, ErrConfigIsNotAStructPointer - } - structinfo := ptrinfo.Elem().Type() - if structinfo.Kind() != reflect.Struct { - return nil, ErrConfigIsNotAStructPointer - } - for i := 0; i < structinfo.NumField(); i++ { - field := structinfo.Field(i) - result[field.Name] = model.ExperimentOptionInfo{ - Doc: field.Tag.Get("ooni"), - Type: field.Type.String(), - } - } - return result, nil -} - -// setOptionBool sets a bool option. -func setOptionBool(field reflect.Value, value any) error { - switch v := value.(type) { - case bool: - field.SetBool(v) - return nil - case string: - if v != "true" && v != "false" { - return fmt.Errorf("%w: %s", ErrInvalidStringRepresentationOfBool, v) - } - field.SetBool(v == "true") - return nil - default: - return fmt.Errorf("%w from a value of type %T", ErrCannotSetBoolOption, value) - } -} - -// setOptionInt sets an int option -func setOptionInt(field reflect.Value, value any) error { - switch v := value.(type) { - case int64: - field.SetInt(v) - return nil - case int32: - field.SetInt(int64(v)) - return nil - case int16: - field.SetInt(int64(v)) - return nil - case int8: - field.SetInt(int64(v)) - return nil - case int: - field.SetInt(int64(v)) - return nil - case string: - number, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return fmt.Errorf("%w: %s", ErrCannotSetIntegerOption, err.Error()) - } - field.SetInt(number) - return nil - default: - return fmt.Errorf("%w from a value of type %T", ErrCannotSetIntegerOption, value) - } -} - -// setOptionString sets a string option -func setOptionString(field reflect.Value, value any) error { - switch v := value.(type) { - case string: - field.SetString(v) - return nil - default: - return fmt.Errorf("%w from a value of type %T", ErrCannotSetStringOption, value) - } -} + Main(ctx context.Context) error -// setOptionAny sets an option given any value. -func setOptionAny(config any, key string, value any) error { - field, err := fieldbyname(config, key) - if err != nil { - return err - } - switch field.Kind() { - case reflect.Int64: - return setOptionInt(field, value) - case reflect.Bool: - return setOptionBool(field, value) - case reflect.String: - return setOptionString(field, value) - default: - return fmt.Errorf("%w: %T", ErrUnsupportedOptionType, value) - } -} - -// SetOptionsAny calls SetOptionAny for each entry inside [options]. -func setOptionsAny(config any, options map[string]any) error { - for key, value := range options { - if err := setOptionAny(config, key, value); err != nil { - return err - } - } - return nil -} + // + SetOptions(options model.ExperimentOptions) -// fieldbyname return v's field whose name is equal to the given key. -func fieldbyname(v interface{}, key string) (reflect.Value, error) { - // See https://stackoverflow.com/a/6396678/4354461 - ptrinfo := reflect.ValueOf(v) - if ptrinfo.Kind() != reflect.Ptr { - return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) - } - structinfo := ptrinfo.Elem() - if structinfo.Kind() != reflect.Struct { - return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) - } - field := structinfo.FieldByName(key) - if !field.IsValid() || !field.CanSet() { - return reflect.Value{}, fmt.Errorf("%w: %s", ErrNoSuchField, key) - } - return field, nil + // + SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error } -func documentationForOptions(name string, config any) string { - var sb strings.Builder - options, err := options(config) - if err != nil || len(options) < 1 { - return "" - } - fmt.Fprint(&sb, "Pass KEY=VALUE options to the experiment. Available options:\n") - for name, info := range options { - if info.Doc == "" { - continue - } - fmt.Fprintf(&sb, "\n") - fmt.Fprintf(&sb, " -O, --option %s=<%s>\n", name, info.Type) - fmt.Fprintf(&sb, " %s\n", info.Doc) - } - return sb.String() +// Factory is a forwarder for the respective experiment's main +type Factory struct { + // BuildFlags initializes the experiment specific flags + BuildFlags func(experimentName string, rootCmd *cobra.Command) model.ExperimentOptions } diff --git a/internal/registryx/fbmessenger.go b/internal/registryx/fbmessenger.go index b24333964f..129b61229c 100644 --- a/internal/registryx/fbmessenger.go +++ b/internal/registryx/fbmessenger.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/fbmessenger" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,41 +13,21 @@ type fbmessengerOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &fbmessengerOptions{} + func init() { options := &fbmessengerOptions{} - AllExperiments["facebook_messenger"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &fbmessenger.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return fbmessengerMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &fbmessengerOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &fbmessenger.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return fbmessengerMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - fbmessengerBuildFlags(experimentName, rootCmd, options, &fbmessenger.Config{}) - }, - } + AllExperimentOptions["facebook_messenger"] = options + AllExperiments["facebook_messenger"] = &fbmessenger.ExperimentMain{} } -func fbmessengerMain(ctx context.Context, sess model.ExperimentSession, options *fbmessengerOptions, - config *fbmessenger.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ - Annotations: annotations, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category +// SetAruments +func (fbo *fbmessengerOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(fbo.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, @@ -63,24 +40,37 @@ func fbmessengerMain(ctx context.Context, sess model.ExperimentSession, options RunType: model.RunTypeManual, Session: sess, } - return fbmessenger.Main(ctx, args, config) } -func fbmessengerBuildFlags(experimentName string, rootCmd *cobra.Command, - options *fbmessengerOptions, config any) { +// ExtraOptions +func (fbo *fbmessengerOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(fbo.ConfigOptions) +} + +// BuildWithOONIRun +func (fbo *fbmessengerOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(fbo, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (fbo *fbmessengerOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := fbmessenger.Config{} flags.StringSliceVarP( - &options.Annotations, + &fbo.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &fbo.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/hhfm.go b/internal/registryx/hhfm.go index 7f62504330..a7f91f78ea 100644 --- a/internal/registryx/hhfm.go +++ b/internal/registryx/hhfm.go @@ -1,17 +1,14 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/hhfm" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) -// hhfmOptions contains options for web connectivity. +// hhfmOptions contains options for hhfm. type hhfmOptions struct { Annotations []string InputFilePaths []string @@ -21,47 +18,25 @@ type hhfmOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &hhfmOptions{} + func init() { options := &hhfmOptions{} - AllExperiments["hhfm"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &hhfm.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return hhfmMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &hhfmOptions{} - options.Inputs = inputs - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &hhfm.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return hhfmMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - hhfmBuildFlags(experimentName, rootCmd, options, &hhfm.Config{}) - }, - } + AllExperimentOptions["hhfm"] = options + AllExperiments["hhfm"] = &hhfm.ExperimentMain{} } -func hhfmMain(ctx context.Context, sess model.ExperimentSession, options *hhfmOptions, - config *hhfm.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +func (hhfmo *hhfmOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(hhfmo.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, - Inputs: options.Inputs, - MaxRuntime: options.MaxRuntime, + Inputs: hhfmo.Inputs, + MaxRuntime: hhfmo.MaxRuntime, MeasurementDir: db.DatabaseResult.MeasurementDir, NoCollector: false, OnWiFi: true, @@ -69,16 +44,27 @@ func hhfmMain(ctx context.Context, sess model.ExperimentSession, options *hhfmOp RunType: model.RunTypeManual, Session: sess, } +} - return hhfm.Main(ctx, args, config) +// ExtraOptions +func (hhfmo *hhfmOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(hhfmo.ConfigOptions) +} + +// BuildWithOONIRun +func (hhfmo *hhfmOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(hhfmo, args); err != nil { + return err + } + return nil } -func hhfmBuildFlags(experimentName string, rootCmd *cobra.Command, - options *hhfmOptions, config any) { +func (hhfmo *hhfmOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := hhfm.Config{} flags.StringSliceVarP( - &options.Annotations, + &hhfmo.Annotations, "annotation", "A", []string{}, @@ -86,7 +72,7 @@ func hhfmBuildFlags(experimentName string, rootCmd *cobra.Command, ) flags.StringSliceVarP( - &options.InputFilePaths, + &hhfmo.InputFilePaths, "input-file", "f", []string{}, @@ -94,7 +80,7 @@ func hhfmBuildFlags(experimentName string, rootCmd *cobra.Command, ) flags.StringSliceVarP( - &options.Inputs, + &hhfmo.Inputs, "input", "i", []string{}, @@ -102,22 +88,22 @@ func hhfmBuildFlags(experimentName string, rootCmd *cobra.Command, ) flags.Int64Var( - &options.MaxRuntime, + &hhfmo.MaxRuntime, "max-runtime", 0, "maximum runtime in seconds for the experiment (zero means infinite)", ) flags.BoolVar( - &options.Random, + &hhfmo.Random, "random", false, "randomize the inputs list", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &hhfmo.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/hirl.go b/internal/registryx/hirl.go index 4aa5a5117b..7ece146574 100644 --- a/internal/registryx/hirl.go +++ b/internal/registryx/hirl.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/hirl" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,19 @@ type hirlOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &hirlOptions{} + func init() { options := &hirlOptions{} - AllExperiments["hirl"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &hirl.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return hirlMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &hirlOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &hirl.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return hirlMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - hirlBuildFlags(experimentName, rootCmd, options, &hirl.Config{}) - }, - } + AllExperimentOptions["hirl"] = options + AllExperiments["hirl"] = &hirl.ExperimentMain{} } -func hirlMain(ctx context.Context, sess model.ExperimentSession, options *hirlOptions, - config *hirl.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +// SetArguments +func (hirlo *hirlOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(hirlo.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +40,37 @@ func hirlMain(ctx context.Context, sess model.ExperimentSession, options *hirlOp RunType: model.RunTypeManual, Session: sess, } - return hirl.Main(ctx, args, config) } -func hirlBuildFlags(experimentName string, rootCmd *cobra.Command, - options *hirlOptions, config any) { +// ExtraOptions +func (hirlo *hirlOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(hirlo.ConfigOptions) +} + +// BuildWithOONIRun +func (hirlo *hirlOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(hirlo, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (hirlo *hirlOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := hirl.Config{} flags.StringSliceVarP( - &options.Annotations, + &hirlo.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &hirlo.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/ndt.go b/internal/registryx/ndt.go index 1f2e6c2388..91277c016d 100644 --- a/internal/registryx/ndt.go +++ b/internal/registryx/ndt.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/ndt7" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,19 @@ type ndtOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &ndtOptions{} + func init() { options := &ndtOptions{} - AllExperiments["ndt"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &ndt7.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return ndtMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &ndtOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &ndt7.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return ndtMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - ndtBuildFlags(experimentName, rootCmd, options, &ndt7.Config{}) - }, - } + AllExperimentOptions["ndt"] = options + AllExperiments["ndt"] = &ndt7.ExperimentMain{} } -func ndtMain(ctx context.Context, sess model.ExperimentSession, options *ndtOptions, - config *ndt7.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +// SetArguments +func (ndto *ndtOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(ndto.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +40,37 @@ func ndtMain(ctx context.Context, sess model.ExperimentSession, options *ndtOpti RunType: model.RunTypeManual, Session: sess, } - return ndt7.Main(ctx, args, config) } -func ndtBuildFlags(experimentName string, rootCmd *cobra.Command, - options *ndtOptions, config any) { +// ExtraOptions +func (ndto *ndtOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(ndto.ConfigOptions) +} + +// BuildWithOONIRun +func (ndto *ndtOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(ndto, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (ndto *ndtOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := ndt7.Config{} flags.StringSliceVarP( - &options.Annotations, + &ndto.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &ndto.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/psiphon.go b/internal/registryx/psiphon.go index cb6a337a35..79d52781ce 100644 --- a/internal/registryx/psiphon.go +++ b/internal/registryx/psiphon.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/psiphon" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,19 @@ type psiphonOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &psiphonOptions{} + func init() { options := &psiphonOptions{} - AllExperiments["psiphon"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &psiphon.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return psiphonMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &psiphonOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &psiphon.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return psiphonMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - psiphonBuildFlags(experimentName, rootCmd, options, &psiphon.Config{}) - }, - } + AllExperimentOptions["psiphon"] = options + AllExperiments["psiphon"] = &psiphon.ExperimentMain{} } -func psiphonMain(ctx context.Context, sess model.ExperimentSession, options *psiphonOptions, - config *psiphon.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +// SetArguments +func (po *psiphonOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(po.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +40,37 @@ func psiphonMain(ctx context.Context, sess model.ExperimentSession, options *psi RunType: model.RunTypeManual, Session: sess, } - return psiphon.Main(ctx, args, config) } -func psiphonBuildFlags(experimentName string, rootCmd *cobra.Command, - options *psiphonOptions, config any) { +// ExtraOptions +func (po *psiphonOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(po.ConfigOptions) +} + +// BuildWithOONIRun +func (po *psiphonOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(po, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (po *psiphonOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := psiphon.Config{} flags.StringSliceVarP( - &options.Annotations, + &po.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &po.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/riseupvpn.go b/internal/registryx/riseupvpn.go index fd982d3c4b..73ba7451b3 100644 --- a/internal/registryx/riseupvpn.go +++ b/internal/registryx/riseupvpn.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/riseupvpn" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,19 @@ type riseupvpnOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &riseupvpnOptions{} + func init() { options := &riseupvpnOptions{} - AllExperiments["riseupvpn"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &riseupvpn.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return riseupvpnMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &riseupvpnOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &riseupvpn.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return riseupvpnMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - riseupvpnBuildFlags(experimentName, rootCmd, options, &riseupvpn.Config{}) - }, - } + AllExperimentOptions["riseupvpn"] = options + AllExperiments["riseupvpn"] = &riseupvpn.ExperimentMain{} } -func riseupvpnMain(ctx context.Context, sess model.ExperimentSession, options *riseupvpnOptions, - config *riseupvpn.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +// SetArguments +func (ro *riseupvpnOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(ro.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +40,37 @@ func riseupvpnMain(ctx context.Context, sess model.ExperimentSession, options *r RunType: model.RunTypeManual, Session: sess, } - return riseupvpn.Main(ctx, args, config) } -func riseupvpnBuildFlags(experimentName string, rootCmd *cobra.Command, - options *riseupvpnOptions, config any) { +// ExtraOptions +func (ro *riseupvpnOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(ro.ConfigOptions) +} + +// BuildWithOONIRun +func (ro *riseupvpnOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(ro, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (ro *riseupvpnOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := &riseupvpn.Config{} flags.StringSliceVarP( - &options.Annotations, + &ro.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &ro.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/signal.go b/internal/registryx/signal.go index 76858638fb..ef92dd19fa 100644 --- a/internal/registryx/signal.go +++ b/internal/registryx/signal.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/signal" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,41 +13,21 @@ type signalOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &signalOptions{} + func init() { options := &signalOptions{} - AllExperiments["signal"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &signal.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return signalMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &signalOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &signal.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return signalMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - signalBuildFlags(experimentName, rootCmd, options, &signal.Config{}) - }, - } + AllExperimentOptions["signal"] = options + AllExperiments["signal"] = &signal.ExperimentMain{} } -func signalMain(ctx context.Context, sess model.ExperimentSession, options *signalOptions, - config *signal.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ - Annotations: annotations, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category +// SetArguments +func (so *signalOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(so.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, @@ -63,24 +40,37 @@ func signalMain(ctx context.Context, sess model.ExperimentSession, options *sign RunType: model.RunTypeManual, Session: sess, } - return signal.Main(ctx, args, config) } -func signalBuildFlags(experimentName string, rootCmd *cobra.Command, - options *signalOptions, config any) { +// ExtraOptions +func (so *signalOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(so.ConfigOptions) +} + +// BuildWithOONIRun +func (so *signalOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(so, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (so *signalOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := &signal.Config{} flags.StringSliceVarP( - &options.Annotations, + &so.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &so.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/telegram.go b/internal/registryx/telegram.go index 4893af6e81..ee4af7b550 100644 --- a/internal/registryx/telegram.go +++ b/internal/registryx/telegram.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/telegram" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,19 @@ type telegramOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &telegramOptions{} + func init() { options := &telegramOptions{} - AllExperiments["telegram"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &telegram.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return telegramMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &telegramOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &telegram.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return telegramMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - telegramBuildFlags(experimentName, rootCmd, options, &telegram.Config{}) - }, - } + AllExperimentOptions["telegram"] = options + AllExperiments["telegram"] = &telegram.ExperimentMain{} } -func telegramMain(ctx context.Context, sess model.ExperimentSession, options *telegramOptions, - config *telegram.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +// SetArguments +func (to *telegramOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(to.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +40,37 @@ func telegramMain(ctx context.Context, sess model.ExperimentSession, options *te RunType: model.RunTypeManual, Session: sess, } - return telegram.Main(ctx, args, config) } -func telegramBuildFlags(experimentName string, rootCmd *cobra.Command, - options *telegramOptions, config any) { +// ExtraOptions +func (to *telegramOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(to.ConfigOptions) +} + +// BuildWithOONIRun +func (to *telegramOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(to, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (to *telegramOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := &telegram.Config{} flags.StringSliceVarP( - &options.Annotations, + &to.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &to.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/tor.go b/internal/registryx/tor.go index bfff71ec3f..13a4d583ec 100644 --- a/internal/registryx/tor.go +++ b/internal/registryx/tor.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/tor" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,39 +13,18 @@ type torOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &torOptions{} + func init() { - options := &torOptions{} - AllExperiments["tor"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &tor.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return torMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &torOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &tor.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return torMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - torBuildFlags(experimentName, rootCmd, options, &tor.Config{}) - }, - } + options := &webConnectivityOptions{} + AllExperimentOptions["tor"] = options + AllExperiments["tor"] = &tor.ExperimentMain{} } -func torMain(ctx context.Context, sess model.ExperimentSession, options *torOptions, - config *tor.Config, db *database.DatabaseProps) error { - annotations := mustMakeMapStringString(options.Annotations) - args := &model.ExperimentMainArgs{ +func (toro *torOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(toro.Annotations) + return &model.ExperimentMainArgs{ Annotations: annotations, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, @@ -63,24 +39,34 @@ func torMain(ctx context.Context, sess model.ExperimentSession, options *torOpti RunType: model.RunTypeManual, Session: sess, } - return tor.Main(ctx, args, config) } -func torBuildFlags(experimentName string, rootCmd *cobra.Command, - options *torOptions, config any) { +func (toro *torOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(toro.ConfigOptions) +} + +func (toro *torOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(toro, args); err != nil { + return err + } + return nil +} + +func (toro *torOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := &tor.Config{} flags.StringSliceVarP( - &options.Annotations, + &toro.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &toro.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/webconnectivity.go b/internal/registryx/webconnectivity.go index 9c1d4beee0..5d460638d8 100644 --- a/internal/registryx/webconnectivity.go +++ b/internal/registryx/webconnectivity.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -21,46 +18,24 @@ type webConnectivityOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &webConnectivityOptions{} + func init() { options := &webConnectivityOptions{} - AllExperiments["web_connectivity"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &webconnectivity.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return webconnectivityMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &webConnectivityOptions{} - options.Inputs = inputs - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &webconnectivity.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return webconnectivityMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - webconnectivityBuildFlags(experimentName, rootCmd, options, &webconnectivity.Config{}) - }, - } + AllExperimentOptions["web_connectivity"] = options + AllExperiments["web_connectivity"] = &webconnectivity.ExperimentMain{} } -func webconnectivityMain(ctx context.Context, sess model.ExperimentSession, options *webConnectivityOptions, - config *webconnectivity.Config, db *database.DatabaseProps) error { - args := &model.ExperimentMainArgs{ +func (wco *webConnectivityOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + return &model.ExperimentMainArgs{ Annotations: map[string]string{}, // TODO(bassosimone): fill CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, - Inputs: options.Inputs, - MaxRuntime: options.MaxRuntime, + Inputs: wco.Inputs, + MaxRuntime: wco.MaxRuntime, MeasurementDir: db.DatabaseResult.MeasurementDir, NoCollector: false, OnWiFi: true, @@ -68,16 +43,26 @@ func webconnectivityMain(ctx context.Context, sess model.ExperimentSession, opti RunType: model.RunTypeManual, Session: sess, } +} - return webconnectivity.Main(ctx, args, config) +func (wco *webConnectivityOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(wco.ConfigOptions) +} + +func (wco *webConnectivityOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + wco.Inputs = inputs + if err := setter.SetOptionsAny(wco, args); err != nil { + return err + } + return nil } -func webconnectivityBuildFlags(experimentName string, rootCmd *cobra.Command, - options *webConnectivityOptions, config any) { +func (wco *webConnectivityOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := &webconnectivity.Config{} flags.StringSliceVarP( - &options.Annotations, + &wco.Annotations, "annotation", "A", []string{}, @@ -85,7 +70,7 @@ func webconnectivityBuildFlags(experimentName string, rootCmd *cobra.Command, ) flags.StringSliceVarP( - &options.InputFilePaths, + &wco.InputFilePaths, "input-file", "f", []string{}, @@ -93,7 +78,7 @@ func webconnectivityBuildFlags(experimentName string, rootCmd *cobra.Command, ) flags.StringSliceVarP( - &options.Inputs, + &wco.Inputs, "input", "i", []string{}, @@ -101,22 +86,22 @@ func webconnectivityBuildFlags(experimentName string, rootCmd *cobra.Command, ) flags.Int64Var( - &options.MaxRuntime, + &wco.MaxRuntime, "max-runtime", 0, "maximum runtime in seconds for the experiment (zero means infinite)", ) flags.BoolVar( - &options.Random, + &wco.Random, "random", false, "randomize the inputs list", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &wco.ConfigOptions, "options", "O", []string{}, diff --git a/internal/registryx/whatsapp.go b/internal/registryx/whatsapp.go index 61650a3f47..6ed34a7d06 100644 --- a/internal/registryx/whatsapp.go +++ b/internal/registryx/whatsapp.go @@ -1,13 +1,10 @@ package registryx import ( - "context" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/database" - "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/experiment/whatsapp" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" "github.com/spf13/cobra" ) @@ -16,40 +13,21 @@ type whatsappOptions struct { ConfigOptions []string } +var _ model.ExperimentOptions = &whatsappOptions{} + func init() { options := &whatsappOptions{} - AllExperiments["whatsapp"] = &Factory{ - Main: func(ctx context.Context, sess *engine.Session, db *database.DatabaseProps) error { - config := &whatsapp.Config{} - configMap := mustMakeMapStringAny(options.ConfigOptions) - if err := setOptionsAny(config, configMap); err != nil { - return err - } - return whatsappMain(ctx, sess, options, config, db) - }, - Oonirun: func(ctx context.Context, sess *engine.Session, inputs []string, - args map[string]any, extraOptions map[string]any, db *database.DatabaseProps) error { - options := &whatsappOptions{} - if err := setOptionsAny(options, args); err != nil { - return err - } - config := &whatsapp.Config{} - if err := setOptionsAny(config, extraOptions); err != nil { - return err - } - return whatsappMain(ctx, sess, options, config, db) - }, - BuildFlags: func(experimentName string, rootCmd *cobra.Command) { - whatsappBuildFlags(experimentName, rootCmd, options, &whatsapp.Config{}) - }, - } + AllExperimentOptions["whatsapp"] = options + AllExperiments["whatsapp"] = &whatsapp.ExperimentMain{} } -func whatsappMain(ctx context.Context, sess model.ExperimentSession, options *whatsappOptions, - config *whatsapp.Config, db *database.DatabaseProps) error { - args := &model.ExperimentMainArgs{ - Annotations: map[string]string{}, // TODO(bassosimone): fill - CategoryCodes: nil, // accept any category +// SetArguments +func (wo *whatsappOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(wo.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, + CategoryCodes: nil, // accept any category Charging: true, Callbacks: model.NewPrinterCallbacks(log.Log), Database: db.Database, @@ -62,24 +40,37 @@ func whatsappMain(ctx context.Context, sess model.ExperimentSession, options *wh RunType: model.RunTypeManual, Session: sess, } - return whatsapp.Main(ctx, args, config) } -func whatsappBuildFlags(experimentName string, rootCmd *cobra.Command, - options *whatsappOptions, config any) { +// ExtraOptions +func (wo *whatsappOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(wo.ConfigOptions) +} + +// BuildWithOONIRun +func (wo *whatsappOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(wo, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (wo *whatsappOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { flags := rootCmd.Flags() + config := &whatsapp.Config{} flags.StringSliceVarP( - &options.Annotations, + &wo.Annotations, "annotation", "A", []string{}, "add KEY=VALUE annotation to the report (can be repeated multiple times)", ) - if doc := documentationForOptions(experimentName, config); doc != "" { + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { flags.StringSliceVarP( - &options.ConfigOptions, + &wo.ConfigOptions, "options", "O", []string{}, diff --git a/internal/setter/setter.go b/internal/setter/setter.go new file mode 100644 index 0000000000..fc5c662002 --- /dev/null +++ b/internal/setter/setter.go @@ -0,0 +1,180 @@ +package setter + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +var ( + // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. + ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") + + // ErrNoSuchField indicates there's no field with the given name. + ErrNoSuchField = errors.New("no such field") + + // ErrCannotSetIntegerOption means SetOptionAny couldn't set an integer option. + ErrCannotSetIntegerOption = errors.New("cannot set integer option") + + // ErrInvalidStringRepresentationOfBool indicates the string you passed + // to SetOptionaAny is not a valid string representation of a bool. + ErrInvalidStringRepresentationOfBool = errors.New("invalid string representation of bool") + + // ErrCannotSetBoolOption means SetOptionAny couldn't set a bool option. + ErrCannotSetBoolOption = errors.New("cannot set bool option") + + // ErrCannotSetStringOption means SetOptionAny couldn't set a string option. + ErrCannotSetStringOption = errors.New("cannot set string option") + + // ErrUnsupportedOptionType means we don't support the type passed to + // the SetOptionAny method as an opaque any type. + ErrUnsupportedOptionType = errors.New("unsupported option type") +) + +// DocumentationForOptions +func DocumentationForOptions(name string, config any) string { + var sb strings.Builder + options, err := options(config) + if err != nil || len(options) < 1 { + return "" + } + fmt.Fprint(&sb, "Pass KEY=VALUE options to the experiment. Available options:\n") + for name, info := range options { + if info.Doc == "" { + continue + } + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, " -O, --option %s=<%s>\n", name, info.Type) + fmt.Fprintf(&sb, " %s\n", info.Doc) + } + return sb.String() +} + +// SetOptionsAny calls SetOptionAny for each entry inside [options]. +func SetOptionsAny(config any, options map[string]any) error { + for key, value := range options { + if err := setOptionAny(config, key, value); err != nil { + return err + } + } + return nil +} + +// options returns the options exposed by this experiment. +func options(config any) (map[string]model.ExperimentOptionInfo, error) { + result := make(map[string]model.ExperimentOptionInfo) + ptrinfo := reflect.ValueOf(config) + if ptrinfo.Kind() != reflect.Ptr { + return nil, ErrConfigIsNotAStructPointer + } + structinfo := ptrinfo.Elem().Type() + if structinfo.Kind() != reflect.Struct { + return nil, ErrConfigIsNotAStructPointer + } + for i := 0; i < structinfo.NumField(); i++ { + field := structinfo.Field(i) + result[field.Name] = model.ExperimentOptionInfo{ + Doc: field.Tag.Get("ooni"), + Type: field.Type.String(), + } + } + return result, nil +} + +// setOptionBool sets a bool option. +func setOptionBool(field reflect.Value, value any) error { + switch v := value.(type) { + case bool: + field.SetBool(v) + return nil + case string: + if v != "true" && v != "false" { + return fmt.Errorf("%w: %s", ErrInvalidStringRepresentationOfBool, v) + } + field.SetBool(v == "true") + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetBoolOption, value) + } +} + +// setOptionInt sets an int option +func setOptionInt(field reflect.Value, value any) error { + switch v := value.(type) { + case int64: + field.SetInt(v) + return nil + case int32: + field.SetInt(int64(v)) + return nil + case int16: + field.SetInt(int64(v)) + return nil + case int8: + field.SetInt(int64(v)) + return nil + case int: + field.SetInt(int64(v)) + return nil + case string: + number, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("%w: %s", ErrCannotSetIntegerOption, err.Error()) + } + field.SetInt(number) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetIntegerOption, value) + } +} + +// setOptionString sets a string option +func setOptionString(field reflect.Value, value any) error { + switch v := value.(type) { + case string: + field.SetString(v) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetStringOption, value) + } +} + +// setOptionAny sets an option given any value. +func setOptionAny(config any, key string, value any) error { + field, err := fieldbyname(config, key) + if err != nil { + return err + } + switch field.Kind() { + case reflect.Int64: + return setOptionInt(field, value) + case reflect.Bool: + return setOptionBool(field, value) + case reflect.String: + return setOptionString(field, value) + default: + return fmt.Errorf("%w: %T", ErrUnsupportedOptionType, value) + } +} + +// fieldbyname return v's field whose name is equal to the given key. +func fieldbyname(v interface{}, key string) (reflect.Value, error) { + // See https://stackoverflow.com/a/6396678/4354461 + ptrinfo := reflect.ValueOf(v) + if ptrinfo.Kind() != reflect.Ptr { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + structinfo := ptrinfo.Elem() + if structinfo.Kind() != reflect.Struct { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + field := structinfo.FieldByName(key) + if !field.IsValid() || !field.CanSet() { + return reflect.Value{}, fmt.Errorf("%w: %s", ErrNoSuchField, key) + } + return field, nil +} diff --git a/internal/setter/setter_test.go b/internal/setter/setter_test.go new file mode 100644 index 0000000000..29e1481cbf --- /dev/null +++ b/internal/setter/setter_test.go @@ -0,0 +1,7 @@ +package setter + +import "testing" + +func TestSetter(t *testing.T) { + // TODO(DecFox): Add tests +}