diff --git a/internal/cmd/minipipeline/main.go b/internal/cmd/minipipeline/main.go index 3c9edf496..654c136a4 100644 --- a/internal/cmd/minipipeline/main.go +++ b/internal/cmd/minipipeline/main.go @@ -12,25 +12,25 @@ import ( ) var ( - // destdir is the -destdir flag - destdir = flag.String("destdir", ".", "destination directory to use") + // destdirFlag is the -destdir flag + destdirFlag = flag.String("destdir", ".", "destination directory to use") - // measurement is the -measurement flag - measurement = flag.String("measurement", "", "measurement file to analyze") + // measurementFlag is the -measurement flag + measurementFlag = flag.String("measurement", "", "measurement file to analyze") // mustWriteFileLn allows overwriting must.WriteFile in tests mustWriteFileFn = must.WriteFile - // prefix is the -prefix flag - prefix = flag.String("prefix", "", "prefix to add to generated files") + // prefixFlag is the -prefix flag + prefixFlag = flag.String("prefix", "", "prefix to add to generated files") - // osExit allows overwriting os.Exit in tests - osExit = os.Exit + // osExitFn allows overwriting os.Exit in tests + osExitFn = os.Exit ) func main() { flag.Parse() - if *measurement == "" { + if *measurementFlag == "" { fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "usage: %s -measurement [-prefix ]\n", filepath.Base(os.Args[0])) fmt.Fprintf(os.Stderr, "\n") @@ -43,20 +43,20 @@ func main() { fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "Use -prefix to add in front of the generated files names.\n") fmt.Fprintf(os.Stderr, "\n") - osExit(1) + osExitFn(1) } // parse the measurement file var parsed minipipeline.WebMeasurement - must.UnmarshalJSON(must.ReadFile(*measurement), &parsed) + must.UnmarshalJSON(must.ReadFile(*measurementFlag), &parsed) // generate and write observations - observationsPath := filepath.Join(*destdir, *prefix+"observations.json") + observationsPath := filepath.Join(*destdirFlag, *prefixFlag+"observations.json") container := runtimex.Try1(minipipeline.IngestWebMeasurement(&parsed)) mustWriteFileFn(observationsPath, must.MarshalAndIndentJSON(container, "", " "), 0600) // generate and write observations analysis - analysisPath := filepath.Join(*destdir, *prefix+"analysis.json") + analysisPath := filepath.Join(*destdirFlag, *prefixFlag+"analysis.json") analysis := minipipeline.AnalyzeWebObservations(container) mustWriteFileFn(analysisPath, must.MarshalAndIndentJSON(analysis, "", " "), 0600) } diff --git a/internal/cmd/minipipeline/main_test.go b/internal/cmd/minipipeline/main_test.go index 9df67a88e..0597cc669 100644 --- a/internal/cmd/minipipeline/main_test.go +++ b/internal/cmd/minipipeline/main_test.go @@ -25,23 +25,15 @@ func mustloaddata(contentmap map[string][]byte, key string) (object map[string]a } func TestMainSuccess(t *testing.T) { - // make sure we set the destination directory - *destdir = "xo" - - // make sure we're reading from the expected input - *measurement = filepath.Join("testdata", "measurement.json") - - // make sure we store the expected output + // reconfigure the global options for main + *destdirFlag = "xo" + *measurementFlag = filepath.Join("testdata", "measurement.json") contentmap := make(map[string][]byte) mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { contentmap[filename] = content } - - // make sure osExit is correct - osExit = os.Exit - - // also check whether we can add a prefix - *prefix = "y-" + osExitFn = os.Exit + *prefixFlag = "y-" // run the main function main() @@ -62,25 +54,18 @@ func TestMainSuccess(t *testing.T) { } func TestMainUsage(t *testing.T) { - // make sure we clear the destination directory - *destdir = "" - - // make sure the expected input file is empty - *measurement = "" - - // make sure we panic if we try to write on disk + // reconfigure the global options for main + *destdirFlag = "" + *measurementFlag = "" mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { panic(errors.New("mustWriteFileFn")) } - - // make sure osExit is correct - osExit = func(code int) { + osExitFn = func(code int) { panic(fmt.Errorf("osExit: %d", code)) } + *prefixFlag = "" - // make sure the prefix is also clean - *prefix = "" - + // run the main function var err error func() { // intercept panic caused by osExit or other panics diff --git a/internal/cmd/qatool/main.go b/internal/cmd/qatool/main.go new file mode 100644 index 000000000..0ea117168 --- /dev/null +++ b/internal/cmd/qatool/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivityqa" + "github.com/ooni/probe-cli/v3/internal/minipipeline" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var ( + // destdirFlag is the -destdir flag + destdirFlag = flag.String("destdir", "", "root directory in which to dump files") + + // disableMeasureFlag is the -disable-measure flag + disableMeasureFlag = flag.Bool("disable-measure", false, "whether to measure again with netemx") + + // disableReprocessFlag is the -disable-reprocess flag + disableReprocessFlag = flag.Bool("disable-reprocess", false, "whether to reprocess existing measurements") + + // helpFlag is the -help flag + helpFlag = flag.Bool("help", false, "print help message") + + // listFlag is the -list flag + listFlag = flag.Bool("list", false, "lists available tests") + + // mustReadFileFn allows to overwrite must.ReadFile in tests + mustReadFileFn = must.ReadFile + + // mustWriteFileFn allows to overwrite must.WriteFile in tests + mustWriteFileFn = must.WriteFile + + // osExitFn allows to overwrite os.Exit in tests + osExitFn = os.Exit + + // osMkdirAllFn allows to overwrite os.MkdirAll in tests + osMkdirAllFn = os.MkdirAll + + // runFlag is the -run flag + runFlag = flag.String("run", "", "regexp to select which test cases to run") +) + +func mustSerializeMkdirAllAndWriteFile(dirname string, filename string, content any) { + rawData := must.MarshalAndIndentJSON(content, "", " ") + runtimex.Try0(osMkdirAllFn(dirname, 0700)) + mustWriteFileFn(filepath.Join(dirname, filename), rawData, 0600) +} + +func runWebConnectivityLTE(tc *webconnectivityqa.TestCase) { + // compute the actual destdir + actualDestdir := filepath.Join(*destdirFlag, tc.Name) + + if !*disableMeasureFlag { + // construct the proper measurer + measurer := webconnectivitylte.NewExperimentMeasurer(&webconnectivitylte.Config{}) + + // run the test case + measurement := runtimex.Try1(webconnectivityqa.MeasureTestCase(measurer, tc)) + + // serialize the original measurement + mustSerializeMkdirAllAndWriteFile(actualDestdir, "measurement.json", measurement) + } + + if !*disableReprocessFlag { + // obtain the web measurement + rawData := mustReadFileFn(filepath.Join(actualDestdir, "measurement.json")) + var webMeasurement minipipeline.WebMeasurement + must.UnmarshalJSON(rawData, &webMeasurement) + + // ingest web measurement + observationsContainer := runtimex.Try1(minipipeline.IngestWebMeasurement(&webMeasurement)) + + // serialize the observations + mustSerializeMkdirAllAndWriteFile(actualDestdir, "observations.json", observationsContainer) + + // analyze the observations + analysis := minipipeline.AnalyzeWebObservations(observationsContainer) + + // serialize the observations analysis + mustSerializeMkdirAllAndWriteFile(actualDestdir, "analysis.json", analysis) + + // print the analysis to stdout + fmt.Printf("%s\n", must.MarshalAndIndentJSON(analysis, "", " ")) + } +} + +func main() { + // parse command line flags + flag.Parse() + + // print usage + if *helpFlag || (*destdirFlag == "" && !*listFlag) { + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "usage: %s -destdir [-run ] [-disable-measure|-disable-reprocess]]\n", filepath.Base(os.Args[0])) + fmt.Fprintf(os.Stderr, " %s -list [-run ]\n", filepath.Base(os.Args[0])) + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "The first form of the command runs the QA tests selected by the given\n") + fmt.Fprintf(os.Stderr, " and creates the corresponding files in .\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "The second form of the command lists the QA tests that would be run\n") + fmt.Fprintf(os.Stderr, "when using the given selector.\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "An empty selector selects all QA tests.\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "Add the -disable-measure flag to the first form of the command to\n") + fmt.Fprintf(os.Stderr, "avoid performing the measurements using netemx. This assums that\n") + fmt.Fprintf(os.Stderr, "you already generated the measurements previously.\n") + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "Add the -disable-reprocess flag to the first form of the command to\n") + fmt.Fprintf(os.Stderr, "avoid reprocessing the measurements using the minipipeline.\n") + fmt.Fprintf(os.Stderr, "\n") + osExitFn(1) + } + + // build the regexp + selector := regexp.MustCompile(*runFlag) + + // select which test cases to run + for _, tc := range webconnectivityqa.AllTestCases() { + name := "webconnectivitylte/" + tc.Name + if *runFlag != "" && !selector.MatchString(name) { + continue + } + if *listFlag { + fmt.Printf("%s\n", name) + continue + } + runWebConnectivityLTE(tc) + } +} diff --git a/internal/cmd/qatool/main_test.go b/internal/cmd/qatool/main_test.go new file mode 100644 index 000000000..b1f231410 --- /dev/null +++ b/internal/cmd/qatool/main_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func TestMainList(t *testing.T) { + // reconfigure the global options for main + *destdirFlag = "" + *listFlag = true + mustReadFileFn = func(filename string) []byte { + panic(errors.New("mustReadFileFn")) + } + mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { + panic(errors.New("mustWriteFileFn")) + } + osExitFn = func(code int) { + panic(fmt.Errorf("osExit: %d", code)) + } + osMkdirAllFn = func(path string, perm os.FileMode) error { + panic(errors.New("osMkdirAllFn")) + } + *runFlag = "" + + // run the main function + main() +} + +func TestMainSuccess(t *testing.T) { + // reconfigure the global options for main + *destdirFlag = "xo" + *listFlag = false + contentmap := make(map[string][]byte) + mustReadFileFn = func(filename string) []byte { + data, found := contentmap[filename] + runtimex.Assert(found, fmt.Sprintf("cannot find %s", filename)) + return data + } + mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { + // make sure we can parse as JSON + var container map[string]any + if err := json.Unmarshal(content, &container); err != nil { + t.Fatal(err) + } + + // register we have written a file + contentmap[filename] = content + } + osExitFn = os.Exit + osMkdirAllFn = func(path string, perm os.FileMode) error { + return nil + } + *runFlag = "dnsBlocking" + + // run the main function + main() + + // make sure we attempted to write the desired files + expect := map[string]bool{ + "xo/dnsBlockingAndroidDNSCacheNoData/measurement.json": true, + "xo/dnsBlockingAndroidDNSCacheNoData/observations.json": true, + "xo/dnsBlockingAndroidDNSCacheNoData/analysis.json": true, + "xo/dnsBlockingNXDOMAIN/measurement.json": true, + "xo/dnsBlockingNXDOMAIN/observations.json": true, + "xo/dnsBlockingNXDOMAIN/analysis.json": true, + } + got := make(map[string]bool) + for key := range contentmap { + got[key] = true + } + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } +} + +func TestMainUsage(t *testing.T) { + // reconfigure the global options for main + *destdirFlag = "" + *listFlag = false + mustReadFileFn = func(filename string) []byte { + panic(errors.New("mustReadFileFn")) + } + mustWriteFileFn = func(filename string, content []byte, mode fs.FileMode) { + panic(errors.New("mustWriteFileFn")) + } + osExitFn = func(code int) { + panic(fmt.Errorf("osExit: %d", code)) + } + osMkdirAllFn = func(path string, perm os.FileMode) error { + panic(errors.New("osMkdirAllFn")) + } + *runFlag = "" + + // run the main function + var err error + func() { + // intercept panic caused by osExit or other panics + defer func() { + if r := recover(); r != nil { + err = r.(error) + } + }() + + // run the main function with the given args + main() + }() + + // make sure we've got the expected error + if err == nil || err.Error() != "osExit: 1" { + t.Fatal("expected", "os.Exit: 1", "got", err) + } +} diff --git a/internal/experiment/webconnectivityqa/run.go b/internal/experiment/webconnectivityqa/run.go index e9c44f777..b851e4d4c 100644 --- a/internal/experiment/webconnectivityqa/run.go +++ b/internal/experiment/webconnectivityqa/run.go @@ -12,8 +12,8 @@ import ( "github.com/ooni/probe-cli/v3/internal/netxlite" ) -// RunTestCase runs a [testCase]. -func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { +// MeasureTestCase returns the JSON measurement produced by a [TestCase]. +func MeasureTestCase(measurer model.ExperimentMeasurer, tc *TestCase) (*model.Measurement, error) { // configure the netemx scenario env := netemx.MustNewScenario(netemx.InternetScenario) defer env.Close() @@ -34,7 +34,8 @@ func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { var err error env.Do(func() { // create an HTTP client inside the env.Do function so we're using netem - // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS + // but they're not needed here httpClient := netxlite.NewHTTPClientStdlib(prefixLogger) arguments := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(prefixLogger), @@ -54,9 +55,19 @@ func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { // handle the case of unexpected result switch { case err != nil && !tc.ExpectErr: - return fmt.Errorf("expected to see no error but got %s", err.Error()) + return nil, fmt.Errorf("expected to see no error but got %s", err.Error()) case err == nil && tc.ExpectErr: - return fmt.Errorf("expected to see an error but got ") + return nil, fmt.Errorf("expected to see an error but got ") + } + + return measurement, nil +} + +// RunTestCase runs a [testCase]. +func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { + measurement, err := MeasureTestCase(measurer, tc) + if err != nil { + return err } // reduce the test keys to a common format diff --git a/internal/experiment/webconnectivityqa/session.go b/internal/experiment/webconnectivityqa/session.go index 0abb31496..5af5eb514 100644 --- a/internal/experiment/webconnectivityqa/session.go +++ b/internal/experiment/webconnectivityqa/session.go @@ -6,7 +6,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/netemx" ) -// nwSession creates a new [model.ExperimentSession]. +// newSession creates a new [model.ExperimentSession]. func newSession(client model.HTTPClient, logger model.Logger) model.ExperimentSession { return &mocks.Session{ MockGetTestHelpersByName: func(name string) ([]model.OOAPIService, bool) { diff --git a/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json index 7aee6a93c..64bdeb18d 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/dnsBlockingBOGON/observations.json @@ -210,4 +210,4 @@ "ControlHTTPResponseTitle": "Default Web Page" } } -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json index 067dafae9..9f5839859 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/analysis.json @@ -13,4 +13,4 @@ "TCPTransactionsWithUnexpectedTLSHandshakeFailures": {}, "TCPTransactionsWithUnexpectedHTTPFailures": {}, "TCPTransactionsWithUnexplainedUnexpectedFailures": {} -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json index 3b200cb5e..993dc6594 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/redirectWithConsistentDNSAndThenNXDOMAIN/observations.json @@ -242,4 +242,4 @@ "ControlHTTPResponseTitle": "Default Web Page" } } -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json index b47650dfa..7fdae2135 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/analysis.json @@ -3,7 +3,7 @@ "DNSTransactionsWithBogons": {}, "DNSTransactionsWithUnexpectedFailures": {}, "DNSPossiblyInvalidAddrs": { - "83.224.65.41": true + "83.224.65.41": true }, "HTTPDiffBodyProportionFactor": 1, "HTTPDiffStatusCodeMatch": true, @@ -22,4 +22,4 @@ "TCPTransactionsWithUnexpectedTLSHandshakeFailures": {}, "TCPTransactionsWithUnexpectedHTTPFailures": {}, "TCPTransactionsWithUnexplainedUnexpectedFailures": {} -} +} \ No newline at end of file diff --git a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json index 156923e8d..3f72ac998 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/tcpBlockingConnectionRefusedWithInconsistentDNS/observations.json @@ -296,4 +296,4 @@ "ControlHTTPResponseTitle": "Default Web Page" } } -} +} \ No newline at end of file