fix(qatool): normalize test keys (ooni#1510)

Zero out times and zero LTE measurement results not used by
minipipeline.

Helps with ooni/probe#2677.

internal/cmd/qatool/main.go

@@ -68,16 +68,35 @@ func runWebConnectivityLTE(tc *webconnectivityqa.TestCase) {
 		// obtain the test keys
 		tk := measurement.TestKeys.(*webconnectivitylte.TestKeys)
 
-		// normalize the test keys
+		// Normalize the test keys
 		//
-		// see https://github.com/ooni/probe/issues/2677
+		// The general idea here is to remove everything that we do not use in the
+		// minipipeline to reduce the sizes of the diffs we commit.
+		//
+		// See https://github.com/ooni/probe/issues/2677
 		tk.Queries = minipipeline.SortDNSLookupResults(tk.Queries)
-		tk.Do53.Queries = minipipeline.SortDNSLookupResults(tk.Do53.Queries)
-		tk.DoH.Queries = minipipeline.SortDNSLookupResults(tk.DoH.Queries)
-		tk.DNSDuplicateResponses = minipipeline.SortDNSLookupResults(tk.DNSDuplicateResponses)
-		tk.NetworkEvents = minipipeline.SortNetworkEvents(tk.NetworkEvents)
+		minipipeline.NormalizeDNSLookupResults(tk.Queries)
+
+		tk.Do53 = nil
+		tk.DoH = nil
+		tk.DNSDuplicateResponses = nil
+		tk.DNSWoami = nil
+		tk.ConnPriorityLog = nil
+
+		tk.NetworkEvents = nil
+
 		tk.TCPConnect = minipipeline.SortTCPConnectResults(tk.TCPConnect)
+		minipipeline.NormalizeTCPConnectResults(tk.TCPConnect)
+
 		tk.TLSHandshakes = minipipeline.SortTLSHandshakeResults(tk.TLSHandshakes)
+		minipipeline.NormalizeTLSHandshakeResults(tk.TLSHandshakes)
+
+		minipipeline.NormalizeHTTPRequestResults(tk.Requests)
+
+		// normalize measurement fields
+		measurement.MeasurementStartTime = "2024-02-12 20:33:47"
+		measurement.MeasurementRuntime = 0
+		measurement.TestStartTime = "2024-02-12 20:33:47"
 
 		// serialize the original measurement
 		mustSerializeMkdirAllAndWriteFile(actualDestdir, "measurement.json", measurement)

internal/minipipeline/normalize.go

@@ -0,0 +1,54 @@
+package minipipeline
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+// NormalizeDNSLookupResults MUTATES values in input to zero its T0 and T fields and applies
+// other normalizations meant to reduce the size of diffs.
+func NormalizeDNSLookupResults(values []*model.ArchivalDNSLookupResult) {
+	for _, entry := range values {
+		switch entry.Engine {
+		case "udp":
+			entry.ResolverAddress = "1.1.1.1:53"
+		case "doh":
+			entry.ResolverAddress = "https://dns.google/dns-query"
+		}
+		entry.T0 = 0
+		entry.T = 0
+		entry.RawResponse = nil
+	}
+}
+
+// NormalizeNetworkEvents is like [NormalizeDNSLookupResults] but for network events.
+func NormalizeNetworkEvents(values []*model.ArchivalNetworkEvent) {
+	for _, entry := range values {
+		entry.T0 = 0
+		entry.T = 0
+	}
+}
+
+// NormalizeTCPConnectResults is like [NormalizeDNSLookupResults] but for TCP connect results.
+func NormalizeTCPConnectResults(values []*model.ArchivalTCPConnectResult) {
+	for _, entry := range values {
+		entry.T0 = 0
+		entry.T = 0
+	}
+}
+
+// NormalizeTLSHandshakeResults is like [NormalizeDNSLookupResults] but for TLS handshake results.
+func NormalizeTLSHandshakeResults(values []*model.ArchivalTLSOrQUICHandshakeResult) {
+	for _, entry := range values {
+		entry.T0 = 0
+		entry.T = 0
+		entry.PeerCertificates = nil
+	}
+}
+
+// NormalizeHTTPRequestResults is like [NormalizeDNSLookupResults] but for HTTP requests.
+func NormalizeHTTPRequestResults(values []*model.ArchivalHTTPRequestResult) {
+	for _, entry := range values {
+		entry.T0 = 0
+		entry.T = 0
+	}
+}

internal/minipipeline/normalize_test.go

@@ -0,0 +1,250 @@
+package minipipeline
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func TestNormalizeDNSLookupResults(t *testing.T) {
+	type testcase struct {
+		name     string
+		inputGen func() []*model.ArchivalDNSLookupResult
+		expect   []*model.ArchivalDNSLookupResult
+	}
+
+	cases := []testcase{{
+		name: "with nil input",
+		inputGen: func() []*model.ArchivalDNSLookupResult {
+			return nil
+		},
+		expect: nil,
+	}, {
+		name: "with empty input",
+		inputGen: func() []*model.ArchivalDNSLookupResult {
+			return []*model.ArchivalDNSLookupResult{}
+		},
+		expect: []*model.ArchivalDNSLookupResult{},
+	}, {
+		name: "with plausible input",
+		inputGen: func() []*model.ArchivalDNSLookupResult {
+			return []*model.ArchivalDNSLookupResult{{
+				Engine:      "udp",
+				RawResponse: []byte("0xdeadbeef"),
+				T0:          0.11,
+				T:           0.4,
+			}, {
+				Engine:      "doh",
+				RawResponse: []byte("0xdeadbeef"),
+				T0:          0.5,
+				T:           0.66,
+			}}
+		},
+		expect: []*model.ArchivalDNSLookupResult{{
+			Engine:          "udp",
+			ResolverAddress: "1.1.1.1:53",
+		}, {
+			Engine:          "doh",
+			ResolverAddress: "https://dns.google/dns-query",
+		}},
+	}}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			values := tc.inputGen()
+
+			NormalizeDNSLookupResults(values)
+
+			if diff := cmp.Diff(tc.expect, values); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNormalizeNetworkEvents(t *testing.T) {
+	type testcase struct {
+		name     string
+		inputGen func() []*model.ArchivalNetworkEvent
+		expect   []*model.ArchivalNetworkEvent
+	}
+
+	cases := []testcase{{
+		name: "with nil input",
+		inputGen: func() []*model.ArchivalNetworkEvent {
+			return nil
+		},
+		expect: nil,
+	}, {
+		name: "with empty input",
+		inputGen: func() []*model.ArchivalNetworkEvent {
+			return []*model.ArchivalNetworkEvent{}
+		},
+		expect: []*model.ArchivalNetworkEvent{},
+	}, {
+		name: "with plausible input",
+		inputGen: func() []*model.ArchivalNetworkEvent {
+			return []*model.ArchivalNetworkEvent{{
+				T0: 0.11,
+				T:  0.4,
+			}, {
+				T0: 0.5,
+				T:  0.66,
+			}}
+		},
+		expect: []*model.ArchivalNetworkEvent{{}, {}},
+	}}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			values := tc.inputGen()
+
+			NormalizeNetworkEvents(values)
+
+			if diff := cmp.Diff(tc.expect, values); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNormalizeTCPConnectResults(t *testing.T) {
+	type testcase struct {
+		name     string
+		inputGen func() []*model.ArchivalTCPConnectResult
+		expect   []*model.ArchivalTCPConnectResult
+	}
+
+	cases := []testcase{{
+		name: "with nil input",
+		inputGen: func() []*model.ArchivalTCPConnectResult {
+			return nil
+		},
+		expect: nil,
+	}, {
+		name: "with empty input",
+		inputGen: func() []*model.ArchivalTCPConnectResult {
+			return []*model.ArchivalTCPConnectResult{}
+		},
+		expect: []*model.ArchivalTCPConnectResult{},
+	}, {
+		name: "with plausible input",
+		inputGen: func() []*model.ArchivalTCPConnectResult {
+			return []*model.ArchivalTCPConnectResult{{
+				T0: 0.11,
+				T:  0.4,
+			}, {
+				T0: 0.5,
+				T:  0.66,
+			}}
+		},
+		expect: []*model.ArchivalTCPConnectResult{{}, {}},
+	}}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			values := tc.inputGen()
+
+			NormalizeTCPConnectResults(values)
+
+			if diff := cmp.Diff(tc.expect, values); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNormalizeTLSHandshakeResults(t *testing.T) {
+	type testcase struct {
+		name     string
+		inputGen func() []*model.ArchivalTLSOrQUICHandshakeResult
+		expect   []*model.ArchivalTLSOrQUICHandshakeResult
+	}
+
+	cases := []testcase{{
+		name: "with nil input",
+		inputGen: func() []*model.ArchivalTLSOrQUICHandshakeResult {
+			return nil
+		},
+		expect: nil,
+	}, {
+		name: "with empty input",
+		inputGen: func() []*model.ArchivalTLSOrQUICHandshakeResult {
+			return []*model.ArchivalTLSOrQUICHandshakeResult{}
+		},
+		expect: []*model.ArchivalTLSOrQUICHandshakeResult{},
+	}, {
+		name: "with plausible input",
+		inputGen: func() []*model.ArchivalTLSOrQUICHandshakeResult {
+			return []*model.ArchivalTLSOrQUICHandshakeResult{{
+				PeerCertificates: []model.ArchivalBinaryData{[]byte("0xdeadbeef")},
+				T0:               0.11,
+				T:                0.4,
+			}, {
+				PeerCertificates: []model.ArchivalBinaryData{[]byte("0xdeadbeef")},
+				T0:               0.5,
+				T:                0.66,
+			}}
+		},
+		expect: []*model.ArchivalTLSOrQUICHandshakeResult{{}, {}},
+	}}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			values := tc.inputGen()
+
+			NormalizeTLSHandshakeResults(values)
+
+			if diff := cmp.Diff(tc.expect, values); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestNormalizeHTTPRequestResults(t *testing.T) {
+	type testcase struct {
+		name     string
+		inputGen func() []*model.ArchivalHTTPRequestResult
+		expect   []*model.ArchivalHTTPRequestResult
+	}
+
+	cases := []testcase{{
+		name: "with nil input",
+		inputGen: func() []*model.ArchivalHTTPRequestResult {
+			return nil
+		},
+		expect: nil,
+	}, {
+		name: "with empty input",
+		inputGen: func() []*model.ArchivalHTTPRequestResult {
+			return []*model.ArchivalHTTPRequestResult{}
+		},
+		expect: []*model.ArchivalHTTPRequestResult{},
+	}, {
+		name: "with plausible input",
+		inputGen: func() []*model.ArchivalHTTPRequestResult {
+			return []*model.ArchivalHTTPRequestResult{{
+				T0: 0.11,
+				T:  0.4,
+			}, {
+				T0: 0.5,
+				T:  0.66,
+			}}
+		},
+		expect: []*model.ArchivalHTTPRequestResult{{}, {}},
+	}}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			values := tc.inputGen()
+
+			NormalizeHTTPRequestResults(values)
+
+			if diff := cmp.Diff(tc.expect, values); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}