diff --git a/internal/cmd/qatool/main.go b/internal/cmd/qatool/main.go index 8afe08a94..07bd2068a 100644 --- a/internal/cmd/qatool/main.go +++ b/internal/cmd/qatool/main.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "regexp" + "sort" + "time" "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" "github.com/ooni/probe-cli/v3/internal/geoipx" @@ -137,6 +139,23 @@ func runWebConnectivityLTE(tc *webconnectivityqa.TestCase) { } } +// override webconnectivitylte algorithm to make it less entropic +func init() { + webconnectivitylte.MaybeSortAddresses = func(entries []webconnectivitylte.DNSEntry) { + sort.SliceStable(entries, func(i, j int) bool { + return entries[i].Addr < entries[j].Addr + }) + } + + webconnectivitylte.MaybeDelayCleartextFlows = func(index int) { + time.Sleep(10 * time.Millisecond * time.Duration(index)) + } + + webconnectivitylte.MaybeDelaySecureFlows = func(index int) { + time.Sleep(10 * time.Millisecond * time.Duration(index)) + } +} + func main() { // parse command line flags flag.Parse() diff --git a/internal/experiment/webconnectivitylte/dnsresolvers.go b/internal/experiment/webconnectivitylte/dnsresolvers.go index 460807508..f5561450d 100644 --- a/internal/experiment/webconnectivitylte/dnsresolvers.go +++ b/internal/experiment/webconnectivitylte/dnsresolvers.go @@ -92,6 +92,11 @@ func (t *DNSResolvers) Start(ctx context.Context) { }() } +// MaybeSortAddresses is an OPTIONAL hook that possibly sorts the resolved addresses. +var MaybeSortAddresses = func(entries []DNSEntry) { + // nothing! +} + // run performs a DNS lookup and returns the looked up addrs func (t *DNSResolvers) run(parentCtx context.Context) []DNSEntry { // create output channels for the lookup @@ -153,6 +158,9 @@ func (t *DNSResolvers) run(parentCtx context.Context) []DNSEntry { entries = append(entries, *entry) } + // allow specific users to sort addresses if needed + MaybeSortAddresses(entries) + return entries } @@ -362,6 +370,11 @@ func (t *DNSResolvers) dohSplitQueries( return } +// MaybeDelayCleartextFlows is an OPTIONAL hook that possibly delays cleartext flows. +var MaybeDelayCleartextFlows = func(index int) { + // nothing +} + // startCleartextFlows starts a TCP measurement flow for each IP addr. func (t *DNSResolvers) startCleartextFlows( ctx context.Context, @@ -377,7 +390,8 @@ func (t *DNSResolvers) startCleartextFlows( if urlPort := t.URL.Port(); urlPort != "" { port = urlPort } - for _, addr := range addresses { + for index, addr := range addresses { + MaybeDelayCleartextFlows(index) // allow specific callers to space flows apart task := &CleartextFlow{ Address: net.JoinHostPort(addr.Addr, port), Depth: t.Depth, @@ -402,6 +416,11 @@ func (t *DNSResolvers) startCleartextFlows( } } +// MaybeDelaySecureFlows is an OPTIONAL hook that possibly delays secure flows. +var MaybeDelaySecureFlows = func(index int) { + // nothing +} + // startSecureFlows starts a TCP+TLS measurement flow for each IP addr. func (t *DNSResolvers) startSecureFlows( ctx context.Context, @@ -421,7 +440,8 @@ func (t *DNSResolvers) startSecureFlows( } port = urlPort } - for _, addr := range addresses { + for index, addr := range addresses { + MaybeDelaySecureFlows(index) // allow specific callers to space flows apart task := &SecureFlow{ Address: net.JoinHostPort(addr.Addr, port), Depth: t.Depth, diff --git a/internal/minipipeline/normalize.go b/internal/minipipeline/normalize.go index 2faa7f5b8..b8b6c0297 100644 --- a/internal/minipipeline/normalize.go +++ b/internal/minipipeline/normalize.go @@ -54,7 +54,7 @@ func NormalizeHTTPRequestResults(values []*model.ArchivalHTTPRequestResult) { // Avoid storing large bodies because that wastes repository space. // // See https://github.com/ooni/probe/issues/2677. - const maxStoreBody = 1 << 14 + const maxStoreBody = 12_000 if len(entry.Response.Body) > maxStoreBody { entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString("") entry.Response.BodyIsTruncated = true diff --git a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis.json b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis.json index c913f8542..c822187e2 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis.json @@ -49,7 +49,7 @@ "HTTPFinalResponseSuccessTLSWithoutControl": null, "HTTPFinalResponseSuccessTLSWithControl": null, "HTTPFinalResponseSuccessTCPWithoutControl": null, - "HTTPFinalResponseSuccessTCPWithControl": 40002, + "HTTPFinalResponseSuccessTCPWithControl": 40001, "HTTPFinalResponseDiffBodyProportionFactor": 0.12263535551206783, "HTTPFinalResponseDiffStatusCodeMatch": true, "HTTPFinalResponseDiffTitleDifferentLongWords": { @@ -63,7 +63,7 @@ "TagDepth": 0, "Type": 3, "Failure": "", - "TransactionID": 40002, + "TransactionID": 40001, "TagFetchBody": true, "DNSTransactionID": 10001, "DNSDomain": "www.example.com", @@ -77,7 +77,7 @@ "IPAddress": "130.192.182.17", "IPAddressASN": 137, "IPAddressBogon": false, - "EndpointTransactionID": 40002, + "EndpointTransactionID": 40001, "EndpointProto": "tcp", "EndpointPort": "80", "EndpointAddress": "130.192.182.17:80", @@ -117,22 +117,22 @@ "Failure": "", "TransactionID": 50002, "TagFetchBody": false, - "DNSTransactionID": 10001, + "DNSTransactionID": 30001, "DNSDomain": "www.example.com", "DNSLookupFailure": "", "DNSQueryType": null, "DNSEngine": null, "DNSResolvedAddrs": [ - "130.192.182.17" + "93.184.216.34" ], "IPAddressOrigin": "dns", - "IPAddress": "130.192.182.17", - "IPAddressASN": 137, + "IPAddress": "93.184.216.34", + "IPAddressASN": 15133, "IPAddressBogon": false, "EndpointTransactionID": 50002, "EndpointProto": "tcp", "EndpointPort": "443", - "EndpointAddress": "130.192.182.17:443", + "EndpointAddress": "93.184.216.34:443", "TCPConnectFailure": "", "TLSHandshakeFailure": "", "TLSServerName": "www.example.com", @@ -169,22 +169,22 @@ "Failure": "", "TransactionID": 50001, "TagFetchBody": false, - "DNSTransactionID": 30001, + "DNSTransactionID": 10001, "DNSDomain": "www.example.com", "DNSLookupFailure": "", "DNSQueryType": null, "DNSEngine": null, "DNSResolvedAddrs": [ - "93.184.216.34" + "130.192.182.17" ], "IPAddressOrigin": "dns", - "IPAddress": "93.184.216.34", - "IPAddressASN": 15133, + "IPAddress": "130.192.182.17", + "IPAddressASN": 137, "IPAddressBogon": false, "EndpointTransactionID": 50001, "EndpointProto": "tcp", "EndpointPort": "443", - "EndpointAddress": "93.184.216.34:443", + "EndpointAddress": "130.192.182.17:443", "TCPConnectFailure": "", "TLSHandshakeFailure": "", "TLSServerName": "www.example.com", @@ -219,7 +219,7 @@ "TagDepth": 0, "Type": 1, "Failure": "", - "TransactionID": 40001, + "TransactionID": 40002, "TagFetchBody": true, "DNSTransactionID": 30001, "DNSDomain": "www.example.com", @@ -233,7 +233,7 @@ "IPAddress": "93.184.216.34", "IPAddressASN": 15133, "IPAddressBogon": false, - "EndpointTransactionID": 40001, + "EndpointTransactionID": 40002, "EndpointProto": "tcp", "EndpointPort": "80", "EndpointAddress": "93.184.216.34:80", diff --git a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis_classic.json b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis_classic.json index 7374ae576..f2c473f77 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis_classic.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/analysis_classic.json @@ -41,7 +41,7 @@ "HTTPFinalResponseSuccessTLSWithoutControl": null, "HTTPFinalResponseSuccessTLSWithControl": null, "HTTPFinalResponseSuccessTCPWithoutControl": null, - "HTTPFinalResponseSuccessTCPWithControl": 40002, + "HTTPFinalResponseSuccessTCPWithControl": 40001, "HTTPFinalResponseDiffBodyProportionFactor": 0.12263535551206783, "HTTPFinalResponseDiffStatusCodeMatch": true, "HTTPFinalResponseDiffTitleDifferentLongWords": { @@ -55,7 +55,7 @@ "TagDepth": 0, "Type": 3, "Failure": "", - "TransactionID": 40002, + "TransactionID": 40001, "TagFetchBody": true, "DNSTransactionID": 10001, "DNSDomain": "www.example.com", @@ -69,7 +69,7 @@ "IPAddress": "130.192.182.17", "IPAddressASN": 137, "IPAddressBogon": false, - "EndpointTransactionID": 40002, + "EndpointTransactionID": 40001, "EndpointProto": "tcp", "EndpointPort": "80", "EndpointAddress": "130.192.182.17:80", diff --git a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json index 502e265d7..32c6615b5 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/measurement.json @@ -195,7 +195,7 @@ "depth=0", "fetch_body=true" ], - "transaction_id": 40002 + "transaction_id": 40001 } ], "tcp_connect": [ @@ -211,7 +211,7 @@ "depth=0", "fetch_body=true" ], - "transaction_id": 40002 + "transaction_id": 40001 }, { "ip": "130.192.182.17", @@ -225,7 +225,7 @@ "depth=0", "fetch_body=false" ], - "transaction_id": 50002 + "transaction_id": 50001 }, { "ip": "93.184.216.34", @@ -239,7 +239,7 @@ "depth=0", "fetch_body=true" ], - "transaction_id": 40001 + "transaction_id": 40002 }, { "ip": "93.184.216.34", @@ -253,7 +253,7 @@ "depth=0", "fetch_body=false" ], - "transaction_id": 50001 + "transaction_id": 50002 } ], "tls_handshakes": [ @@ -272,7 +272,7 @@ "fetch_body=false" ], "tls_version": "TLSv1.3", - "transaction_id": 50002 + "transaction_id": 50001 }, { "network": "tcp", @@ -289,7 +289,7 @@ "fetch_body=false" ], "tls_version": "TLSv1.3", - "transaction_id": 50001 + "transaction_id": 50002 } ], "x_control_request": { @@ -306,10 +306,10 @@ ] }, "tcp_connect": [ - "93.184.216.34:443", - "93.184.216.34:80", "130.192.182.17:443", - "130.192.182.17:80" + "130.192.182.17:80", + "93.184.216.34:443", + "93.184.216.34:80" ], "x_quic_enabled": false }, diff --git a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations.json b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations.json index 91a0ebf82..434495113 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations.json @@ -262,38 +262,38 @@ "KnownTCPEndpoints": { "40001": { "TagDepth": 0, - "Type": 1, + "Type": 3, "Failure": "", "TransactionID": 40001, "TagFetchBody": true, - "DNSTransactionID": 30001, + "DNSTransactionID": 10001, "DNSDomain": "www.example.com", "DNSLookupFailure": "", "DNSQueryType": null, "DNSEngine": null, "DNSResolvedAddrs": [ - "93.184.216.34" + "130.192.182.17" ], "IPAddressOrigin": "dns", - "IPAddress": "93.184.216.34", - "IPAddressASN": 15133, + "IPAddress": "130.192.182.17", + "IPAddressASN": 137, "IPAddressBogon": false, "EndpointTransactionID": 40001, "EndpointProto": "tcp", "EndpointPort": "80", - "EndpointAddress": "93.184.216.34:80", + "EndpointAddress": "130.192.182.17:80", "TCPConnectFailure": "", "TLSHandshakeFailure": null, "TLSServerName": null, - "HTTPRequestURL": null, - "HTTPFailure": null, - "HTTPResponseStatusCode": null, - "HTTPResponseBodyLength": null, - "HTTPResponseBodyIsTruncated": null, - "HTTPResponseHeadersKeys": null, + "HTTPRequestURL": "http://www.example.com/", + "HTTPFailure": "", + "HTTPResponseStatusCode": 200, + "HTTPResponseBodyLength": 188, + "HTTPResponseBodyIsTruncated": false, + "HTTPResponseHeadersKeys": {}, "HTTPResponseLocation": null, - "HTTPResponseTitle": null, - "HTTPResponseIsFinal": null, + "HTTPResponseTitle": "Access Denied", + "HTTPResponseIsFinal": true, "ControlDNSDomain": "www.example.com", "ControlDNSLookupFailure": "", "ControlDNSResolvedAddrs": [ @@ -314,38 +314,38 @@ }, "40002": { "TagDepth": 0, - "Type": 3, + "Type": 1, "Failure": "", "TransactionID": 40002, "TagFetchBody": true, - "DNSTransactionID": 10001, + "DNSTransactionID": 30001, "DNSDomain": "www.example.com", "DNSLookupFailure": "", "DNSQueryType": null, "DNSEngine": null, "DNSResolvedAddrs": [ - "130.192.182.17" + "93.184.216.34" ], "IPAddressOrigin": "dns", - "IPAddress": "130.192.182.17", - "IPAddressASN": 137, + "IPAddress": "93.184.216.34", + "IPAddressASN": 15133, "IPAddressBogon": false, "EndpointTransactionID": 40002, "EndpointProto": "tcp", "EndpointPort": "80", - "EndpointAddress": "130.192.182.17:80", + "EndpointAddress": "93.184.216.34:80", "TCPConnectFailure": "", "TLSHandshakeFailure": null, "TLSServerName": null, - "HTTPRequestURL": "http://www.example.com/", - "HTTPFailure": "", - "HTTPResponseStatusCode": 200, - "HTTPResponseBodyLength": 188, - "HTTPResponseBodyIsTruncated": false, - "HTTPResponseHeadersKeys": {}, + "HTTPRequestURL": null, + "HTTPFailure": null, + "HTTPResponseStatusCode": null, + "HTTPResponseBodyLength": null, + "HTTPResponseBodyIsTruncated": null, + "HTTPResponseHeadersKeys": null, "HTTPResponseLocation": null, - "HTTPResponseTitle": "Access Denied", - "HTTPResponseIsFinal": true, + "HTTPResponseTitle": null, + "HTTPResponseIsFinal": null, "ControlDNSDomain": "www.example.com", "ControlDNSLookupFailure": "", "ControlDNSResolvedAddrs": [ @@ -370,22 +370,22 @@ "Failure": "", "TransactionID": 50001, "TagFetchBody": false, - "DNSTransactionID": 30001, + "DNSTransactionID": 10001, "DNSDomain": "www.example.com", "DNSLookupFailure": "", "DNSQueryType": null, "DNSEngine": null, "DNSResolvedAddrs": [ - "93.184.216.34" + "130.192.182.17" ], "IPAddressOrigin": "dns", - "IPAddress": "93.184.216.34", - "IPAddressASN": 15133, + "IPAddress": "130.192.182.17", + "IPAddressASN": 137, "IPAddressBogon": false, "EndpointTransactionID": 50001, "EndpointProto": "tcp", "EndpointPort": "443", - "EndpointAddress": "93.184.216.34:443", + "EndpointAddress": "130.192.182.17:443", "TCPConnectFailure": "", "TLSHandshakeFailure": "", "TLSServerName": "www.example.com", @@ -422,22 +422,22 @@ "Failure": "", "TransactionID": 50002, "TagFetchBody": false, - "DNSTransactionID": 10001, + "DNSTransactionID": 30001, "DNSDomain": "www.example.com", "DNSLookupFailure": "", "DNSQueryType": null, "DNSEngine": null, "DNSResolvedAddrs": [ - "130.192.182.17" + "93.184.216.34" ], "IPAddressOrigin": "dns", - "IPAddress": "130.192.182.17", - "IPAddressASN": 137, + "IPAddress": "93.184.216.34", + "IPAddressASN": 15133, "IPAddressBogon": false, "EndpointTransactionID": 50002, "EndpointProto": "tcp", "EndpointPort": "443", - "EndpointAddress": "130.192.182.17:443", + "EndpointAddress": "93.184.216.34:443", "TCPConnectFailure": "", "TLSHandshakeFailure": "", "TLSServerName": "www.example.com", diff --git a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations_classic.json b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations_classic.json index db6a7bcc0..b79059424 100644 --- a/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations_classic.json +++ b/internal/minipipeline/testdata/webconnectivity/generated/httpDiffWithInconsistentDNS/observations_classic.json @@ -55,11 +55,11 @@ } ], "KnownTCPEndpoints": { - "40002": { + "40001": { "TagDepth": 0, "Type": 3, "Failure": "", - "TransactionID": 40002, + "TransactionID": 40001, "TagFetchBody": true, "DNSTransactionID": 10001, "DNSDomain": "www.example.com", @@ -73,7 +73,7 @@ "IPAddress": "130.192.182.17", "IPAddressASN": 137, "IPAddressBogon": false, - "EndpointTransactionID": 40002, + "EndpointTransactionID": 40001, "EndpointProto": "tcp", "EndpointPort": "80", "EndpointAddress": "130.192.182.17:80",