x

ooni · Feb 9, 2024 · 30700aa · 30700aa
1 parent 912c3e6
commit 30700aa
Show file tree

Hide file tree

Showing 209 changed files with 9,021 additions and 11,333 deletions.
diff --git a/internal/experiment/webconnectivitylte/cleartextflow.go b/internal/experiment/webconnectivitylte/cleartextflow.go
@@ -8,7 +8,6 @@ package webconnectivitylte
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -102,8 +101,7 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) error {
 	}
 
 	// create trace
-	trace := measurexlite.NewTrace(index, t.ZeroTime, fmt.Sprintf("depth=%d", t.Depth),
-		fmt.Sprintf("fetch_body=%v", t.PrioSelector != nil))
+	trace := measurexlite.NewTrace(index, t.ZeroTime, generateTagsForEndpoints(t.Depth, t.PrioSelector != nil)...)
 
 	// start measuring throttling
 	sampler := throttling.NewSampler(trace)

diff --git a/internal/experiment/webconnectivitylte/secureflow.go b/internal/experiment/webconnectivitylte/secureflow.go
@@ -9,7 +9,6 @@ package webconnectivitylte
 import (
 	"context"
 	"crypto/tls"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -109,8 +108,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error {
 	}
 
 	// create trace
-	trace := measurexlite.NewTrace(index, t.ZeroTime, fmt.Sprintf("depth=%d", t.Depth),
-		fmt.Sprintf("fetch_body=%v", t.PrioSelector != nil))
+	trace := measurexlite.NewTrace(index, t.ZeroTime, generateTagsForEndpoints(t.Depth, t.PrioSelector != nil)...)
 
 	// start measuring throttling
 	sampler := throttling.NewSampler(trace)

diff --git a/internal/experiment/webconnectivitylte/tags.go b/internal/experiment/webconnectivitylte/tags.go
@@ -0,0 +1,13 @@
+package webconnectivitylte
+
+import "fmt"
+
+// generateTagsForEndpoints generates the tags for the endpoints.
+func generateTagsForEndpoints(depth int64, fetchBody bool) (output []string) {
+	output = append(output, fmt.Sprintf("fetch_body=%v", fetchBody))
+	output = append(output, fmt.Sprintf("depth=%d", depth))
+	if depth < 1 {
+		output = append(output, "tcptls_experiment")
+	}
+	return output
+}
diff --git a/...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json b/...inipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis.json
@@ -58,7 +58,7 @@
       "Failure": "ssl_invalid_certificate",
       "TransactionID": 3,
       "TagFetchBody": true,
-      "DNSTransactionID": 2,
+      "DNSTransactionID": 1,
       "DNSDomain": "expired.badssl.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,

diff --git a/...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json b/...ine/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/analysis_classic.json
@@ -55,7 +55,7 @@
       "Failure": "ssl_invalid_certificate",
       "TransactionID": 3,
       "TagFetchBody": true,
-      "DNSTransactionID": 2,
+      "DNSTransactionID": 1,
       "DNSDomain": "expired.badssl.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,

diff --git a/...pipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json b/...pipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json
@@ -9,7 +9,7 @@
     "tunnel": 0
   },
   "input": "https://expired.badssl.com/",
-  "measurement_start_time": "2024-01-23 16:07:26",
+  "measurement_start_time": "2024-02-09 10:05:52",
   "probe_asn": "AS137",
   "probe_cc": "IT",
   "probe_ip": "127.0.0.1",
@@ -37,33 +37,43 @@
         "failure": null,
         "operation": "connect",
         "proto": "tcp",
-        "t0": 0.011584,
-        "t": 0.017182,
+        "t0": 0.006282,
+        "t": 0.011606,
         "transaction_id": 3,
         "tags": [
+          "fetch_body=true",
           "depth=0",
-          "fetch_body=true"
+          "tcptls_experiment"
         ]
       },
       {
         "address": "104.154.89.105:443",
         "failure": null,
-        "num_bytes": 2262,
+        "num_bytes": 2263,
         "operation": "bytes_received_cumulative",
         "proto": "tcp",
-        "t0": 0.02732,
-        "t": 0.02732,
+        "t0": 0.022826,
+        "t": 0.022826,
         "transaction_id": 3,
         "tags": [
+          "fetch_body=true",
           "depth=0",
-          "fetch_body=true"
+          "tcptls_experiment"
         ]
       }
     ],
     "x_dns_whoami": {
-      "system_v4": null,
+      "system_v4": [
+        {
+          "address": "130.192.91.211"
+        }
+      ],
       "udp_v4": {
-        "8.8.4.4:53": null
+        "208.67.222.222:53": [
+          {
+            "address": "130.192.91.211"
+          }
+        ]
       }
     },
     "x_doh": {
@@ -78,60 +88,60 @@
         {
           "failure": null,
           "operation": "resolve_start",
-          "t0": 0.000066,
-          "t": 0.000066,
+          "t0": 0.000068,
+          "t": 0.000068,
           "transaction_id": 1,
           "tags": [
             "depth=0"
           ]
         },
         {
-          "address": "8.8.4.4:53",
+          "address": "208.67.222.222:53",
           "failure": null,
           "num_bytes": 36,
           "operation": "write",
           "proto": "udp",
-          "t0": 0.000103,
-          "t": 0.000123,
+          "t0": 0.000117,
+          "t": 0.000143,
           "transaction_id": 1,
           "tags": [
             "depth=0"
           ]
         },
         {
-          "address": "8.8.4.4:53",
+          "address": "208.67.222.222:53",
           "failure": null,
           "num_bytes": 36,
           "operation": "write",
           "proto": "udp",
-          "t0": 0.000127,
-          "t": 0.000131,
+          "t0": 0.000155,
+          "t": 0.00016,
           "transaction_id": 1,
           "tags": [
             "depth=0"
           ]
         },
         {
-          "address": "8.8.4.4:53",
+          "address": "208.67.222.222:53",
           "failure": null,
-          "num_bytes": 36,
+          "num_bytes": 70,
           "operation": "read",
           "proto": "udp",
-          "t0": 0.000128,
-          "t": 0.005533,
+          "t0": 0.000165,
+          "t": 0.005241,
           "transaction_id": 1,
           "tags": [
             "depth=0"
           ]
         },
         {
-          "address": "8.8.4.4:53",
+          "address": "208.67.222.222:53",
           "failure": null,
-          "num_bytes": 70,
+          "num_bytes": 36,
           "operation": "read",
           "proto": "udp",
-          "t0": 0.000139,
-          "t": 0.005879,
+          "t0": 0.000159,
+          "t": 0.005815,
           "transaction_id": 1,
           "tags": [
             "depth=0"
@@ -140,8 +150,8 @@
         {
           "failure": null,
           "operation": "resolve_done",
-          "t0": 0.005913,
-          "t": 0.005913,
+          "t0": 0.005851,
+          "t": 0.005851,
           "transaction_id": 1,
           "tags": [
             "depth=0"
@@ -162,32 +172,33 @@
             "ttl": null
           }
         ],
-        "engine": "getaddrinfo",
+        "engine": "udp",
         "failure": null,
         "hostname": "expired.badssl.com",
-        "query_type": "ANY",
+        "query_type": "A",
+        "raw_response": "4sKBAAABAAEAAAAAB2V4cGlyZWQGYmFkc3NsA2NvbQAAAQABB2V4cGlyZWQGYmFkc3NsA2NvbQAAAQABAAAOEAAEaJpZaQ==",
         "resolver_hostname": null,
         "resolver_port": null,
-        "resolver_address": "",
-        "t0": 0.00008,
-        "t": 0.005767,
+        "resolver_address": "208.67.222.222:53",
+        "t0": 0.000088,
+        "t": 0.005248,
         "tags": [
           "depth=0"
         ],
-        "transaction_id": 2
+        "transaction_id": 1
       },
       {
         "answers": null,
         "engine": "udp",
         "failure": "dns_no_answer",
         "hostname": "expired.badssl.com",
         "query_type": "AAAA",
-        "raw_response": "Qn+BAAABAAAAAAAAB2V4cGlyZWQGYmFkc3NsA2NvbQAAHAAB",
+        "raw_response": "truBAAABAAAAAAAAB2V4cGlyZWQGYmFkc3NsA2NvbQAAHAAB",
         "resolver_hostname": null,
         "resolver_port": null,
-        "resolver_address": "8.8.4.4:53",
-        "t0": 0.000076,
-        "t": 0.005538,
+        "resolver_address": "208.67.222.222:53",
+        "t0": 0.000082,
+        "t": 0.005829,
         "tags": [
           "depth=0"
         ],
@@ -203,20 +214,19 @@
             "ttl": null
           }
         ],
-        "engine": "udp",
+        "engine": "getaddrinfo",
         "failure": null,
         "hostname": "expired.badssl.com",
-        "query_type": "A",
-        "raw_response": "AWmBAAABAAEAAAAAB2V4cGlyZWQGYmFkc3NsA2NvbQAAAQABB2V4cGlyZWQGYmFkc3NsA2NvbQAAAQABAAAOEAAEaJpZaQ==",
+        "query_type": "ANY",
         "resolver_hostname": null,
         "resolver_port": null,
-        "resolver_address": "8.8.4.4:53",
-        "t0": 0.00011,
-        "t": 0.005884,
+        "resolver_address": "",
+        "t0": 0.000082,
+        "t": 0.006185,
         "tags": [
           "depth=0"
         ],
-        "transaction_id": 1
+        "transaction_id": 2
       }
     ],
     "requests": [],
@@ -228,11 +238,12 @@
           "failure": null,
           "success": true
         },
-        "t0": 0.011584,
-        "t": 0.017182,
+        "t0": 0.006282,
+        "t": 0.011606,
         "tags": [
+          "fetch_body=true",
           "depth=0",
-          "fetch_body=true"
+          "tcptls_experiment"
         ],
         "transaction_id": 3
       }
@@ -247,16 +258,17 @@
         "no_tls_verify": false,
         "peer_certificates": [
           {
-            "data": "MIIDeDCCAmCgAwIBAgIUDFii3o/E+8RXCvfb8zpfNhlAqkEwDQYJKoZIhvcNAQELBQAwHzENMAsGA1UEChMET09OSTEOMAwGA1UEAxMFamFmYXIwHhcNMTcwNzE2MjMwMDAwWhcNMTcwNzE3MDEwMDAwWjA1MRYwFAYDVQQKEw1PT05JIE5ldGVtIENBMRswGQYDVQQDExJleHBpcmVkLmJhZHNzbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDlXTSSEC7UqkIbbCkZw4jCAUoA5POEkqPrkVC2gYKP8O69GjqVU0zg9nCKtl2Bymu+Gn9mHDK8BvRVxcMSz+Iieq0QFE2QgRJYRS6wfvwCseRffbkhGz3dbc+Etz9F2nTgkwCHOtl5ZrXr5mB9F4k2RMmtF+rRRqfBTGEnOjDCA3iiUTrU/Fn015y4rOVgDP0sa6iwATUjzDOKzdFI69Kjr2p2wKXE5MQn0v7P99fGIUfcjCeQtf9NGvJiLGvJ5u0HA1G1mq4iGIt4ys/8uoQS+EWKiOOpr0hsslAUTxaI32gay10ql0eSj573Ncm8+LoqoPvE4iotUwF/Bamdih75AgMBAAGjgZUwgZIwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFFGFlo8SP5sBLhZY49xYlcP/khsUMB8GA1UdIwQYMBaAFJqH2J23b2bH/VWSzxVZdR6WJjOwMB0GA1UdEQQWMBSCEmV4cGlyZWQuYmFkc3NsLmNvbTANBgkqhkiG9w0BAQsFAAOCAQEACdpL6NAWcenzZJ1rXky1rZcqYEeRQ/cH5FZeoxZA/wcMyrHaqNptAraPi2tnSwvAl9aR7Go8tc8j0UAX+hECQaMibkazpeiRcHZYM74XKLjxFX7FyzWGEuU8OMzDTxsYF4FUdv7Pkv211nEPZf05gMWVRBSgYYplnPu5YYpPaRwWwl0i0bbrsTfd1RhTdrAjlSp/gRDRlCUdiPtNtPtW4EKk5PLBWyLMR8TFO7MwtsgBwgY7qTHPEpzAX4CQMBq2iuNL7wKR7e6zZQGK92E5fUuvcuSNPy6FxpYs8nMZgD72de3Pnumk3gyN0745xXCq4mLBWagMv8BsHeeipJ3Zqw==",
+            "data": "MIIDeTCCAmGgAwIBAgIVAOHk8jvh+0I/SHdrZ7BMcXei+udyMA0GCSqGSIb3DQEBCwUAMB8xDTALBgNVBAoTBE9PTkkxDjAMBgNVBAMTBWphZmFyMB4XDTE3MDcxNjIzMDAwMFoXDTE3MDcxNzAxMDAwMFowNTEWMBQGA1UEChMNT09OSSBOZXRlbSBDQTEbMBkGA1UEAxMSZXhwaXJlZC5iYWRzc2wuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw5TlPLfhBKTcSfloQ42bYikYXtrr8kiQ1Ez0viXUDrnREcc0QILvW8AzgkFb2m7ChIFqHsXRrygJIA9ip7oYfOLq7EADHlTMgD5QY0FMxWnKJfTEnmkQVVspk0GB9o0PdZ3cmIWm9RgVOEc/ZINH/UO4b8VT6M8XmbzKEsfGypo4Kk35dKeMx37XweeCM+naxeCy5KDJ0Z1CPieGJLrnFTKFFpYhJvxX2ni97Qvbqe2LnagcH5RKjiUvwPQpw8Xc/K80Sd9uJtz4Chrt+CxHfghIJdxmubA/3ytKyERWkWCQiyn1lEYfPVE90Fqlhh9M+C+UDkS9DOcHb4VGh2d7ZQIDAQABo4GVMIGSMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBScYEk6DsKDIta5hIBYm5De2mNwuDAfBgNVHSMEGDAWgBTvEj/ucU4ehiuZYCO8SB+AakHxizAdBgNVHREEFjAUghJleHBpcmVkLmJhZHNzbC5jb20wDQYJKoZIhvcNAQELBQADggEBADcWUAq1UNSbd4QgB4hebcdI8sGnbA28UDG93fFqcfn++dLGc9lElSLaY4Pj5G3Se9/K9h7HQw5eADtubOL2dWiZdxaffbfM/UsL6hCNQejMOY2Qjo65PakUIZJYMQSoZ1MmqFVUejD9pp1Q5VBsCMJOP9PRoRPG9cpdgJi50u7TmH92OsU2g32fwSlGfSY1eUkCqYz4UZ5hxVVEHwGcXQtsF5qtyjOUWtOzldSdU4ZjSxu5GF0y7dZ8+JxEfHOY2lUVSzEMbUBhA9Bq7a1ITKrDeLBRPs8oGTF6BHmmR2skFt3JZYUxnZe5GO6AFWP++dnp3yWV4KMZKk3ZhaZFZCo=",
             "format": "base64"
           }
         ],
         "server_name": "expired.badssl.com",
-        "t0": 0.017197,
-        "t": 0.027298,
+        "t0": 0.011628,
+        "t": 0.02279,
         "tags": [
+          "fetch_body=true",
           "depth=0",
-          "fetch_body=true"
+          "tcptls_experiment"
         ],
         "tls_version": "",
         "transaction_id": 3
@@ -321,7 +333,7 @@
     "x_conn_priority_log": [
       {
         "msg": "create with [{Addr:104.154.89.105 Flags:3}]",
-        "t": 0.011551
+        "t": 0.006238
       }
     ],
     "control_failure": null,
@@ -340,7 +352,7 @@
     "accessible": false
   },
   "test_name": "web_connectivity",
-  "test_runtime": 0.506011,
-  "test_start_time": "2024-01-23 16:07:26",
+  "test_runtime": 0.506336,
+  "test_start_time": "2024-02-09 10:05:52",
   "test_version": "0.5.28"
 }
diff --git a/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json b/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json
@@ -51,13 +51,13 @@
       "TagDepth": 0,
       "Type": 0,
       "Failure": "",
-      "TransactionID": 2,
+      "TransactionID": 1,
       "TagFetchBody": null,
-      "DNSTransactionID": 2,
+      "DNSTransactionID": 1,
       "DNSDomain": "expired.badssl.com",
       "DNSLookupFailure": "",
-      "DNSQueryType": "ANY",
-      "DNSEngine": "getaddrinfo",
+      "DNSQueryType": "A",
+      "DNSEngine": "udp",
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
@@ -98,13 +98,13 @@
       "TagDepth": 0,
       "Type": 0,
       "Failure": "",
-      "TransactionID": 1,
+      "TransactionID": 2,
       "TagFetchBody": null,
-      "DNSTransactionID": 1,
+      "DNSTransactionID": 2,
       "DNSDomain": "expired.badssl.com",
       "DNSLookupFailure": "",
-      "DNSQueryType": "A",
-      "DNSEngine": "udp",
+      "DNSQueryType": "ANY",
+      "DNSEngine": "getaddrinfo",
       "DNSResolvedAddrs": [
         "104.154.89.105"
       ],
@@ -149,7 +149,7 @@
       "Failure": "ssl_invalid_certificate",
       "TransactionID": 3,
       "TagFetchBody": true,
-      "DNSTransactionID": 2,
+      "DNSTransactionID": 1,
       "DNSDomain": "expired.badssl.com",
       "DNSLookupFailure": "",
       "DNSQueryType": null,