x

ooni · Nov 29, 2023 · 907114f · 907114f
1 parent 4e7f82b
commit 907114f
Show file tree

Hide file tree

Showing 69 changed files with 1,174 additions and 262 deletions.
diff --git a/internal/cmd/minipipeline/testdata/observations.json b/internal/cmd/minipipeline/testdata/observations.json
@@ -25,6 +25,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "nexa.polito.it",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -62,6 +64,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "nexa.polito.it",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -101,6 +105,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -138,6 +144,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -175,6 +183,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -229,8 +239,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino",
       "HTTPResponseIsFinal": true,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/internal/cmd/minipipeline/testdata/observations_classic.json b/internal/cmd/minipipeline/testdata/observations_classic.json
@@ -26,6 +26,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -80,8 +82,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino",
       "HTTPResponseIsFinal": true,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "nexa.polito.it",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json b/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json
@@ -25,6 +25,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "expired.badssl.com",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -64,6 +66,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -101,6 +105,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -140,8 +146,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "expired.badssl.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json b/...testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations_classic.json
@@ -26,6 +26,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -65,8 +67,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "expired.badssl.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...a/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json b/...a/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json
@@ -25,6 +25,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "untrusted-root.badssl.com",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -62,6 +64,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "untrusted-root.badssl.com",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -101,6 +105,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -138,6 +144,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -175,6 +183,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -214,8 +224,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "untrusted-root.badssl.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...nectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations_classic.json b/...nectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations_classic.json
@@ -26,6 +26,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -65,8 +67,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "untrusted-root.badssl.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json b/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json
@@ -25,6 +25,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "www.example.com",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -64,6 +66,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -101,6 +105,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -140,8 +146,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "www.example.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": false,
       "MatchWithControlIPAddressASN": false,
@@ -187,6 +195,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": "Default Web Page",
       "HTTPResponseIsFinal": true,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": "",

diff --git a/...ctivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json b/...ctivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations_classic.json
@@ -26,6 +26,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -65,8 +67,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "www.example.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": false,
       "MatchWithControlIPAddressASN": false,

diff --git a/...nipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations.json b/...nipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations.json
@@ -25,6 +25,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": "wrong.host.badssl.com",
       "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": null,
@@ -64,6 +66,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -101,6 +105,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -140,8 +146,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "wrong.host.badssl.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...ne/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations_classic.json b/...ne/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations_classic.json
@@ -26,6 +26,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -65,8 +67,10 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
-      "ControlDNSDomain": null,
-      "ControlDNSLookupFailure": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
+      "ControlDNSDomain": "wrong.host.badssl.com",
+      "ControlDNSLookupFailure": "",
       "ControlTCPConnectFailure": "",
       "MatchWithControlIPAddress": true,
       "MatchWithControlIPAddressASN": true,

diff --git a/...data/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations.json b/...data/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations.json
@@ -25,6 +25,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -64,6 +66,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -101,6 +105,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": null,
       "HTTPResponseIsFinal": null,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,
@@ -145,6 +151,8 @@
       "HTTPResponseLocation": null,
       "HTTPResponseTitle": "Default Web Page",
       "HTTPResponseIsFinal": true,
+      "TagDepth": null,
+      "TagFetchBody": null,
       "ControlDNSDomain": null,
       "ControlDNSLookupFailure": null,
       "ControlTCPConnectFailure": null,