Merge branch 'master' into issue/2634

ooni · Nov 28, 2023 · 1488d05 · 1488d05
2 parents 383eb69 + 18c38ee
commit 1488d05
Show file tree

Hide file tree

Showing 38 changed files with 69 additions and 374 deletions.
diff --git a/internal/cmd/minipipeline/testdata/analysis.json b/internal/cmd/minipipeline/testdata/analysis.json
@@ -20,4 +20,4 @@
   "TCPTransactionsWithUnexpectedTLSHandshakeFailures": {},
   "TCPTransactionsWithUnexpectedHTTPFailures": {},
   "TCPTransactionsWithUnexplainedUnexpectedFailures": {}
-}
+}
diff --git a/internal/cmd/minipipeline/testdata/observations.json b/internal/cmd/minipipeline/testdata/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -46,7 +45,6 @@
       "DNSEngine": "doh",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "udp",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
-      "IPAddressOrg": "Consortium GARR",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -124,7 +121,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
-      "IPAddressOrg": "Consortium GARR",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -162,7 +158,6 @@
       "DNSEngine": "doh",
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
-      "IPAddressOrg": "Consortium GARR",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -202,7 +197,6 @@
       "DNSEngine": null,
       "IPAddress": "130.192.16.171",
       "IPAddressASN": 137,
-      "IPAddressOrg": "Consortium GARR",
       "IPAddressBogon": false,
       "EndpointTransactionID": 4,
       "EndpointProto": "tcp",
@@ -263,4 +257,4 @@
       "ControlHTTPResponseTitle": "Nexa Center for Internet \u0026 Society | Il centro Nexa è un centro di ricerca del Dipartimento di Automatica e Informatica del Politecnico di Torino"
     }
   }
-}
+}
diff --git a/internal/minipipeline/observation.go b/internal/minipipeline/observation.go
@@ -371,7 +371,7 @@ func (c *WebObservationsContainer) IngestHTTPRoundTripEvents(evs ...*model.Archi
 		obs.HTTPFailure = optional.Some(utilsStringPointerToString(ev.Failure))
 
 		// consider the response authoritative only in case of success
-		if ev.Failure != nil {
+		if ev.Failure == nil {
 			obs.HTTPResponseStatusCode = optional.Some(ev.Response.Code)
 			obs.HTTPResponseBodyLength = optional.Some(int64(len(ev.Response.Body)))
 			obs.HTTPResponseBodyIsTruncated = optional.Some(ev.Request.BodyIsTruncated)
@@ -513,30 +513,18 @@ func (c *WebObservationsContainer) controlXrefTLSFailures(resp *model.THResponse
 }
 
 func (c *WebObservationsContainer) controlSetHTTPFinalResponseExpectation(resp *model.THResponse) {
-	// Implementation note: the TH response does not have a clear semantics for "missing" values
-	// therefore we are accepting as valid values within the correct range
-	//
-	// also note that we add control information to all endpoints and then we check for "final"
-	// responses and only compare against "final" responses during the analysis
 	for _, obs := range c.KnownTCPEndpoints {
 		obs.ControlHTTPFailure = optional.Some(utilsStringPointerToString(resp.HTTPRequest.Failure))
-		if value := resp.HTTPRequest.StatusCode; value > 0 {
-			obs.ControlHTTPResponseStatusCode = optional.Some(value)
-		}
-		if value := resp.HTTPRequest.BodyLength; value >= 0 {
-			obs.ControlHTTPResponseBodyLength = optional.Some(value)
-		}
 
-		controlHTTPResponseHeadersKeys := make(map[string]bool)
-		for key := range resp.HTTPRequest.Headers {
-			controlHTTPResponseHeadersKeys[key] = true
-		}
-		if len(controlHTTPResponseHeadersKeys) > 0 {
-			obs.ControlHTTPResponseHeadersKeys = optional.Some(controlHTTPResponseHeadersKeys)
+		// leave everything else nil if there was a failure, like we
+		// already do when processing the probe events
+		if resp.HTTPRequest.Failure != nil {
+			continue
 		}
 
-		if v := resp.HTTPRequest.Title; v != "" {
-			obs.ControlHTTPResponseTitle = optional.Some(v)
-		}
+		obs.ControlHTTPResponseStatusCode = optional.Some(resp.HTTPRequest.StatusCode)
+		obs.ControlHTTPResponseBodyLength = optional.Some(resp.HTTPRequest.BodyLength)
+		obs.ControlHTTPResponseHeadersKeys = utilsExtractHTTPHeaderKeys(resp.HTTPRequest.Headers)
+		obs.ControlHTTPResponseTitle = optional.Some(resp.HTTPRequest.Title)
 	}
 }
diff --git a/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json b/...ipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -48,7 +47,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "udp",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -126,7 +123,6 @@
       "DNSEngine": null,
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": 3,
       "EndpointProto": "tcp",

diff --git a/...a/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json b/...a/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -46,7 +45,6 @@
       "DNSEngine": "doh",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -124,7 +121,6 @@
       "DNSEngine": "udp",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -162,7 +158,6 @@
       "DNSEngine": "doh",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -202,7 +197,6 @@
       "DNSEngine": null,
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": 4,
       "EndpointProto": "tcp",

diff --git a/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json b/...webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -48,7 +47,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "udp",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -126,7 +123,6 @@
       "DNSEngine": null,
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": 3,
       "EndpointProto": "tcp",
@@ -169,7 +165,6 @@
       "DNSEngine": null,
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": 4,
       "EndpointProto": "tcp",

diff --git a/...nipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations.json b/...nipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -48,7 +47,6 @@
       "DNSEngine": "udp",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -126,7 +123,6 @@
       "DNSEngine": null,
       "IPAddress": "104.154.89.105",
       "IPAddressASN": 396982,
-      "IPAddressOrg": "Google LLC",
       "IPAddressBogon": false,
       "EndpointTransactionID": 3,
       "EndpointProto": "tcp",

diff --git a/...data/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations.json b/...data/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -48,7 +47,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "udp",
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -126,7 +123,6 @@
       "DNSEngine": null,
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": 3,
       "EndpointProto": "tcp",

diff --git a/...tdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/observations.json b/...tdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPWebsite/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -48,7 +47,6 @@
       "DNSEngine": "udp",
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -86,7 +84,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -126,7 +123,6 @@
       "DNSEngine": null,
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": 3,
       "EndpointProto": "tcp",
@@ -169,7 +165,6 @@
       "DNSEngine": null,
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": 4,
       "EndpointProto": "tcp",

diff --git a/...ine/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/observations.json b/...ine/testdata/webconnectivity/generated/dnsBlockingAndroidDNSCacheNoData/observations.json
@@ -8,7 +8,6 @@
       "DNSEngine": "udp",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -46,7 +45,6 @@
       "DNSEngine": "getaddrinfo",
       "IPAddress": null,
       "IPAddressASN": null,
-      "IPAddressOrg": null,
       "IPAddressBogon": null,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -124,7 +122,6 @@
       "DNSEngine": "udp",
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": null,
       "EndpointProto": null,
@@ -164,7 +161,6 @@
       "DNSEngine": null,
       "IPAddress": "93.184.216.34",
       "IPAddressASN": 15133,
-      "IPAddressOrg": "Edgecast Inc.",
       "IPAddressBogon": false,
       "EndpointTransactionID": 3,
       "EndpointProto": "tcp",