From 65565816e49c1cbd20e0b24ca34addb8d44dbdfa Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 13 Dec 2023 11:12:13 +0100 Subject: [PATCH 1/6] chore: update C deps Part of https://github.com/ooni/probe/issues/2556 --- internal/cmd/buildtool/android_test.go | 32 +++++++++++------------ internal/cmd/buildtool/cdepsopenssl.go | 10 +++---- internal/cmd/buildtool/cdepstor.go | 10 +++---- internal/cmd/buildtool/ios_test.go | 24 ++++++++--------- internal/cmd/buildtool/linuxcdeps_test.go | 8 +++--- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/internal/cmd/buildtool/android_test.go b/internal/cmd/buildtool/android_test.go index 30169927bf..e8192eeb62 100644 --- a/internal/cmd/buildtool/android_test.go +++ b/internal/cmd/buildtool/android_test.go @@ -702,12 +702,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -757,12 +757,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -812,12 +812,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -867,12 +867,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -1738,12 +1738,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, @@ -1815,12 +1815,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, @@ -1892,12 +1892,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, @@ -1969,12 +1969,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, diff --git a/internal/cmd/buildtool/cdepsopenssl.go b/internal/cmd/buildtool/cdepsopenssl.go index fba0eb6535..d873746c7a 100644 --- a/internal/cmd/buildtool/cdepsopenssl.go +++ b/internal/cmd/buildtool/cdepsopenssl.go @@ -27,13 +27,13 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie defer restore() // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/o/openssl@3.rb - cdepsMustFetch("https://www.openssl.org/source/openssl-3.1.3.tar.gz") + cdepsMustFetch("https://www.openssl.org/source/openssl-3.2.0.tar.gz") deps.VerifySHA256( // must be mockable - "f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6", - "openssl-3.1.3.tar.gz", + "14c826f07c7e433706fb5c69fa9e25dab95684844b4c962a2cf1bf183eb4690e", + "openssl-3.2.0.tar.gz", ) - must.Run(log.Log, "tar", "-xf", "openssl-3.1.3.tar.gz") - _ = deps.MustChdir("openssl-3.1.3") // must be mockable + must.Run(log.Log, "tar", "-xf", "openssl-3.2.0.tar.gz") + _ = deps.MustChdir("openssl-3.2.0") // must be mockable mydir := filepath.Join(topdir, "CDEPS", "openssl") for _, patch := range cdepsMustListPatches(mydir) { diff --git a/internal/cmd/buildtool/cdepstor.go b/internal/cmd/buildtool/cdepstor.go index 2c47aa7a22..6a181fb6dd 100644 --- a/internal/cmd/buildtool/cdepstor.go +++ b/internal/cmd/buildtool/cdepstor.go @@ -27,13 +27,13 @@ func cdepsTorBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) { defer restore() // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/t/tor.rb - cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.7.tar.gz") + cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.10.tar.gz") deps.VerifySHA256( // must be mockable - "b20d2b9c74db28a00c07f090ee5b0241b2b684f3afdecccc6b8008931c557491", - "tor-0.4.8.7.tar.gz", + "e628b4fab70edb4727715b23cf2931375a9f7685ac08f2c59ea498a178463a86", + "tor-0.4.8.10.tar.gz", ) - must.Run(log.Log, "tar", "-xf", "tor-0.4.8.7.tar.gz") - _ = deps.MustChdir("tor-0.4.8.7") // must be mockable + must.Run(log.Log, "tar", "-xf", "tor-0.4.8.10.tar.gz") + _ = deps.MustChdir("tor-0.4.8.10") // must be mockable mydir := filepath.Join(topdir, "CDEPS", "tor") for _, patch := range cdepsMustListPatches(mydir) { diff --git a/internal/cmd/buildtool/ios_test.go b/internal/cmd/buildtool/ios_test.go index 13cc46bc88..82d928139e 100644 --- a/internal/cmd/buildtool/ios_test.go +++ b/internal/cmd/buildtool/ios_test.go @@ -349,12 +349,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -399,12 +399,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -449,12 +449,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -1142,12 +1142,12 @@ func TestIOSBuildCdepsTor(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, @@ -1220,12 +1220,12 @@ func TestIOSBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, @@ -1298,12 +1298,12 @@ func TestIOSBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, diff --git a/internal/cmd/buildtool/linuxcdeps_test.go b/internal/cmd/buildtool/linuxcdeps_test.go index 40d4b5817e..6ae5aca748 100644 --- a/internal/cmd/buildtool/linuxcdeps_test.go +++ b/internal/cmd/buildtool/linuxcdeps_test.go @@ -92,12 +92,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.2.0.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.3.tar.gz", + "tar", "-xf", "openssl-3.2.0.tar.gz", }, }, { Env: []string{}, @@ -322,12 +322,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.7.tar.gz", + "tar", "-xf", "tor-0.4.8.10.tar.gz", }, }, { Env: []string{}, From 887accf681bc61f85d0b90e2728e919463a0674e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 13 Dec 2023 11:21:07 +0100 Subject: [PATCH 2/6] fix: update patches for OpenSSL --- CDEPS/openssl/000.patch | 2 +- CDEPS/openssl/001.patch | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CDEPS/openssl/000.patch b/CDEPS/openssl/000.patch index e313ac356a..6273aeece6 100644 --- a/CDEPS/openssl/000.patch +++ b/CDEPS/openssl/000.patch @@ -2,7 +2,7 @@ diff --git a/crypto/comp/c_zlib.c b/crypto/comp/c_zlib.c index b819337..e479bd6 100644 --- a/crypto/comp/c_zlib.c +++ b/crypto/comp/c_zlib.c -@@ -34,6 +34,10 @@ static COMP_METHOD zlib_method_nozlib = { +@@ -34,6 +34,10 @@ COMP_METHOD *COMP_zlib(void); # include diff --git a/CDEPS/openssl/001.patch b/CDEPS/openssl/001.patch index 844752e292..80c426b2de 100644 --- a/CDEPS/openssl/001.patch +++ b/CDEPS/openssl/001.patch @@ -1,15 +1,15 @@ diff --git a/include/openssl/opensslv.h.in b/include/openssl/opensslv.h.in -index 3f47a2a..af1db51 100644 +index 3f47a2ac08..b413461ca7 100644 --- a/include/openssl/opensslv.h.in +++ b/include/openssl/opensslv.h.in -@@ -101,6 +101,10 @@ extern "C" { - |(OPENSSL_VERSION_PATCH<<4) \ - |_OPENSSL_VERSION_PRE_RELEASE ) +@@ -13,6 +13,10 @@ + # define OPENSSL_OPENSSLV_H + # pragma once +/* OPENSSL_OONI is used by dependencies to ensure they are using the + correct OpenSSL headers and not some other headers. */ +#define OPENSSL_OONI 1 + # ifdef __cplusplus - } + extern "C" { # endif From d0b8a16f4dc472ad465a8c9b87bbe32f176322fa Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 13 Dec 2023 11:23:30 +0100 Subject: [PATCH 3/6] x --- CDEPS/openssl/000.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CDEPS/openssl/000.patch b/CDEPS/openssl/000.patch index 6273aeece6..bdcb40cd5c 100644 --- a/CDEPS/openssl/000.patch +++ b/CDEPS/openssl/000.patch @@ -1,15 +1,15 @@ diff --git a/crypto/comp/c_zlib.c b/crypto/comp/c_zlib.c -index b819337..e479bd6 100644 +index 0fbab8f014..0dc8ff53d4 100644 --- a/crypto/comp/c_zlib.c +++ b/crypto/comp/c_zlib.c -@@ -34,6 +34,10 @@ COMP_METHOD *COMP_zlib(void); - +@@ -26,6 +26,10 @@ COMP_METHOD *COMP_zlib(void); + # include - + +#ifndef ZLIB_OONI +# error "We're not including the correct zlib.h file" +#endif + static int zlib_stateful_init(COMP_CTX *ctx); static void zlib_stateful_finish(COMP_CTX *ctx); - static int zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, + static ossl_ssize_t zlib_stateful_compress_block(COMP_CTX *ctx, unsigned char *out, From 9c0e92e396019fec18e8313340d9f97b934a23c8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 13 Dec 2023 14:28:02 +0100 Subject: [PATCH 4/6] x --- MOBILE/ios/check-xcode-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MOBILE/ios/check-xcode-version b/MOBILE/ios/check-xcode-version index e4f7b9a94d..76f62a4f08 100755 --- a/MOBILE/ios/check-xcode-version +++ b/MOBILE/ios/check-xcode-version @@ -2,7 +2,7 @@ set -euo pipefail -EXPECTED_XCODE_VERSION=${EXPECTED_XCODE_VERSION:-15.0} +EXPECTED_XCODE_VERSION=${EXPECTED_XCODE_VERSION:-15.1} printf "checking for xcodebuild... " command -v xcodebuild || { From c52bc26430df0e74b7e839ac438f760b8d96f3c2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 13 Dec 2023 17:50:57 +0000 Subject: [PATCH 5/6] x --- internal/cmd/buildtool/linuxstatic.go | 1 + internal/cmd/buildtool/linuxstatic_test.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/internal/cmd/buildtool/linuxstatic.go b/internal/cmd/buildtool/linuxstatic.go index dfe8d00aeb..2cac3b8f3e 100644 --- a/internal/cmd/buildtool/linuxstatic.go +++ b/internal/cmd/buildtool/linuxstatic.go @@ -85,6 +85,7 @@ func linuxStaticBuildPackage( envp.Append("CGO_ENABLED", "1") envp.Append("GOOS", "linux") envp.Append("GOARCH", goarch) + envp.Append("CGO_CFLAGS", "-D_LARGEFILE64_SOURCE") if goarm > 0 { envp.Append("GOARM", strconv.FormatInt(goarm, 10)) } diff --git a/internal/cmd/buildtool/linuxstatic_test.go b/internal/cmd/buildtool/linuxstatic_test.go index 314d251c34..58bfbc2d08 100644 --- a/internal/cmd/buildtool/linuxstatic_test.go +++ b/internal/cmd/buildtool/linuxstatic_test.go @@ -49,6 +49,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "CGO_ENABLED=1", "GOARCH=arm64", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-tags", "ooni_psiphon_config", @@ -62,6 +63,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "CGO_ENABLED=1", "GOARCH=arm64", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-tags", "ooni_psiphon_config", @@ -84,6 +86,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "CGO_ENABLED=1", "GOARCH=amd64", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-ldflags", "-s -w -extldflags -static", @@ -96,6 +99,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "CGO_ENABLED=1", "GOARCH=amd64", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-ldflags", "-s -w -extldflags -static", @@ -118,6 +122,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "GOARCH=arm", "GOARM=7", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-tags", "ooni_psiphon_config", @@ -132,6 +137,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "GOARCH=arm", "GOARM=7", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-tags", "ooni_psiphon_config", @@ -155,6 +161,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "GOARCH=arm", "GOARM=6", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-ldflags", "-s -w -extldflags -static", @@ -168,6 +175,7 @@ func TestLinuxStaticBuildAll(t *testing.T) { "GOARCH=arm", "GOARM=6", "GOOS=linux", + "CGO_CFLAGS=-D_LARGEFILE64_SOURCE", }, Argv: []string{ "go", "build", "-ldflags", "-s -w -extldflags -static", From a7ce050430805a720f2e70f54bb0ad84fffb8cdd Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 13 Dec 2023 17:52:39 +0000 Subject: [PATCH 6/6] x --- internal/cmd/buildtool/linuxstatic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/buildtool/linuxstatic.go b/internal/cmd/buildtool/linuxstatic.go index 2cac3b8f3e..d04cfe99f1 100644 --- a/internal/cmd/buildtool/linuxstatic.go +++ b/internal/cmd/buildtool/linuxstatic.go @@ -85,7 +85,7 @@ func linuxStaticBuildPackage( envp.Append("CGO_ENABLED", "1") envp.Append("GOOS", "linux") envp.Append("GOARCH", goarch) - envp.Append("CGO_CFLAGS", "-D_LARGEFILE64_SOURCE") + envp.Append("CGO_CFLAGS", "-D_LARGEFILE64_SOURCE") // See https://github.com/ooni/probe-cli/pull/1434 if goarm > 0 { envp.Append("GOARM", strconv.FormatInt(goarm, 10)) }