[WIP]Inner shuffle hash

.gitmodules

@@ -253,9 +253,6 @@
 [submodule "contrib/qpl"]
 	path = contrib/qpl
 	url = https://github.com/intel/qpl
-[submodule "contrib/idxd-config"]
-	path = contrib/idxd-config
-	url = https://github.com/intel/idxd-config
 [submodule "contrib/wyhash"]
 	path = contrib/wyhash
 	url = https://github.com/wangyi-fudan/wyhash
@@ -296,6 +293,9 @@
 [submodule "contrib/libdivide"]
 	path = contrib/libdivide
 	url = https://github.com/ridiculousfish/libdivide
+[submodule "contrib/libbcrypt"]
+	path = contrib/libbcrypt
+	url = https://github.com/rg3/libbcrypt.git
 [submodule "contrib/ulid-c"]
 	path = contrib/ulid-c
 	url = https://github.com/ClickHouse/ulid-c.git

CHANGELOG.md

@@ -7,7 +7,7 @@
 
 # 2023 Changelog
 
-### <a id="234"></a> ClickHouse release 23.4 LTS, 2023-04-26
+### <a id="234"></a> ClickHouse release 23.4, 2023-04-26
 
 #### Backward Incompatible Change
 * Formatter '%M' in function formatDateTime() now prints the month name instead of the minutes. This makes the behavior consistent with MySQL. The previous behavior can be restored using setting "formatdatetime_parsedatetime_m_is_month_name = 0". [#47246](https://github.com/ClickHouse/ClickHouse/pull/47246) ([Robert Schulze](https://github.com/rschu1ze)).

CMakeLists.txt

@@ -170,12 +170,6 @@ else ()
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
-option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
-if (OS_DARWIN)
-    # Disable the curl, azure, senry build on MacOS
-    set (ENABLE_CURL_BUILD OFF)
-endif ()
-
 if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
     # Can be lld or ld-lld or lld-13 or /path/to/lld.
     if (LINKER_NAME MATCHES "lld")
@@ -393,9 +387,9 @@ else()
 endif ()
 
 option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON)
-# We use mmap for allocations more heavily in debug builds, 
-# but GWP-ASan also wants to use mmap frequently, 
-# and due to a large number of memory mappings, 
+# We use mmap for allocations more heavily in debug builds,
+# but GWP-ASan also wants to use mmap frequently,
+# and due to a large number of memory mappings,
 # it does not work together well.
 if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
     set(ENABLE_GWP_ASAN OFF)

SECURITY.md

@@ -22,13 +22,7 @@ The following versions of ClickHouse server are currently being supported with s
 | 22.10 | ❌ |
 | 22.9 | ❌ |
 | 22.8 | ✔️ |
-| 22.7 | ❌ |
-| 22.6 | ❌ |
-| 22.5 | ❌ |
-| 22.4 | ❌ |
-| 22.3 | ❌ |
-| 22.2 | ❌ |
-| 22.1 | ❌ |
+| 22.* | ❌ |
 | 21.* | ❌ |
 | 20.* | ❌ |
 | 19.* | ❌ |

base/harmful/harmful.c

@@ -31,7 +31,8 @@ TRAP(argp_state_help)
 TRAP(argp_usage)
 TRAP(asctime)
 TRAP(clearenv)
-TRAP(crypt)
+// Redefined at contrib/libbcrypt/crypt_blowfish/wrapper.c:186
+// TRAP(crypt)
 TRAP(ctime)
 TRAP(cuserid)
 TRAP(drand48)

cmake/fuzzer.cmake

@@ -7,10 +7,6 @@ if (FUZZER)
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link")
 
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=fuzzer-no-link")
-        endif()
-
         # NOTE: oss-fuzz can change LIB_FUZZING_ENGINE variable
         if (NOT LIB_FUZZING_ENGINE)
             set (LIB_FUZZING_ENGINE "-fsanitize=fuzzer")

cmake/sanitize.cmake

@@ -16,49 +16,24 @@ if (SANITIZE)
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}")
 
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_FLAGS}")
-        endif()
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan")
-        endif ()
-
     elseif (SANITIZE STREQUAL "memory")
         # MemorySanitizer flags are set according to the official documentation:
         # https://clang.llvm.org/docs/MemorySanitizer.html#usage
-        #
-        # For now, it compiles with `cmake -DSANITIZE=memory -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_FLAGS_ADD="-O1" -DCMAKE_C_FLAGS_ADD="-O1"`
-        # Compiling with -DCMAKE_BUILD_TYPE=Debug leads to ld.lld failures because
-        # of large files (was not tested with ld.gold). This is why we compile with
-        # RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to
-        # keep the binary size down.
-        # TODO: try compiling with -Og and with ld.gold.
+
+        # Linking can fail due to relocation overflows (see #49145), caused by too big object files / libraries.
+        # Work around this with position-independent builds (-fPIC and -fpie), this is slightly slower than non-PIC/PIE but that's okay.
         set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt")
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
 
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory")
-        endif()
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan")
-        endif ()
-
     elseif (SANITIZE STREQUAL "thread")
         set (TSAN_FLAGS "-fsanitize=thread")
         if (COMPILER_CLANG)
             set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt")
         endif()
 
-
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
-        endif()
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan")
-        endif ()
 
     elseif (SANITIZE STREQUAL "undefined")
         set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
@@ -77,12 +52,6 @@ if (SANITIZE)
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
-        endif()
-        if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan")
-        endif ()
 
         # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan.
         set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "")

contrib/CMakeLists.txt

@@ -141,20 +141,19 @@ add_contrib (libuv-cmake libuv)
 add_contrib (liburing-cmake liburing)
 add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
 add_contrib (cassandra-cmake cassandra) # requires: libuv
-
-if (ENABLE_CURL_BUILD)
+if (NOT OS_DARWIN)
     add_contrib (curl-cmake curl)
     add_contrib (azure-cmake azure)
     add_contrib (sentry-native-cmake sentry-native) # requires: curl
 endif()
-
 add_contrib (fmtlib-cmake fmtlib)
 add_contrib (krb5-cmake krb5)
 add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5
 add_contrib (libgsasl-cmake libgsasl) # requires krb5
 add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
 add_contrib (nats-io-cmake nats-io)
-add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5
+add_contrib (isa-l-cmake isa-l)
+add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5, isa-l
 add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3
 add_contrib (cppkafka-cmake cppkafka)
 add_contrib (libpqxx-cmake libpqxx)
@@ -178,21 +177,17 @@ add_contrib (s2geometry-cmake s2geometry)
 add_contrib (c-ares-cmake c-ares)
 add_contrib (qpl-cmake qpl)
 add_contrib (morton-nd-cmake morton-nd)
-
 if (ARCH_S390X)
     add_contrib(crc32-s390x-cmake crc32-s390x)
 endif()
-
 add_contrib (annoy-cmake annoy)
-
 add_contrib (xxHash-cmake xxHash)
 
-add_contrib (google-benchmark-cmake google-benchmark)
+add_contrib (libbcrypt-cmake libbcrypt)
 
+add_contrib (google-benchmark-cmake google-benchmark)
 add_contrib (ulid-c-cmake ulid-c)
 
-add_contrib (isa-l-cmake isa-l)
-
 # Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
 # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
 # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,

contrib/curl-cmake/CMakeLists.txt

@@ -12,6 +12,9 @@ set (SRCS
     "${LIBRARY_DIR}/lib/noproxy.c"
     "${LIBRARY_DIR}/lib/idn.c"
     "${LIBRARY_DIR}/lib/cfilters.c"
+    "${LIBRARY_DIR}/lib/cf-socket.c"
+    "${LIBRARY_DIR}/lib/cf-haproxy.c"
+    "${LIBRARY_DIR}/lib/cf-https-connect.c"
     "${LIBRARY_DIR}/lib/file.c"
     "${LIBRARY_DIR}/lib/timeval.c"
     "${LIBRARY_DIR}/lib/base64.c"
@@ -37,8 +40,8 @@ set (SRCS
     "${LIBRARY_DIR}/lib/strcase.c"
     "${LIBRARY_DIR}/lib/easy.c"
     "${LIBRARY_DIR}/lib/curl_fnmatch.c"
+    "${LIBRARY_DIR}/lib/curl_log.c"
     "${LIBRARY_DIR}/lib/fileinfo.c"
-    "${LIBRARY_DIR}/lib/wildcard.c"
     "${LIBRARY_DIR}/lib/krb5.c"
     "${LIBRARY_DIR}/lib/memdebug.c"
     "${LIBRARY_DIR}/lib/http_chunks.c"
@@ -96,6 +99,7 @@ set (SRCS
     "${LIBRARY_DIR}/lib/rand.c"
     "${LIBRARY_DIR}/lib/curl_multibyte.c"
     "${LIBRARY_DIR}/lib/conncache.c"
+    "${LIBRARY_DIR}/lib/cf-h1-proxy.c"
     "${LIBRARY_DIR}/lib/http2.c"
     "${LIBRARY_DIR}/lib/smb.c"
     "${LIBRARY_DIR}/lib/curl_endian.c"
@@ -113,12 +117,13 @@ set (SRCS
     "${LIBRARY_DIR}/lib/altsvc.c"
     "${LIBRARY_DIR}/lib/socketpair.c"
     "${LIBRARY_DIR}/lib/bufref.c"
+    "${LIBRARY_DIR}/lib/bufq.c"
     "${LIBRARY_DIR}/lib/dynbuf.c"
+    "${LIBRARY_DIR}/lib/dynhds.c"
     "${LIBRARY_DIR}/lib/hsts.c"
     "${LIBRARY_DIR}/lib/http_aws_sigv4.c"
     "${LIBRARY_DIR}/lib/mqtt.c"
     "${LIBRARY_DIR}/lib/rename.c"
-    "${LIBRARY_DIR}/lib/h2h3.c"
     "${LIBRARY_DIR}/lib/headers.c"
     "${LIBRARY_DIR}/lib/timediff.c"
     "${LIBRARY_DIR}/lib/vauth/vauth.c"
@@ -133,6 +138,7 @@ set (SRCS
     "${LIBRARY_DIR}/lib/vauth/oauth2.c"
     "${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c"
     "${LIBRARY_DIR}/lib/vauth/spnego_sspi.c"
+    "${LIBRARY_DIR}/lib/vquic/vquic.c"
     "${LIBRARY_DIR}/lib/vtls/openssl.c"
     "${LIBRARY_DIR}/lib/vtls/gtls.c"
     "${LIBRARY_DIR}/lib/vtls/vtls.c"
@@ -147,9 +153,6 @@ set (SRCS
     "${LIBRARY_DIR}/lib/vtls/keylog.c"
     "${LIBRARY_DIR}/lib/vtls/x509asn1.c"
     "${LIBRARY_DIR}/lib/vtls/hostcheck.c"
-    "${LIBRARY_DIR}/lib/vquic/ngtcp2.c"
-    "${LIBRARY_DIR}/lib/vquic/quiche.c"
-    "${LIBRARY_DIR}/lib/vquic/msh3.c"
     "${LIBRARY_DIR}/lib/vssh/libssh2.c"
     "${LIBRARY_DIR}/lib/vssh/libssh.c"
 )

contrib/isa-l-cmake/CMakeLists.txt

@@ -1,6 +1,23 @@
+option(ENABLE_ISAL_LIBRARY "Enable ISA-L library" ${ENABLE_LIBRARIES})
+if (ARCH_AARCH64)
+    # Disable ISA-L libray on aarch64.
+    set (ENABLE_ISAL_LIBRARY OFF)
+endif ()
+
+if (NOT ENABLE_ISAL_LIBRARY)
+    message(STATUS "Not using isa-l")
+    return()
+endif()
+
 set(ISAL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/isa-l")
 
-# check nasm compiler
+# The YASM and NASM assembers are somewhat mutually compatible. ISAL specifically needs NASM. If only YASM is installed, then check_language(ASM_NASM)
+# below happily finds YASM, leading to weird errors at build time. Therefore, do an explicit check for NASM here.
+find_program(NASM_PATH NAMES nasm)
+if (NOT NASM_PATH)
+    message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!")
+endif ()
+
 include(CheckLanguage)
 check_language(ASM_NASM)
 if(NOT CMAKE_ASM_NASM_COMPILER)

contrib/libbcrypt-cmake/CMakeLists.txt

@@ -0,0 +1,19 @@
+option(ENABLE_BCRYPT "Enable bcrypt" ${ENABLE_LIBRARIES})
+
+if (NOT ENABLE_BCRYPT)
+    message(STATUS "Not using bcrypt")
+    return()
+endif()
+
+set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libbcrypt")
+
+set(SRCS 
+    "${LIBRARY_DIR}/bcrypt.c"
+    "${LIBRARY_DIR}/crypt_blowfish/crypt_blowfish.c"
+    "${LIBRARY_DIR}/crypt_blowfish/crypt_gensalt.c"
+    "${LIBRARY_DIR}/crypt_blowfish/wrapper.c"
+)
+
+add_library(_bcrypt ${SRCS})
+target_include_directories(_bcrypt SYSTEM PUBLIC "${LIBRARY_DIR}")
+add_library(ch_contrib::bcrypt ALIAS _bcrypt)

contrib/libhdfs3-cmake/CMakeLists.txt

@@ -172,8 +172,10 @@ if (TARGET OpenSSL::SSL)
     target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL)
 endif()
 
-target_link_libraries(_hdfs3 PRIVATE ch_contrib::isal)
-add_definitions(-DHADOOP_ISAL_LIBRARY)
+if (TARGET ch_contrib::isal)
+    target_link_libraries(_hdfs3 PRIVATE ch_contrib::isal)
+    add_definitions(-DHADOOP_ISAL_LIBRARY)
+endif()
 
 add_library(ch_contrib::hdfs ALIAS _hdfs3)
 

docker/images.json

@@ -123,7 +123,8 @@
             "docker/test/stateless",
             "docker/test/integration/base",
             "docker/test/fuzzer",
-            "docker/test/keeper-jepsen"
+            "docker/test/keeper-jepsen",
+            "docker/test/server-jepsen"
          ]
     },
     "docker/test/integration/kerberized_hadoop": {
@@ -139,6 +140,10 @@
         "name": "clickhouse/keeper-jepsen-test",
         "dependent": []
     },
+    "docker/test/server-jepsen": {
+        "name": "clickhouse/server-jepsen-test",
+        "dependent": []
+    },
     "docker/test/install/deb": {
         "name": "clickhouse/install-deb-test",
         "dependent": []

docker/keeper/Dockerfile

@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
     esac
 
 ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release"
-ARG VERSION="23.4.1.1943"
+ARG VERSION="23.4.2.11"
 ARG PACKAGES="clickhouse-keeper"
 
 # user/group precreated explicitly with fixed uid/gid on purpose.

docker/server/Dockerfile.alpine

@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="23.4.1.1943"
+ARG VERSION="23.4.2.11"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # user/group precreated explicitly with fixed uid/gid on purpose.

docker/server/Dockerfile.ubuntu

@@ -22,7 +22,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="23.4.1.1943"
+ARG VERSION="23.4.2.11"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # set non-empty deb_location_url url to create a docker image

docker/test/server-jepsen/Dockerfile

@@ -16,6 +16,8 @@ ENV TESTS_TO_RUN="8"
 ENV TIME_LIMIT="30"
 
 ENV KEEPER_NODE=""
+ENV NEMESIS=""
+ENV WORKLOAD=""
 
 
 # volumes