Possible release v0.21.2-0.5.0 - showcase new vendor script

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.21.2-0.5.0
+
+* Vendor Bitcoin Core subtree to v0.21.2
+
 ## v0.20.2-0.5.0
 
 * Upgrade Bitcoin Core subtree to v0.20.2

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "bitcoinconsensus"
 # The first part is the Bitcoin Core version, the second part is the version of this lib.
-version = "0.20.2-0.5.0"
+version = "0.21.2-0.5.0"
 authors = ["Tamas Blummer <[email protected]>"]
 license = "Apache-2.0"
 homepage = "https://github.com/rust-bitcoin/rust-bitcoinconsensus/"

README.md

@@ -26,18 +26,9 @@ For example, if we upgrade the Bitcoin Core code by a `Patch` version we also bu
 One side effect of this is that `crates.io` shows our release versions in yellow as if they were pre-release versions, this is due to us using a `-` which, in semantic versioning, implies a pre-release version.
 
 
-## Bitcoin Core subtree
-
-We use a git subtree to vendor the Bitcoin Core code. This can be seen from the following commits that were created using `git subtree add --prefix='depend/bitcoin' [email protected]:bitcoin/bitcoin.git v0.19.2 --squash`.
-```
-f751613e62 Squashed 'depend/bitcoin/' content from commit 204cc0f575
-264282188a Merge commit 'f751613e6203770fa94143b9aba1d116512f0ce7' as 'depend/bitcoin'
-```
-
-To use a later version of Bitcoin Core, for example, v0.20.2
-```
-git subtree pull --prefix='depend/bitcoin' [email protected]:bitcoin/bitcoin.git v0.20.2 --squash
-```
+## Vendor Bitcoin Core
+
+We use a script to vendor the Bitcoin Core code, `./vendor-bitcoin-core.sh v0.21.2`
 
 
 ## MSRV

build.rs

@@ -21,15 +21,21 @@ fn main() {
     let is_big_endian = env::var("CARGO_CFG_TARGET_ENDIAN").expect("No endian is set") == "big";
     let mut base_config = cc::Build::new();
     base_config
-        .cpp(true)
-        .include("depend/bitcoin/src")
         .include("depend/bitcoin/src/secp256k1/include")
         .define("__STDC_FORMAT_MACROS", None);
 
+    if target.contains("windows") {
+        base_config.define("WIN32", "1");
+    }
+
+    let mut secp_config = base_config.clone();
+    let mut consensus_config = base_config;
+
     // **Secp256k1**
     if !cfg!(feature = "external-secp") {
-        base_config
+        secp_config
             .include("depend/bitcoin/src/secp256k1")
+            .include("depend/bitcoin/src/secp256k1/src")
             .flag_if_supported("-Wno-unused-function") // some ecmult stuff is defined but not used upstream
             .define("SECP256K1_BUILD", "1")
             // Bitcoin core defines libsecp to *not* use libgmp.
@@ -38,34 +44,40 @@ fn main() {
             .define("USE_SCALAR_INV_BUILTIN", "1")
             // Technically libconsensus doesn't require the recovery feautre, but `pubkey.cpp` does.
             .define("ENABLE_MODULE_RECOVERY", "1")
+            .define("ECMULT_WINDOW_SIZE", "15")
+            .define("ECMULT_GEN_PREC_BITS", "4")
+            .define("ENABLE_MODULE_SCHNORRSIG", "1")
+            .define("ENABLE_MODULE_EXTRAKEYS", "1")
             // The actual libsecp256k1 C code.
             .file("depend/bitcoin/src/secp256k1/src/secp256k1.c");
 
         if is_big_endian {
-            base_config.define("WORDS_BIGENDIAN", "1");
+            secp_config.define("WORDS_BIGENDIAN", "1");
         }
 
         if use_64bit_compilation {
-            base_config
+            secp_config
                 .define("USE_FIELD_5X52", "1")
                 .define("USE_SCALAR_4X64", "1")
                 .define("HAVE___INT128", "1");
         } else {
-            base_config.define("USE_FIELD_10X26", "1").define("USE_SCALAR_8X32", "1");
+            secp_config.define("USE_FIELD_10X26", "1").define("USE_SCALAR_8X32", "1");
         }
+
+        secp_config.compile("libsecp256k1.a");
     }
 
-    let tool = base_config.get_compiler();
+    let tool = consensus_config.get_compiler();
     if tool.is_like_msvc() {
-        base_config.flag("/std:c++14").flag("/wd4100");
+        consensus_config.flag("/std:c++17").flag("/wd4100");
     } else if tool.is_like_clang() || tool.is_like_gnu() {
-        base_config.flag("-std=c++11").flag("-Wno-unused-parameter");
+        consensus_config.flag("-std=c++17").flag("-Wno-unused-parameter");
     }
 
-    if target.contains("windows") {
-        base_config.define("WIN32", "1");
-    }
-    base_config
+    consensus_config
+        .cpp(true)
+        .include("depend/bitcoin/src")
+        .include("depend/bitcoin/src/secp256k1/include")
         .file("depend/bitcoin/src/util/strencodings.cpp")
         .file("depend/bitcoin/src/uint256.cpp")
         .file("depend/bitcoin/src/pubkey.cpp")

depend/bitcoin-HEAD-revision.txt

@@ -0,0 +1,2 @@
+# This file was automatically created by vendor-bitcoin-core.sh
+af591f2068d0363c92d9756ca39c43db85e5804c

depend/bitcoin/.appveyor.yml

@@ -7,29 +7,24 @@ clone_depth: 5
 environment:
   PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%'
   PYTHONUTF8: 1
-  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
-  QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
+  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt598x64_vs2019_v1681/qt598_x64_vs2019_1681.zip'
+  QT_DOWNLOAD_HASH: '00cf7327818c07d74e0b1a4464ffe987c2728b00d49d4bf333065892af0515c3'
   QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
-  VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
-  VCPKG_COMMIT_ID: 'f3f329a048eaff759c1992c458f2e12351486bc7'
+  VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f'
 install:
 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
 # - cmd: pip install zmq
-# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is:
+# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is:
 #    a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary,
-#    b. Install the missing packages.
+#    b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins).
+# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed
+# as part of the msbuild command using vcpkg mainfests.
 - ps: |
-      $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt
-      Write-Host "vcpkg installing packages: $env:PACKAGES"
       cd c:\tools\vcpkg
       $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead.
-      git pull origin master > $null
-      git -c advice.detachedHead=false checkout $env:VCPKG_COMMIT_ID
+      git -c advice.detachedHead=false checkout $env:VCPKG_TAG
       .\bootstrap-vcpkg.bat > $null
       Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)"
-      .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null
-      Write-Host "vcpkg packages installed successfully."
-      .\vcpkg integrate install
       cd "$env:APPVEYOR_BUILD_FOLDER"
 before_build:
 # Powershell block below is to download and extract the Qt static libraries. The pseudo code is:
@@ -55,11 +50,14 @@ after_build:
 #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe
 test_script:
 - cmd: src\test_bitcoin.exe -l test_suite
-- cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL
+- cmd: src\bench_bitcoin.exe > NUL
 - ps:  python test\util\bitcoin-util-test.py
 - cmd: python test\util\rpcauth-test.py
 # Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted.
-- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
+# functional tests disabled for now. See
+# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202
+# https://github.com/bitcoin/bitcoin/issues/18623
+# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
 artifacts:
 #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip
 deploy: off

depend/bitcoin/.cirrus.yml

@@ -1,23 +1,146 @@
-task:
-  name: "x86_64 Linux  [GOAL: install]  [bionic]  [Using ./ci/ system]"
-  container:
-    image: ubuntu:18.04
-    cpu: 8
-    memory: 8G
-  timeout_in: 60m
-  env:
-    MAKEJOBS: "-j9"
-    DANGER_RUN_CI_ON_HOST: "1"
-    TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-    CCACHE_SIZE: "200M"
-    CCACHE_DIR: "/tmp/ccache_dir"
+### Global defaults
+
+timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
+container:
+  # https://cirrus-ci.org/faq/#are-there-any-limits
+  # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
+  cpu: 2
+  memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
+  kvm: true  # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093
+env:
+  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
+  MAKEJOBS: "-j4"
+  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
+  TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
+  CCACHE_SIZE: "200M"
+  CCACHE_DIR: "/tmp/ccache_dir"
+
+### Global task template
+
+# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
+global_task_template: &GLOBAL_TASK_TEMPLATE
+  skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
   ccache_cache:
     folder: "/tmp/ccache_dir"
   depends_built_cache:
     folder: "/tmp/cirrus-ci-build/depends/built"
-  install_script:
-    - apt-get update
-    - apt-get -y install git bash ccache
-    - ccache --max-size=${CCACHE_SIZE}
+  depends_sdk_cache:
+    folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+  depends_releases_cache:
+    folder: "/tmp/cirrus-ci-build/releases"
+  merge_base_script:
+    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    - bash -c "$PACKAGE_MANAGER_INSTALL git"
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
+    - git config --global user.email "[email protected]"
+    - git config --global user.name "ci"
+    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
   ci_script:
     - ./ci/test_run_all.sh
+
+#task:
+#  name: "Windows"
+#  windows_container:
+#    image: cirrusci/windowsservercore:2019
+#  env:
+#    CIRRUS_SHELL: powershell
+#    PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%'
+#    PYTHONUTF8: 1
+#    QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
+#    QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
+#    QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
+#    VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
+#    VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74'
+#  install_script:
+#    - choco install python --version=3.7.7 -y
+
+task:
+  name: 'ARM  [GOAL: install]  [buster]  [unit tests, no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: debian:buster
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
+
+task:
+  name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no boost::process, no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [bionic]  [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [depends, sanitizers: thread (TSan), no gui]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+    cpu: 4  # Double CPU and Memory to avoid timeout
+    memory: 16G
+  env:
+    MAKEJOBS: "-j8"
+    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [depends, sanitizers: memory (MSan)]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [no depends, only system libs, sanitizers: fuzzer,address,undefined]'
+  only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_BASE_BRANCH == $CIRRUS_DEFAULT_BRANCH
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
+
+task:
+  name: 'x86_64 Linux [GOAL: install]  [focal]  [multiprocess]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
+
+task:
+  name: 'macOS 10.12  [GOAL: deploy] [no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+
+task:
+  name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]'
+  brew_install_script:
+    - brew update
+    - brew install boost libevent berkeley-db4 qt@5 miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
+  << : *GLOBAL_TASK_TEMPLATE
+  osx_instance:
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
+    image: catalina-xcode-12.1  # https://cirrus-ci.org/guide/macOS
+  env:
+    DANGER_RUN_CI_ON_HOST: "true"
+    CI_USE_APT_INSTALL: "no"
+    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
+    FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"

depend/bitcoin/.fuzzbuzz.yml

@@ -0,0 +1,16 @@
+base: ubuntu:16.04
+language: c++
+engine: libFuzzer
+environment:
+  - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1
+setup:
+  - sudo apt-get update
+  - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libboost-program-options-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libprotobuf-dev libqt5core5a libqt5dbus5 libqt5gui5 libssl-dev libtool llvm llvm-dev llvm-runtime pkg-config protobuf-compiler qttools5-dev qttools5-dev-tools software-properties-common
+  - ./autogen.sh
+  - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined
+  - make
+  - git clone https://github.com/bitcoin-core/qa-assets
+auto_targets:
+  find_targets_command: find src/test/fuzz/ -executable -type f ! -name "*.cpp" ! -name "*.h"
+  base_corpus_dir: qa-assets/fuzz_seed_corpus/
+  memory_limit: none

depend/bitcoin/.github/ISSUE_TEMPLATE.md

@@ -4,7 +4,11 @@ General bitcoin questions and/or support requests are best directed to the Bitco
 
 For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/.
 
-If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! -->
+If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue!
+
+Any report, issue or feature request related to the GUI should be reported at
+https://github.com/bitcoin-core/gui/issues/
+-->
 
 <!-- Describe the issue -->
 <!--- What behavior did you expect? -->

depend/bitcoin/.github/ISSUE_TEMPLATE/good_first_issue.md

@@ -2,20 +2,21 @@
 name: Good first issue
 about: '(Regular devs only): Suggest a new good first issue'
 title: ''
-labels: good first issue
+labels: ''
 assignees: ''
 
 ---
 
+<!-- Needs the label "good first issue" assigned manually before or after opening -->
+
+<!-- A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution -->
+
+<!-- Motivate the issue and explain the solution briefly -->
 
 #### Useful skills:
 
 <!-- (For example, “C++11 std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) -->
 
 #### Want to work on this issue?
 
-The purpose of the `good first issue` label is to highlight which issues are suitable for a new contributor without a deep understanding of the codebase.
-
-You do not need to request permission to start working on this. You are encouraged to comment on the issue if you are planning to work on it. This will help other contributors monitor which issues are actively being addressed and is also an effective way to request assistance if and when you need it.
-
 For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request.