diff --git a/.github/workflows/build-centos-7.yml b/.github/workflows/build-centos-7.yml index 9a1d680360a2..dc9510d8d455 100644 --- a/.github/workflows/build-centos-7.yml +++ b/.github/workflows/build-centos-7.yml @@ -2,6 +2,10 @@ name: build-centos-7 on: push: + paths-ignore: + # Don't run this workflow if only files under doc/ have been + # modified. + - "doc/**" pull_request: workflow_dispatch: inputs: diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index ac51b07dcd51..cdb37174c7a6 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -2,6 +2,10 @@ name: builds on: push: + paths-ignore: + # Don't run this workflow if only files under doc/ have been + # modified. + - "doc/**" pull_request: workflow_dispatch: inputs: @@ -19,9 +23,6 @@ concurrency: permissions: read-all env: - DEFAULT_SV_REPO: https://github.com/OISF/suricata-verify - DEFAULT_SV_BRANCH: master - DEFAULT_CFLAGS: "-Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-function" # Apt sometimes likes to ask for user input, this will prevent that. @@ -36,116 +37,7 @@ jobs: prepare-deps: name: Prepare dependencies - runs-on: ubuntu-latest - steps: - - name: Dumping github context for debugging - run: echo $JSON - env: - JSON: ${{ toJSON(github) }} - - run: sudo apt update && sudo apt -y install jq curl - - name: Parse repo and branch information - env: - # We fetch the actual pull request to get the latest body as - # github.event.pull_request.body has the body from the - # initial pull request. - PR_HREF: ${{ github.event.pull_request._links.self.href }} - run: | - if test "${PR_HREF}"; then - body=$(curl -s "${PR_HREF}" | jq -r .body | tr -d '\r') - - echo "Parsing branch and PR info from:" - echo "${body}" - - LIBHTP_REPO=$(echo "${body}" | awk -F = '/^LIBHTP_REPO=/ { print $2 }') - LIBHTP_BRANCH=$(echo "${body}" | awk -F = '/^LIBHTP_BRANCH=/ { print $2 }') - - SU_REPO=$(echo "${body}" | awk -F = '/^SU_REPO=/ { print $2 }') - SU_BRANCH=$(echo "${body}" | awk -F = '/^SU_BRANCH=/ { print $2 }') - - SV_REPO=$(echo "${body}" | awk -F = '/^SV_REPO=/ { print $2 }') - SV_BRANCH=$(echo "${body}" | awk -F = '/^SV_BRANCH=/ { print $2 }') - else - echo "No pull request body, will use inputs or defaults." - LIBHTP_REPO=${{ inputs.LIBHTP_REPO }} - LIBHTP_BRANCH=${{ inputs.LIBHTP_BRANCH }} - SU_REPO=${{ inputs.SU_REPO }} - SU_BRANCH=${{ inputs.SU_BRANCH }} - SV_REPO=${{ inputs.SV_REPO }} - SV_BRANCH=${{ inputs.SV_BRANCH }} - fi - - # If the _REPO variables don't contain a full URL, add GitHub. - if [ "${LIBHTP_REPO}" ] && ! echo "${LIBHTP_REPO}" | grep -q '^https://'; then - LIBHTP_REPO="https://github.com/${LIBHTP_REPO}" - fi - if [ "${SU_REPO}" ] && ! echo "${SU_REPO}" | grep -q '^https://'; then - SU_REPO="https://github.com/${SU_REPO}" - fi - if [ "${SV_REPO}" ] && ! echo "${SV_REPO}" | grep -q '^https://'; then - SV_REPO="https://github.com/${SV_REPO}" - fi - - echo LIBHTP_REPO=${LIBHTP_REPO} | tee -a ${GITHUB_ENV} - echo LIBHTP_BRANCH=${LIBHTP_BRANCH} | tee -a ${GITHUB_ENV} - - echo SU_REPO=${SU_REPO} | tee -a ${GITHUB_ENV} - echo SU_BRANCH=${SU_BRANCH} | tee -a ${GITHUB_ENV} - - echo SV_REPO=${SV_REPO:-${DEFAULT_SV_REPO}} | tee -a ${GITHUB_ENV} - echo SV_BRANCH=${SV_BRANCH:-${DEFAULT_SV_BRANCH}} | tee -a ${GITHUB_ENV} - - - name: Annotate output - run: | - echo "::notice:: LIBHTP_REPO=${LIBHTP_REPO}" - echo "::notice:: LIBHTP_BRANCH=${LIBHTP_BRANCH}" - echo "::notice:: SU_REPO=${SU_REPO}" - echo "::notice:: SU_BRANCH=${SU_BRANCH}" - echo "::notice:: SV_REPO=${SV_REPO}" - echo "::notice:: SV_BRANCH=${SV_BRANCH}" - - # Now checkout Suricata for the bundle script. - - name: Checking out Suricata - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - run: git config --global --add safe.directory /__w/suricata/suricata - - - name: Fetching libhtp - run: | - DESTDIR=./bundle ./scripts/bundle.sh libhtp - tar zcf libhtp.tar.gz -C bundle libhtp - - name: Fetching suricata-update - run: | - DESTDIR=./bundle ./scripts/bundle.sh suricata-update - tar zcf suricata-update.tar.gz -C bundle suricata-update - - - name: Fetching suricata-verify - run: | - # Looking for a pull request number. in the SV_BRANCH - # value. This could be "pr/NNN", "pull/NNN" or a link to an - # OISF/suricata-verify pull request. - pr=$(echo "${SV_BRANCH}" | sed -n \ - -e 's/^https:\/\/github.com\/OISF\/suricata-verify\/pull\/\([0-9]*\)$/\1/p' \ - -e 's/^pull\/\([0-9]*\)$/\1/p' \ - -e 's/^pr\/\([0-9]*\)$/\1/p') - if [ "${pr}" ]; then - SV_BRANCH="refs/pull/${pr}/head" - echo "Using suricata-verify pull-request ${SV_BRANCH}" - else - echo "Using suricata-verify branch ${SV_BRANCH}" - fi - git clone --depth 1 ${SV_REPO} suricata-verify - cd suricata-verify - git fetch --depth 1 origin ${SV_BRANCH} - git -c advice.detachedHead=false checkout FETCH_HEAD - cd .. - tar zcf suricata-verify.tar.gz suricata-verify - - name: Uploading prep archive - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 - with: - name: prep - path: | - libhtp.tar.gz - suricata-update.tar.gz - suricata-verify.tar.gz + uses: ./.github/workflows/prepare-deps.yml prepare-cbindgen: name: Prepare cbindgen @@ -416,6 +308,81 @@ jobs: run: cargo clippy --all-features working-directory: rust + almalinux-9-non-bundled-libhtp: + name: AlmaLinux 9 Non-Bundled LibHTP + runs-on: ubuntu-latest + container: almalinux:9 + needs: [prepare-deps, debian-12-dist] + steps: + # Cache Rust stuff. + - name: Cache cargo registry + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 + with: + path: ~/.cargo + key: ${{ github.job }}-cargo + + - name: Cache RPMs + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 + with: + path: /var/cache/dnf + key: ${{ github.job }}-dnf + - run: echo "keepcache=1" >> /etc/dnf/dnf.conf + + - name: Determine number of CPUs + run: echo CPUS=$(nproc --all) >> $GITHUB_ENV + + - name: Install system packages + run: | + dnf -y install dnf-plugins-core epel-release + dnf config-manager --set-enabled crb + dnf -y install \ + autoconf \ + automake \ + cargo-vendor \ + cbindgen \ + diffutils \ + numactl-devel \ + dpdk-devel \ + file-devel \ + gcc \ + gcc-c++ \ + git \ + jansson-devel \ + jq \ + libtool \ + libyaml-devel \ + libnfnetlink-devel \ + libnetfilter_queue-devel \ + libnet-devel \ + libcap-ng-devel \ + libevent-devel \ + libmaxminddb-devel \ + libpcap-devel \ + libtool \ + lz4-devel \ + make \ + pcre2-devel \ + pkgconfig \ + python3-devel \ + python3-sphinx \ + python3-yaml \ + rust-toolset \ + sudo \ + which \ + zlib-devel + + - name: Download suricata.tar.gz + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e + with: + name: dist + + - run: tar xf suricata-*.tar.gz --strip-components=1 + - run: cd libhtp && ./configure --prefix=/usr/local + - run: cd libhtp && make -j ${{ env.CPUS }} + - run: cd libhtp && make install + + - run: PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./configure --enable-non-bundled-htp --with-libhtp-includes=/usr/local/include --with-libhtp-libraries=/usr/local/lib + almalinux-8: name: AlmaLinux 8 runs-on: ubuntu-latest @@ -1286,7 +1253,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - run: git config --global --add safe.directory /__w/suricata/suricata - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e - name: Install minimal dependencies run: ./scripts/docs-almalinux9-minimal-build.sh @@ -1566,13 +1533,13 @@ jobs: run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.63.0 -y - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - run: git config --global --add safe.directory /__w/suricata/suricata - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: prep path: prep - run: tar xf prep/libhtp.tar.gz - run: tar xf prep/suricata-verify.tar.gz - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: cbindgen path: prep @@ -1681,12 +1648,12 @@ jobs: run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.63.0 -y - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - run: git config --global --add safe.directory /__w/suricata/suricata - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: prep path: prep - run: tar xf prep/libhtp.tar.gz - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: cbindgen path: prep @@ -1825,7 +1792,7 @@ jobs: dpdk-dev - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - run: git config --global --add safe.directory /__w/suricata/suricata - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: prep path: prep @@ -1913,7 +1880,7 @@ jobs: dpdk-dev - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - run: git config --global --add safe.directory /__w/suricata/suricata - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: prep path: prep @@ -2500,7 +2467,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - run: git config --global --add safe.directory /__w/suricata/suricata - - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e with: name: prep path: prep diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000000..233f75f54d29 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,108 @@ +name: docs + +on: + push: + paths: + # Something has to change in doc/ for thos workflow to be run. + - "doc/**" + pull_request: + workflow_dispatch: + inputs: + LIBHTP_REPO: + LIBHTP_BRANCH: + SU_REPO: + SU_BRANCH: + SV_REPO: + SV_BRANCH: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: read-all + +env: + DEFAULT_CFLAGS: "-Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-function" + + # Apt sometimes likes to ask for user input, this will prevent that. + DEBIAN_FRONTEND: "noninteractive" + +jobs: + + prepare-deps: + name: Prepare dependencies + uses: ./.github/workflows/prepare-deps.yml + + debian-12-dist: + name: Debian 12 Dist Builder + runs-on: ubuntu-latest + container: debian:12 + needs: [prepare-deps] + steps: + # Cache Rust stuff. + - name: Cache cargo registry + uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 + with: + path: ~/.cargo + key: ${{ github.job }}-cargo + + - name: Determine number of CPUs + run: echo CPUS=$(nproc --all) >> $GITHUB_ENV + + - run: apt update + - run: | + apt -y install \ + autoconf \ + automake \ + build-essential \ + cargo \ + cbindgen \ + cmake \ + curl \ + git \ + jq \ + make \ + libpcre3 \ + libpcre3-dbg \ + libpcre3-dev \ + libpcre2-dev \ + libtool \ + libpcap-dev \ + libnet1-dev \ + libyaml-0-2 \ + libyaml-dev \ + libcap-ng-dev \ + libcap-ng0 \ + libmagic-dev \ + libjansson-dev \ + libjansson4 \ + liblz4-dev \ + libssl-dev \ + liblzma-dev \ + pkg-config \ + python3 \ + python3-yaml \ + rustc \ + sphinx-doc \ + sphinx-common \ + texlive-latex-base \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + texlive-latex-extra \ + zlib1g \ + zlib1g-dev + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - run: git config --global --add safe.directory /__w/suricata/suricata + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e + with: + name: prep + path: prep + - run: tar xf prep/libhtp.tar.gz + - run: tar xf prep/suricata-update.tar.gz + - run: ./autogen.sh + - run: CFLAGS="${DEFAULT_CFLAGS}" ./configure + - run: make dist + - run: test -e doc/userguide/suricata.1 + - run: test -e doc/userguide/userguide.pdf + - name: Building HTML documentation + run: cd doc/userguide && make html diff --git a/.github/workflows/prepare-deps.yml b/.github/workflows/prepare-deps.yml new file mode 100644 index 000000000000..9cd95dc9b665 --- /dev/null +++ b/.github/workflows/prepare-deps.yml @@ -0,0 +1,121 @@ +on: + workflow_call: + +env: + DEFAULT_SV_REPO: https://github.com/OISF/suricata-verify + DEFAULT_SV_BRANCH: master + +jobs: + + prepare-deps: + name: Prepare dependencies + runs-on: ubuntu-latest + steps: + - name: Dumping github context for debugging + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - run: sudo apt update && sudo apt -y install jq curl + - name: Parse repo and branch information + env: + # We fetch the actual pull request to get the latest body as + # github.event.pull_request.body has the body from the + # initial pull request. + PR_HREF: ${{ github.event.pull_request._links.self.href }} + run: | + if test "${PR_HREF}"; then + body=$(curl -s "${PR_HREF}" | jq -r .body | tr -d '\r') + + echo "Parsing branch and PR info from:" + echo "${body}" + + LIBHTP_REPO=$(echo "${body}" | awk -F = '/^LIBHTP_REPO=/ { print $2 }') + LIBHTP_BRANCH=$(echo "${body}" | awk -F = '/^LIBHTP_BRANCH=/ { print $2 }') + + SU_REPO=$(echo "${body}" | awk -F = '/^SU_REPO=/ { print $2 }') + SU_BRANCH=$(echo "${body}" | awk -F = '/^SU_BRANCH=/ { print $2 }') + + SV_REPO=$(echo "${body}" | awk -F = '/^SV_REPO=/ { print $2 }') + SV_BRANCH=$(echo "${body}" | awk -F = '/^SV_BRANCH=/ { print $2 }') + else + echo "No pull request body, will use inputs or defaults." + LIBHTP_REPO=${{ inputs.LIBHTP_REPO }} + LIBHTP_BRANCH=${{ inputs.LIBHTP_BRANCH }} + SU_REPO=${{ inputs.SU_REPO }} + SU_BRANCH=${{ inputs.SU_BRANCH }} + SV_REPO=${{ inputs.SV_REPO }} + SV_BRANCH=${{ inputs.SV_BRANCH }} + fi + + # If the _REPO variables don't contain a full URL, add GitHub. + if [ "${LIBHTP_REPO}" ] && ! echo "${LIBHTP_REPO}" | grep -q '^https://'; then + LIBHTP_REPO="https://github.com/${LIBHTP_REPO}" + fi + if [ "${SU_REPO}" ] && ! echo "${SU_REPO}" | grep -q '^https://'; then + SU_REPO="https://github.com/${SU_REPO}" + fi + if [ "${SV_REPO}" ] && ! echo "${SV_REPO}" | grep -q '^https://'; then + SV_REPO="https://github.com/${SV_REPO}" + fi + + echo LIBHTP_REPO=${LIBHTP_REPO} | tee -a ${GITHUB_ENV} + echo LIBHTP_BRANCH=${LIBHTP_BRANCH} | tee -a ${GITHUB_ENV} + + echo SU_REPO=${SU_REPO} | tee -a ${GITHUB_ENV} + echo SU_BRANCH=${SU_BRANCH} | tee -a ${GITHUB_ENV} + + echo SV_REPO=${SV_REPO:-${DEFAULT_SV_REPO}} | tee -a ${GITHUB_ENV} + echo SV_BRANCH=${SV_BRANCH:-${DEFAULT_SV_BRANCH}} | tee -a ${GITHUB_ENV} + + - name: Annotate output + run: | + echo "::notice:: LIBHTP_REPO=${LIBHTP_REPO}" + echo "::notice:: LIBHTP_BRANCH=${LIBHTP_BRANCH}" + echo "::notice:: SU_REPO=${SU_REPO}" + echo "::notice:: SU_BRANCH=${SU_BRANCH}" + echo "::notice:: SV_REPO=${SV_REPO}" + echo "::notice:: SV_BRANCH=${SV_BRANCH}" + + # Now checkout Suricata for the bundle script. + - name: Checking out Suricata + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - run: git config --global --add safe.directory /__w/suricata/suricata + + - name: Fetching libhtp + run: | + DESTDIR=./bundle ./scripts/bundle.sh libhtp + tar zcf libhtp.tar.gz -C bundle libhtp + - name: Fetching suricata-update + run: | + DESTDIR=./bundle ./scripts/bundle.sh suricata-update + tar zcf suricata-update.tar.gz -C bundle suricata-update + + - name: Fetching suricata-verify + run: | + # Looking for a pull request number. in the SV_BRANCH + # value. This could be "pr/NNN", "pull/NNN" or a link to an + # OISF/suricata-verify pull request. + pr=$(echo "${SV_BRANCH}" | sed -n \ + -e 's/^https:\/\/github.com\/OISF\/suricata-verify\/pull\/\([0-9]*\)$/\1/p' \ + -e 's/^pull\/\([0-9]*\)$/\1/p' \ + -e 's/^pr\/\([0-9]*\)$/\1/p') + if [ "${pr}" ]; then + SV_BRANCH="refs/pull/${pr}/head" + echo "Using suricata-verify pull-request ${SV_BRANCH}" + else + echo "Using suricata-verify branch ${SV_BRANCH}" + fi + git clone --depth 1 ${SV_REPO} suricata-verify + cd suricata-verify + git fetch --depth 1 origin ${SV_BRANCH} + git -c advice.detachedHead=false checkout FETCH_HEAD + cd .. + tar zcf suricata-verify.tar.gz suricata-verify + - name: Uploading prep archive + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 + with: + name: prep + path: | + libhtp.tar.gz + suricata-update.tar.gz + suricata-verify.tar.gz