From 4acc0b2f82303aaba8c4a51a25345e200635ff69 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 22:05:08 -0400 Subject: [PATCH 01/21] KaaS Update for image --- src/containers/docker/Dockerfile | 105 ++++++++++++++++++++++++++++ src/containers/docker/README.rst | 21 ++++++ src/containers/docker/Rprofile.site | 4 ++ 3 files changed, 130 insertions(+) create mode 100644 src/containers/docker/Dockerfile create mode 100644 src/containers/docker/README.rst create mode 100644 src/containers/docker/Rprofile.site diff --git a/src/containers/docker/Dockerfile b/src/containers/docker/Dockerfile new file mode 100644 index 00000000..4eadf301 --- /dev/null +++ b/src/containers/docker/Dockerfile @@ -0,0 +1,105 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/base-notebook +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for fully functional notebook server +# added curl and gnupg for analytics +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + # Common useful utilities + git \ + curl \ + gnupg\ + nano-tiny \ + tzdata \ + unzip \ + vim-tiny \ + # git-over-ssh + openssh-client \ + # less is needed to run help in R + # see: https://github.com/jupyter/docker-stacks/issues/1588 + less \ + # nbconvert dependencies + # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex + texlive-xetex \ + texlive-fonts-recommended \ + texlive-plain-generic \ + # Enable clipboard on Linux host systems + xclip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# analytics prereq +RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash + +RUN sudo apt-get install --yes git-lfs + +# Install kestrel-LANG +RUN pip install kestrel-lang + +# Create alternative for nano -> nano-tiny +RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10 + +RUN mkdir -p /opt/kaas && chown ${NB_UID}:${NB_UID} /opt/kaas + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} + +# Install kestrel-JUPYTER +RUN pip install kestrel-jupyter + +# setup Jupyter +RUN python -m kestrel_jupyter_kernel.setup + +RUN mv "${HOME}/.local/share/jupyter/kernels/kestrel"* "${CONDA_DIR}/share/jupyter/kernels/" && \ + chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \ + rm -rf "${HOME}/.local" + +# Add R mimetype option to specify how the plot returns from R to the browser +RUN curl https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Rprofile.site -o ${HOME}/Rprofile.site +RUN chown ${NB_UID}:${NB_GID} ${HOME}/Rprofile.site && mkdir -p /opt/conda/lib/R/etc && mv Rprofile.site /opt/conda/lib/R/etc/Rprofile.site +#COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/ + +# Install kestrel-ANALYTICS +RUN git clone https://github.com/opencybersecurityalliance/data-bucket-kestrel.git /opt/kaas/data-bucket-kestrel + +RUN mkdir -p /opt/kaas/analytic-tutorials && \ + mkdir -p /opt/kaas/huntbooks && \ + mkdir -p /opt/kaas/tutorial && \ + mkdir -p /opt/kaas/blackhat22 + +# for generic huntbooks like BH22 +RUN cp -r /opt/kaas/data-bucket-kestrel/stix-bundles /opt/kaas/analytic-tutorials + +# for the tutorial +RUN cp /opt/kaas/data-bucket-kestrel/stix-bundles/lab101.json /opt/kaas/analytic-tutorials + +RUN git clone https://github.com/opencybersecurityalliance/kestrel-analytics.git /opt/kaas/kestrel-analytics + +RUN cp /opt/kaas/data-bucket-kestrel/GeoLite2/GeoLite2-City.mmdb /opt/kaas/kestrel-analytics/analytics/piniponmap + +RUN rm -rf "/opt/kaas/data-bucket-kestrel" + +# install kestrel-HUNTBOOK +RUN git clone https://github.com/opencybersecurityalliance/kestrel-huntbook.git /opt/kaas/kestrel-huntbook + +RUN mkdir -p /home/jovyan/.config/kestrel && \ + mv /opt/kaas/kestrel-huntbook/config/stixshifter.yaml /home/jovyan/.config/kestrel/ && \ + ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/huntbooks/stixshifter.yaml && \ + ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/tutorial/stixshifter.yaml && \ + ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/blackhat22/stixshifter.yaml + +RUN rmdir /opt/kaas/kestrel-huntbook/config + +RUN cp /opt/kaas/kestrel-analytics/pythonanalytics_sample.yaml /home/jovyan/.config/kestrel/pythonanalytics.yaml + +#RUN jupyter trust */*.ipynb \ No newline at end of file diff --git a/src/containers/docker/README.rst b/src/containers/docker/README.rst new file mode 100644 index 00000000..eae73861 --- /dev/null +++ b/src/containers/docker/README.rst @@ -0,0 +1,21 @@ +The jupyter hub docker images are at: https://github.com/jupyter/docker-stacks. We will extend them for Kestrel-as-a-Service and other use cases. At the moment the docker image is getting pushed to - https://hub.docker.com/repository/docker/kpeeples/kaas-baseline/general. The dockerfile in this repo includes kestrel-lang, kestrel-analytics, kestrel-huntbook and tutorials. The workflow file automatically builds the image upon commits. + +Steps to manually create the image: + +- Login to dockerhub using one of the examples +A. $ docker login +Username: +Password: +Login Succeeded +B. $ docker login --username demo --password example +C. $ cat password.txt | docker login --username demo --password-stdin +Note: you can use a credential helper - https://docs.docker.com/engine/reference/commandline/login/#credentials-store + +- Manually Build image using one of the examples, the below url should change to the kestrel-lang url +A. $ docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v1 +B. $ sudo docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v2 https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Dockerfile + +- Push image, version should be incremental and will be standardized later +A. $ sudo docker push kpeeples/kaas-baseline:v1 +B. $ sudo docker push kpeeples/kaas-baseline:latest + diff --git a/src/containers/docker/Rprofile.site b/src/containers/docker/Rprofile.site new file mode 100644 index 00000000..3d6a93cc --- /dev/null +++ b/src/containers/docker/Rprofile.site @@ -0,0 +1,4 @@ +# Add R mimetype to specify how the plot returns from R to the browser. +# https://notebook.community/andrie/jupyter-notebook-samples/Changing%20R%20plot%20options%20in%20Jupyter + +options(jupyter.plot_mimetypes = c('text/plain', 'image/png', 'image/jpeg', 'image/svg+xml', 'application/pdf')) From 48db74fb66c7cd801d75a3b0f7e8c45ebacdd75f Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 22:05:56 -0400 Subject: [PATCH 02/21] Update for KaaS image --- src/containers/oci/README.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/containers/oci/README.rst diff --git a/src/containers/oci/README.rst b/src/containers/oci/README.rst new file mode 100644 index 00000000..d584cb45 --- /dev/null +++ b/src/containers/oci/README.rst @@ -0,0 +1 @@ +This is a placeholder for any future oci formats From c7462a74010918925ca7454d345a123977bb3981 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 22:29:13 -0400 Subject: [PATCH 03/21] add docker build and push image --- .github/workflows/kaas-docker-image.yml | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/kaas-docker-image.yml diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml new file mode 100644 index 00000000..acabeeb4 --- /dev/null +++ b/.github/workflows/kaas-docker-image.yml @@ -0,0 +1,48 @@ +name: Build and publish a Docker image +on: + push: + branches: + - '*' + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + pull_request: + branches: + - '*' +jobs: + build: + name: Build & push docker image + runs-on: ubuntu-latest + env: + IMG_NAME: kaas-baseline + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Debug + run: | + echo "github.ref -> {{ github.ref }}" + + - name: Docker metadata + id: metadata + uses: docker/metadata-action@v3 + with: + images: ${{ env.IMG_NAME }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value={{sha}},enable=${{ github.ref_type != 'tag' }} + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + if: ${{ github.ref_type == 'tag' }} + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event.base_ref =='refs/heads/main' && github.ref_type == 'tag' && !startsWith(github.ref, 'refs/tags/v0.')}} + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} \ No newline at end of file From 5b5c9bb1a98726e3527524c6d0ba840494c75663 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 22:45:12 -0400 Subject: [PATCH 04/21] update workflow file --- .github/workflows/kaas-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index acabeeb4..232f862e 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -13,7 +13,7 @@ jobs: name: Build & push docker image runs-on: ubuntu-latest env: - IMG_NAME: kaas-baseline + IMG_NAME: ${{ github.repository }} steps: - name: Checkout uses: actions/checkout@v3 From a202128cac41c970f4e37c3a2fc74e264c0b2dc2 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 23:08:31 -0400 Subject: [PATCH 05/21] update latest --- .github/workflows/kaas-docker-image.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index 232f862e..8f8e2c88 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -1,4 +1,4 @@ -name: Build and publish a Docker image +name: Docker image on: push: branches: @@ -13,36 +13,34 @@ jobs: name: Build & push docker image runs-on: ubuntu-latest env: - IMG_NAME: ${{ github.repository }} + IMG_NAME: https://github.com/kpeeples/kestrel-lang/tree/kaas-container/src/containers/docker steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v3.5.3 + + - name: Info + run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}" - - name: Debug - run: | - echo "github.ref -> {{ github.ref }}" - - name: Docker metadata id: metadata - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ env.IMG_NAME }} tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - type=raw,value={{sha}},enable=${{ github.ref_type != 'tag' }} + type=raw,value=dev-{{date 'YYYYMMDDhhmm'}}.{{sha}},enable=${{ github.ref_type != 'tag' }} - name: Log in to Docker Hub - uses: docker/login-action@v1 - if: ${{ github.ref_type == 'tag' }} + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . push: ${{ github.event.base_ref =='refs/heads/main' && github.ref_type == 'tag' && !startsWith(github.ref, 'refs/tags/v0.')}} tags: ${{ steps.metadata.outputs.tags }} - labels: ${{ steps.metadata.outputs.labels }} \ No newline at end of file + labels: ${{ steps.metadata.outputs.labels }} From 51e81651c73f88402f852067164fd4a07e06744f Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 23:25:17 -0400 Subject: [PATCH 06/21] update image location --- .github/workflows/kaas-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index 8f8e2c88..f7db3786 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -40,7 +40,7 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v4 with: - context: . + context: "{{defaultContext}}:src/containers/docker" push: ${{ github.event.base_ref =='refs/heads/main' && github.ref_type == 'tag' && !startsWith(github.ref, 'refs/tags/v0.')}} tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} From 2d8cbee03287ae4dd9471d46c6062c6975e928e9 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 23:27:41 -0400 Subject: [PATCH 07/21] update image location --- .github/workflows/kaas-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index f7db3786..6142f598 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -13,7 +13,7 @@ jobs: name: Build & push docker image runs-on: ubuntu-latest env: - IMG_NAME: https://github.com/kpeeples/kestrel-lang/tree/kaas-container/src/containers/docker + IMG_NAME: ${{ github.repository }} steps: - name: Checkout uses: actions/checkout@v3.5.3 From a84a00fb6908905ee1eaefd0039daa2e93d93b4d Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Tue, 15 Aug 2023 23:52:41 -0400 Subject: [PATCH 08/21] update workflow --- .github/workflows/kaas-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index 6142f598..35396175 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -13,7 +13,7 @@ jobs: name: Build & push docker image runs-on: ubuntu-latest env: - IMG_NAME: ${{ github.repository }} + IMG_NAME: "kpeeples/kaas-baseline" steps: - name: Checkout uses: actions/checkout@v3.5.3 @@ -43,4 +43,4 @@ jobs: context: "{{defaultContext}}:src/containers/docker" push: ${{ github.event.base_ref =='refs/heads/main' && github.ref_type == 'tag' && !startsWith(github.ref, 'refs/tags/v0.')}} tags: ${{ steps.metadata.outputs.tags }} - labels: ${{ steps.metadata.outputs.labels }} + From 9037bfab2faf002d841dd0e18372b0816d12027a Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Wed, 16 Aug 2023 00:01:39 -0400 Subject: [PATCH 09/21] update workflow --- .github/workflows/kaas-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index 35396175..32dc9159 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -13,7 +13,7 @@ jobs: name: Build & push docker image runs-on: ubuntu-latest env: - IMG_NAME: "kpeeples/kaas-baseline" + IMG_NAME: "kaas-baseline" steps: - name: Checkout uses: actions/checkout@v3.5.3 From a7ddc89c651ea94820f6f9a4ae13557ef394ce01 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Wed, 16 Aug 2023 00:17:04 -0400 Subject: [PATCH 10/21] update workflow --- .github/workflows/kaas-docker-image.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index 32dc9159..c335dc38 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -12,8 +12,6 @@ jobs: build: name: Build & push docker image runs-on: ubuntu-latest - env: - IMG_NAME: "kaas-baseline" steps: - name: Checkout uses: actions/checkout@v3.5.3 @@ -21,16 +19,6 @@ jobs: - name: Info run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}" - - name: Docker metadata - id: metadata - uses: docker/metadata-action@v4 - with: - images: ${{ env.IMG_NAME }} - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=dev-{{date 'YYYYMMDDhhmm'}}.{{sha}},enable=${{ github.ref_type != 'tag' }} - - name: Log in to Docker Hub uses: docker/login-action@v2 with: @@ -41,6 +29,6 @@ jobs: uses: docker/build-push-action@v4 with: context: "{{defaultContext}}:src/containers/docker" - push: ${{ github.event.base_ref =='refs/heads/main' && github.ref_type == 'tag' && !startsWith(github.ref, 'refs/tags/v0.')}} - tags: ${{ steps.metadata.outputs.tags }} + push: true + tags: kpeeples/kaas-baseline:latest From 281f17484dfdf7e13945757e1ae72fcfa4f11c66 Mon Sep 17 00:00:00 2001 From: Kenneth Peeples Date: Wed, 30 Aug 2023 21:04:47 -0400 Subject: [PATCH 11/21] Update from PR comments and recommendations --- .github/workflows/kaas-docker-image.yml | 15 +++++---------- docs/deployment/dockerhub.rst | 19 +++++++++++++++++++ docs/deployment/index.rst | 13 +++++++++++++ docs/deployment/oci.rst | 2 ++ docs/index.rst | 1 + src/containers/docker/README.rst | 21 --------------------- 6 files changed, 40 insertions(+), 31 deletions(-) create mode 100644 docs/deployment/dockerhub.rst create mode 100644 docs/deployment/index.rst create mode 100644 docs/deployment/oci.rst delete mode 100644 src/containers/docker/README.rst diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index c335dc38..4b8c50ca 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -1,13 +1,8 @@ name: Docker image on: - push: - branches: - - '*' - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - branches: - - '*' + release: + types: [published] + workflow_dispatch: jobs: build: name: Build & push docker image @@ -22,8 +17,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build and push Docker image uses: docker/build-push-action@v4 diff --git a/docs/deployment/dockerhub.rst b/docs/deployment/dockerhub.rst new file mode 100644 index 00000000..580dbdee --- /dev/null +++ b/docs/deployment/dockerhub.rst @@ -0,0 +1,19 @@ +| The jupyter hub docker images are at: https://github.com/jupyter/docker-stacks. We will extend them for Kestrel-as-a-Service and other use cases. At the moment the docker image is getting pushed to - https://hub.docker.com/repository/docker/kpeeples/kaas-baseline/general. The dockerfile in this repo includes kestrel-lang, kestrel-analytics, kestrel-huntbook and tutorials. The workflow file automatically builds the image upon commits. +| +| Login to dockerhub using one of the examples: +| A. $ docker login +| Username: +| Password: +| Login Succeeded +| B. $ docker login --username demo --password example +| C. $ cat password.txt | docker login --username demo --password-stdin +| Note: you can use a credential helper - https://docs.docker.com/engine/reference/commandline/login/#credentials-store +| +| Manually Build image using one of the examples, the below url should change to the kestrel-lang url +| A. $ docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v1 +| B. $ sudo docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v2 https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Dockerfile +| +| Push image, version should be incremental and will be standardized later +| A. $ sudo docker push kpeeples/kaas-baseline:v1 +| B. $ sudo docker push kpeeples/kaas-baseline:latest +| diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst new file mode 100644 index 00000000..1c93f9bf --- /dev/null +++ b/docs/deployment/index.rst @@ -0,0 +1,13 @@ +================================ +Container Formats for Deployment +================================ + +Docker Format for Dockerhub +=========================== + +.. include:: dockerhub.rst + +OCI Format +========== + +.. include:: oci.rst \ No newline at end of file diff --git a/docs/deployment/oci.rst b/docs/deployment/oci.rst new file mode 100644 index 00000000..f2b2f664 --- /dev/null +++ b/docs/deployment/oci.rst @@ -0,0 +1,2 @@ + +Placeholder for future OCI (Open Container Initiative) - https://opencontainers.org/ diff --git a/docs/index.rst b/docs/index.rst index ce4deb35..57fe4142 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ talks, community entrances, and more. configuration debug runtime + deployment/index theory talks contributing diff --git a/src/containers/docker/README.rst b/src/containers/docker/README.rst deleted file mode 100644 index eae73861..00000000 --- a/src/containers/docker/README.rst +++ /dev/null @@ -1,21 +0,0 @@ -The jupyter hub docker images are at: https://github.com/jupyter/docker-stacks. We will extend them for Kestrel-as-a-Service and other use cases. At the moment the docker image is getting pushed to - https://hub.docker.com/repository/docker/kpeeples/kaas-baseline/general. The dockerfile in this repo includes kestrel-lang, kestrel-analytics, kestrel-huntbook and tutorials. The workflow file automatically builds the image upon commits. - -Steps to manually create the image: - -- Login to dockerhub using one of the examples -A. $ docker login -Username: -Password: -Login Succeeded -B. $ docker login --username demo --password example -C. $ cat password.txt | docker login --username demo --password-stdin -Note: you can use a credential helper - https://docs.docker.com/engine/reference/commandline/login/#credentials-store - -- Manually Build image using one of the examples, the below url should change to the kestrel-lang url -A. $ docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v1 -B. $ sudo docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v2 https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Dockerfile - -- Push image, version should be incremental and will be standardized later -A. $ sudo docker push kpeeples/kaas-baseline:v1 -B. $ sudo docker push kpeeples/kaas-baseline:latest - From 196ff055914102a9d3b845eafc59c4fbc6bf6e1a Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Fri, 1 Sep 2023 14:23:42 -0400 Subject: [PATCH 12/21] reorg container folder and revise doc --- .github/workflows/kaas-docker-image.yml | 7 +-- AUTHORS.rst | 2 + .../docker/Dockerfile | 0 .../docker/Rprofile.site | 0 {src/containers => containers}/oci/README.rst | 0 docs/deployment/dockerhub.rst | 44 +++++++++++-------- docs/deployment/index.rst | 16 +++---- docs/deployment/oci.rst | 3 +- 8 files changed, 41 insertions(+), 31 deletions(-) rename {src/containers => containers}/docker/Dockerfile (100%) rename {src/containers => containers}/docker/Rprofile.site (100%) rename {src/containers => containers}/oci/README.rst (100%) diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml index 4b8c50ca..52d594d7 100644 --- a/.github/workflows/kaas-docker-image.yml +++ b/.github/workflows/kaas-docker-image.yml @@ -1,8 +1,10 @@ -name: Docker image +name: Build Docker image + on: release: types: [published] workflow_dispatch: + jobs: build: name: Build & push docker image @@ -23,7 +25,6 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v4 with: - context: "{{defaultContext}}:src/containers/docker" + context: "{{defaultContext}}:containers/docker" push: true tags: kpeeples/kaas-baseline:latest - diff --git a/AUTHORS.rst b/AUTHORS.rst index 569c8334..4278536b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -19,6 +19,7 @@ Contributors - `Constantin Adam`_ - `Ting Dai`_ - `Leila Rashidi`_ +- `Kenneth Peeples`_ .. _Xiaokui Shu: https://github.com/subbyte .. _Paul Coccoli: https://github.com/pcoccoli @@ -30,3 +31,4 @@ Contributors .. _Constantin Adam: https://github.com/cmadam .. _Ting Dai: https://github.com/tingdai .. _Leila Rashidi: https://github.com/leila-rashidi +.. _Kenneth Peeples: https://github.com/kpeeples diff --git a/src/containers/docker/Dockerfile b/containers/docker/Dockerfile similarity index 100% rename from src/containers/docker/Dockerfile rename to containers/docker/Dockerfile diff --git a/src/containers/docker/Rprofile.site b/containers/docker/Rprofile.site similarity index 100% rename from src/containers/docker/Rprofile.site rename to containers/docker/Rprofile.site diff --git a/src/containers/oci/README.rst b/containers/oci/README.rst similarity index 100% rename from src/containers/oci/README.rst rename to containers/oci/README.rst diff --git a/docs/deployment/dockerhub.rst b/docs/deployment/dockerhub.rst index 580dbdee..ee2abbf9 100644 --- a/docs/deployment/dockerhub.rst +++ b/docs/deployment/dockerhub.rst @@ -1,19 +1,25 @@ -| The jupyter hub docker images are at: https://github.com/jupyter/docker-stacks. We will extend them for Kestrel-as-a-Service and other use cases. At the moment the docker image is getting pushed to - https://hub.docker.com/repository/docker/kpeeples/kaas-baseline/general. The dockerfile in this repo includes kestrel-lang, kestrel-analytics, kestrel-huntbook and tutorials. The workflow file automatically builds the image upon commits. -| -| Login to dockerhub using one of the examples: -| A. $ docker login -| Username: -| Password: -| Login Succeeded -| B. $ docker login --username demo --password example -| C. $ cat password.txt | docker login --username demo --password-stdin -| Note: you can use a credential helper - https://docs.docker.com/engine/reference/commandline/login/#credentials-store -| -| Manually Build image using one of the examples, the below url should change to the kestrel-lang url -| A. $ docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v1 -| B. $ sudo docker build -t kpeeples/kaas-baseline:latest -t kpeeples/kaas-baseline:v2 https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Dockerfile -| -| Push image, version should be incremental and will be standardized later -| A. $ sudo docker push kpeeples/kaas-baseline:v1 -| B. $ sudo docker push kpeeples/kaas-baseline:latest -| +Besides Python package release on PyPI, Kestrel is also released into Docker +container image on DockerHub Kestrel. + +The image provides a full Kestrel runtime composed of the basic Kestrel +runtime, `kestrel-jupyter`_ package, open-source Kestrel analytics in the +`kestrel-analytics repo`_, and open-source Kestrel huntbooks and tutorials in +the `kestrel-huntbook repo`_. + +The image is based on the `docker-stacks`_ Jupyter image, maintained by +`Kenneth Peeples`_, and currently located under Kenneth's DockerHub account: +`kpeeples/kaas-baseline`_. + +To launch the Kestrel container: + +.. code-block:: console + + $ docker run kpeeples/kaas-baseline:latest + + +.. _kestrel-jupyter: https://github.com/opencybersecurityalliance/kestrel-jupyter +.. _kestrel-analytics repo: https://github.com/opencybersecurityalliance/kestrel-analytics +.. _kestrel-huntbook repo: https://github.com/opencybersecurityalliance/kestrel-huntbook +.. _docker-stacks: https://github.com/jupyter/docker-stacks +.. _Kenneth Peeples: https://github.com/kpeeples +.. _kpeeples/kaas-baseline: https://hub.docker.com/repository/docker/kpeeples/kaas-baseline diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst index 1c93f9bf..8b3ef4cf 100644 --- a/docs/deployment/index.rst +++ b/docs/deployment/index.rst @@ -1,13 +1,13 @@ -================================ -Container Formats for Deployment -================================ +==================== +Container Deployment +==================== -Docker Format for Dockerhub -=========================== +Docker (at Dockerhub) +===================== .. include:: dockerhub.rst -OCI Format -========== +OCI +=== -.. include:: oci.rst \ No newline at end of file +.. include:: oci.rst diff --git a/docs/deployment/oci.rst b/docs/deployment/oci.rst index f2b2f664..9ac89af6 100644 --- a/docs/deployment/oci.rst +++ b/docs/deployment/oci.rst @@ -1,2 +1,3 @@ +Placeholder for future `Open Container Initiative`_ (OCI) -Placeholder for future OCI (Open Container Initiative) - https://opencontainers.org/ +.. _Open Container Initiative: https://opencontainers.org/ From f16ea97c70d4d8a9140bb235a8d0ec8296a6899b Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Fri, 1 Sep 2023 14:25:38 -0400 Subject: [PATCH 13/21] typo fix in doc --- docs/deployment/dockerhub.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/dockerhub.rst b/docs/deployment/dockerhub.rst index ee2abbf9..4f097052 100644 --- a/docs/deployment/dockerhub.rst +++ b/docs/deployment/dockerhub.rst @@ -1,5 +1,5 @@ Besides Python package release on PyPI, Kestrel is also released into Docker -container image on DockerHub Kestrel. +container image on DockerHub. The image provides a full Kestrel runtime composed of the basic Kestrel runtime, `kestrel-jupyter`_ package, open-source Kestrel analytics in the From 08b2669e89dc7492534802cf834d5239db04fc0b Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Fri, 1 Sep 2023 15:56:35 -0400 Subject: [PATCH 14/21] container doc update --- docs/deployment/dockerhub.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/deployment/dockerhub.rst b/docs/deployment/dockerhub.rst index 4f097052..e317832a 100644 --- a/docs/deployment/dockerhub.rst +++ b/docs/deployment/dockerhub.rst @@ -1,5 +1,5 @@ -Besides Python package release on PyPI, Kestrel is also released into Docker -container image on DockerHub. +Besides Python package (PyPI), Kestrel is also released into Docker container +image on DockerHub. The image provides a full Kestrel runtime composed of the basic Kestrel runtime, `kestrel-jupyter`_ package, open-source Kestrel analytics in the @@ -7,14 +7,15 @@ runtime, `kestrel-jupyter`_ package, open-source Kestrel analytics in the the `kestrel-huntbook repo`_. The image is based on the `docker-stacks`_ Jupyter image, maintained by -`Kenneth Peeples`_, and currently located under Kenneth's DockerHub account: -`kpeeples/kaas-baseline`_. +`Kenneth Peeples`_, and currently located under `Kenneth's DockerHub account`_. -To launch the Kestrel container: +To launch the Kestrel container (opening Jupyter on host port 8888): .. code-block:: console - $ docker run kpeeples/kaas-baseline:latest + $ docker run -d -p 8888:8888 kpeeples/kaas-baseline:latest + +To have Kestrel syntax highlighting support, use the Jupyter Notebook URL (``http://hostname:8888/tree``) instead of Jupyter Lab (``http://hostname:8888/lab``) for Kestrel huntbooks. .. _kestrel-jupyter: https://github.com/opencybersecurityalliance/kestrel-jupyter @@ -22,4 +23,4 @@ To launch the Kestrel container: .. _kestrel-huntbook repo: https://github.com/opencybersecurityalliance/kestrel-huntbook .. _docker-stacks: https://github.com/jupyter/docker-stacks .. _Kenneth Peeples: https://github.com/kpeeples -.. _kpeeples/kaas-baseline: https://hub.docker.com/repository/docker/kpeeples/kaas-baseline +.. _Kenneth's DockerHub account: https://hub.docker.com/repository/docker/kpeeples/kaas-baseline From 7c664e2bcaeb1dcf7d396ccc9fc337b153e0cef8 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Fri, 1 Sep 2023 16:06:56 -0400 Subject: [PATCH 15/21] README update on Kestrel container --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index b1d514e8..a7dda3a9 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ Visit `Kestrel documentation`_ to learn Kestrel: - `How to connect to your data sources`_ - `How to execute an analytic hunt step in Python/Docker`_ - `How to use Kestrel via API`_ + - `How to launch Kestrel as a Docker container`_ Kestrel Huntbooks And Analytics =============================== @@ -180,6 +181,7 @@ Connecting With The Community .. _How to execute an analytic hunt step in Python/Docker: https://kestrel.readthedocs.io/en/stable/installation/analytics.html .. _Language reference book: https://kestrel.readthedocs.io/en/stable/language/commands.html .. _How to use Kestrel via API: https://kestrel.readthedocs.io/en/stable/source/kestrel.session.html +.. _How to launch Kestrel as a Docker container: https://kestrel.readthedocs.io/en/stable/deployment/ .. _Kestrel documentation on talks: https://kestrel.readthedocs.io/en/stable/talks.html .. _Kestrel huntbook: https://github.com/opencybersecurityalliance/kestrel-huntbook From af4e431699742b187b49d7289bad1bf71ec22d9c Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 7 Sep 2023 10:03:53 -0400 Subject: [PATCH 16/21] update stix-shifter to 6.2.1 --- setup.cfg | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.cfg b/setup.cfg index c5031d97..c353be9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,18 +29,18 @@ scripts = bin/stix-shifter-diag python_requires = >= 3.8 install_requires = - typeguard>=4.0.0 - pyyaml>=6.0 + typeguard>=4.1.3 + pyyaml>=6.0.1 lxml>=4.9.3 - lark>=1.1.5 - pandas>=2.0.0 - pyarrow>=5.0.0 - docker>=5.0.0 + lark>=1.1.7 + pandas>=2.1.0 + pyarrow>=13.0.0 + docker>=6.1.3 requests>=2.31.0 - nest-asyncio>=1.5.6 - stix-shifter==6.0.3 - stix-shifter-utils==6.0.3 - firepit>=2.3.25 + nest-asyncio>=1.5.7 + stix-shifter==6.2.1 + stix-shifter-utils==6.2.1 + firepit>=2.3.27 tests_require = pytest From 69b4acc840550fee58848c4cfe5bcef9e395a7f3 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 7 Sep 2023 10:05:36 -0400 Subject: [PATCH 17/21] add manual trigger of unit tests --- .github/workflows/unit-testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 4b3535bd..9e4f3274 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -21,6 +21,7 @@ on: - opened - reopened - synchronize + workflow_dispatch: jobs: unittest: From d897f1c56613762593a24a9356733c1a827da948 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 7 Sep 2023 10:07:25 -0400 Subject: [PATCH 18/21] trigger unit tests on dependency change --- .github/workflows/unit-testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 9e4f3274..b3b59d42 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -9,6 +9,7 @@ on: - 'src/**' - 'bin/**' - 'tests/**' + - 'setup.cfg' pull_request: branches: - develop @@ -17,11 +18,11 @@ on: - 'src/**' - 'bin/**' - 'tests/**' + - 'setup.cfg' types: - opened - reopened - synchronize - workflow_dispatch: jobs: unittest: From ff5e9b663030da2efd8cc5e4d83deca281ef8431 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 7 Sep 2023 10:08:43 -0400 Subject: [PATCH 19/21] trigger integration test on dependency change --- .github/workflows/integration-testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 229a4140..7ded3a93 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -7,6 +7,7 @@ on: - release paths: - 'src/**' + - 'setup.cfg' pull_request: branches: - develop @@ -14,6 +15,7 @@ on: - release paths: - 'src/**' + - 'setup.cfg' types: - opened - synchronize From 9d1d74b15f86da68cf23934fbb9cff0bd859025e Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 7 Sep 2023 10:09:28 -0400 Subject: [PATCH 20/21] downgrade pandas requirement for Python 3.8 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index c353be9d..288acf3c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ install_requires = pyyaml>=6.0.1 lxml>=4.9.3 lark>=1.1.7 - pandas>=2.1.0 + pandas>=2.0.0 pyarrow>=13.0.0 docker>=6.1.3 requests>=2.31.0 From 07adff171881916c7218e941dce3afd969f856d6 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 7 Sep 2023 11:12:51 -0400 Subject: [PATCH 21/21] v1.7.5 --- CHANGELOG.rst | 10 ++++++++++ setup.cfg | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eb6b963f..7c1ed163 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,16 @@ The format is based on `Keep a Changelog`_. Unreleased ========== +1.7.5 (2023-09-07) +================== + +Added +----- + +- Kestrel Docker container image in DockerHub +- Documentation on how to use Kestrel Docker container +- case insensitive option ``(?i)`` for Elasticserach via stix-shifter data source interface (stix-shifter v6.2.1) + 1.7.4 (2023-08-03) ================== diff --git a/setup.cfg b/setup.cfg index 288acf3c..5c02bd94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kestrel-lang -version = 1.7.4 +version = 1.7.5 description = Kestrel Threat Hunting Language long_description = file:README.rst long_description_content_type = text/x-rst