diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 229a4140..7ded3a93 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -7,6 +7,7 @@ on: - release paths: - 'src/**' + - 'setup.cfg' pull_request: branches: - develop @@ -14,6 +15,7 @@ on: - release paths: - 'src/**' + - 'setup.cfg' types: - opened - synchronize diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml new file mode 100644 index 00000000..52d594d7 --- /dev/null +++ b/.github/workflows/kaas-docker-image.yml @@ -0,0 +1,30 @@ +name: Build Docker image + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + build: + name: Build & push docker image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3.5.3 + + - name: Info + run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}" + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: "{{defaultContext}}:containers/docker" + push: true + tags: kpeeples/kaas-baseline:latest diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 4b3535bd..b3b59d42 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -9,6 +9,7 @@ on: - 'src/**' - 'bin/**' - 'tests/**' + - 'setup.cfg' pull_request: branches: - develop @@ -17,6 +18,7 @@ on: - 'src/**' - 'bin/**' - 'tests/**' + - 'setup.cfg' types: - opened - reopened diff --git a/AUTHORS.rst b/AUTHORS.rst index 569c8334..4278536b 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -19,6 +19,7 @@ Contributors - `Constantin Adam`_ - `Ting Dai`_ - `Leila Rashidi`_ +- `Kenneth Peeples`_ .. _Xiaokui Shu: https://github.com/subbyte .. _Paul Coccoli: https://github.com/pcoccoli @@ -30,3 +31,4 @@ Contributors .. _Constantin Adam: https://github.com/cmadam .. _Ting Dai: https://github.com/tingdai .. _Leila Rashidi: https://github.com/leila-rashidi +.. _Kenneth Peeples: https://github.com/kpeeples diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eb6b963f..7c1ed163 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,16 @@ The format is based on `Keep a Changelog`_. Unreleased ========== +1.7.5 (2023-09-07) +================== + +Added +----- + +- Kestrel Docker container image in DockerHub +- Documentation on how to use Kestrel Docker container +- case insensitive option ``(?i)`` for Elasticserach via stix-shifter data source interface (stix-shifter v6.2.1) + 1.7.4 (2023-08-03) ================== diff --git a/README.rst b/README.rst index b1d514e8..a7dda3a9 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,7 @@ Visit `Kestrel documentation`_ to learn Kestrel: - `How to connect to your data sources`_ - `How to execute an analytic hunt step in Python/Docker`_ - `How to use Kestrel via API`_ + - `How to launch Kestrel as a Docker container`_ Kestrel Huntbooks And Analytics =============================== @@ -180,6 +181,7 @@ Connecting With The Community .. _How to execute an analytic hunt step in Python/Docker: https://kestrel.readthedocs.io/en/stable/installation/analytics.html .. _Language reference book: https://kestrel.readthedocs.io/en/stable/language/commands.html .. _How to use Kestrel via API: https://kestrel.readthedocs.io/en/stable/source/kestrel.session.html +.. _How to launch Kestrel as a Docker container: https://kestrel.readthedocs.io/en/stable/deployment/ .. _Kestrel documentation on talks: https://kestrel.readthedocs.io/en/stable/talks.html .. _Kestrel huntbook: https://github.com/opencybersecurityalliance/kestrel-huntbook diff --git a/containers/docker/Dockerfile b/containers/docker/Dockerfile new file mode 100644 index 00000000..4eadf301 --- /dev/null +++ b/containers/docker/Dockerfile @@ -0,0 +1,105 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/base-notebook +FROM $BASE_CONTAINER + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for fully functional notebook server +# added curl and gnupg for analytics +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + # Common useful utilities + git \ + curl \ + gnupg\ + nano-tiny \ + tzdata \ + unzip \ + vim-tiny \ + # git-over-ssh + openssh-client \ + # less is needed to run help in R + # see: https://github.com/jupyter/docker-stacks/issues/1588 + less \ + # nbconvert dependencies + # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex + texlive-xetex \ + texlive-fonts-recommended \ + texlive-plain-generic \ + # Enable clipboard on Linux host systems + xclip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# analytics prereq +RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash + +RUN sudo apt-get install --yes git-lfs + +# Install kestrel-LANG +RUN pip install kestrel-lang + +# Create alternative for nano -> nano-tiny +RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10 + +RUN mkdir -p /opt/kaas && chown ${NB_UID}:${NB_UID} /opt/kaas + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} + +# Install kestrel-JUPYTER +RUN pip install kestrel-jupyter + +# setup Jupyter +RUN python -m kestrel_jupyter_kernel.setup + +RUN mv "${HOME}/.local/share/jupyter/kernels/kestrel"* "${CONDA_DIR}/share/jupyter/kernels/" && \ + chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \ + rm -rf "${HOME}/.local" + +# Add R mimetype option to specify how the plot returns from R to the browser +RUN curl https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Rprofile.site -o ${HOME}/Rprofile.site +RUN chown ${NB_UID}:${NB_GID} ${HOME}/Rprofile.site && mkdir -p /opt/conda/lib/R/etc && mv Rprofile.site /opt/conda/lib/R/etc/Rprofile.site +#COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/ + +# Install kestrel-ANALYTICS +RUN git clone https://github.com/opencybersecurityalliance/data-bucket-kestrel.git /opt/kaas/data-bucket-kestrel + +RUN mkdir -p /opt/kaas/analytic-tutorials && \ + mkdir -p /opt/kaas/huntbooks && \ + mkdir -p /opt/kaas/tutorial && \ + mkdir -p /opt/kaas/blackhat22 + +# for generic huntbooks like BH22 +RUN cp -r /opt/kaas/data-bucket-kestrel/stix-bundles /opt/kaas/analytic-tutorials + +# for the tutorial +RUN cp /opt/kaas/data-bucket-kestrel/stix-bundles/lab101.json /opt/kaas/analytic-tutorials + +RUN git clone https://github.com/opencybersecurityalliance/kestrel-analytics.git /opt/kaas/kestrel-analytics + +RUN cp /opt/kaas/data-bucket-kestrel/GeoLite2/GeoLite2-City.mmdb /opt/kaas/kestrel-analytics/analytics/piniponmap + +RUN rm -rf "/opt/kaas/data-bucket-kestrel" + +# install kestrel-HUNTBOOK +RUN git clone https://github.com/opencybersecurityalliance/kestrel-huntbook.git /opt/kaas/kestrel-huntbook + +RUN mkdir -p /home/jovyan/.config/kestrel && \ + mv /opt/kaas/kestrel-huntbook/config/stixshifter.yaml /home/jovyan/.config/kestrel/ && \ + ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/huntbooks/stixshifter.yaml && \ + ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/tutorial/stixshifter.yaml && \ + ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/blackhat22/stixshifter.yaml + +RUN rmdir /opt/kaas/kestrel-huntbook/config + +RUN cp /opt/kaas/kestrel-analytics/pythonanalytics_sample.yaml /home/jovyan/.config/kestrel/pythonanalytics.yaml + +#RUN jupyter trust */*.ipynb \ No newline at end of file diff --git a/containers/docker/Rprofile.site b/containers/docker/Rprofile.site new file mode 100644 index 00000000..3d6a93cc --- /dev/null +++ b/containers/docker/Rprofile.site @@ -0,0 +1,4 @@ +# Add R mimetype to specify how the plot returns from R to the browser. +# https://notebook.community/andrie/jupyter-notebook-samples/Changing%20R%20plot%20options%20in%20Jupyter + +options(jupyter.plot_mimetypes = c('text/plain', 'image/png', 'image/jpeg', 'image/svg+xml', 'application/pdf')) diff --git a/containers/oci/README.rst b/containers/oci/README.rst new file mode 100644 index 00000000..d584cb45 --- /dev/null +++ b/containers/oci/README.rst @@ -0,0 +1 @@ +This is a placeholder for any future oci formats diff --git a/docs/deployment/dockerhub.rst b/docs/deployment/dockerhub.rst new file mode 100644 index 00000000..e317832a --- /dev/null +++ b/docs/deployment/dockerhub.rst @@ -0,0 +1,26 @@ +Besides Python package (PyPI), Kestrel is also released into Docker container +image on DockerHub. + +The image provides a full Kestrel runtime composed of the basic Kestrel +runtime, `kestrel-jupyter`_ package, open-source Kestrel analytics in the +`kestrel-analytics repo`_, and open-source Kestrel huntbooks and tutorials in +the `kestrel-huntbook repo`_. + +The image is based on the `docker-stacks`_ Jupyter image, maintained by +`Kenneth Peeples`_, and currently located under `Kenneth's DockerHub account`_. + +To launch the Kestrel container (opening Jupyter on host port 8888): + +.. code-block:: console + + $ docker run -d -p 8888:8888 kpeeples/kaas-baseline:latest + +To have Kestrel syntax highlighting support, use the Jupyter Notebook URL (``http://hostname:8888/tree``) instead of Jupyter Lab (``http://hostname:8888/lab``) for Kestrel huntbooks. + + +.. _kestrel-jupyter: https://github.com/opencybersecurityalliance/kestrel-jupyter +.. _kestrel-analytics repo: https://github.com/opencybersecurityalliance/kestrel-analytics +.. _kestrel-huntbook repo: https://github.com/opencybersecurityalliance/kestrel-huntbook +.. _docker-stacks: https://github.com/jupyter/docker-stacks +.. _Kenneth Peeples: https://github.com/kpeeples +.. _Kenneth's DockerHub account: https://hub.docker.com/repository/docker/kpeeples/kaas-baseline diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst new file mode 100644 index 00000000..8b3ef4cf --- /dev/null +++ b/docs/deployment/index.rst @@ -0,0 +1,13 @@ +==================== +Container Deployment +==================== + +Docker (at Dockerhub) +===================== + +.. include:: dockerhub.rst + +OCI +=== + +.. include:: oci.rst diff --git a/docs/deployment/oci.rst b/docs/deployment/oci.rst new file mode 100644 index 00000000..9ac89af6 --- /dev/null +++ b/docs/deployment/oci.rst @@ -0,0 +1,3 @@ +Placeholder for future `Open Container Initiative`_ (OCI) + +.. _Open Container Initiative: https://opencontainers.org/ diff --git a/docs/index.rst b/docs/index.rst index ce4deb35..57fe4142 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ talks, community entrances, and more. configuration debug runtime + deployment/index theory talks contributing diff --git a/setup.cfg b/setup.cfg index c5031d97..5c02bd94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kestrel-lang -version = 1.7.4 +version = 1.7.5 description = Kestrel Threat Hunting Language long_description = file:README.rst long_description_content_type = text/x-rst @@ -29,18 +29,18 @@ scripts = bin/stix-shifter-diag python_requires = >= 3.8 install_requires = - typeguard>=4.0.0 - pyyaml>=6.0 + typeguard>=4.1.3 + pyyaml>=6.0.1 lxml>=4.9.3 - lark>=1.1.5 + lark>=1.1.7 pandas>=2.0.0 - pyarrow>=5.0.0 - docker>=5.0.0 + pyarrow>=13.0.0 + docker>=6.1.3 requests>=2.31.0 - nest-asyncio>=1.5.6 - stix-shifter==6.0.3 - stix-shifter-utils==6.0.3 - firepit>=2.3.25 + nest-asyncio>=1.5.7 + stix-shifter==6.2.1 + stix-shifter-utils==6.2.1 + firepit>=2.3.27 tests_require = pytest