Merge pull request #398 from opencybersecurityalliance/develop

v1.7.5

.github/workflows/integration-testing.yml

@@ -7,13 +7,15 @@ on:
       - release
     paths:
       - 'src/**'
+      - 'setup.cfg'
   pull_request:
     branches:
       - develop
       - develop_*
       - release
     paths:
       - 'src/**'
+      - 'setup.cfg'
     types:
       - opened
       - synchronize

.github/workflows/kaas-docker-image.yml

@@ -0,0 +1,30 @@
+name: Build Docker image
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build & push docker image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+
+      - name: Info
+        run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}"
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: "{{defaultContext}}:containers/docker"
+          push: true
+          tags: kpeeples/kaas-baseline:latest

.github/workflows/unit-testing.yml

@@ -9,6 +9,7 @@ on:
       - 'src/**'
       - 'bin/**'
       - 'tests/**'
+      - 'setup.cfg'
   pull_request:
     branches:
       - develop
@@ -17,6 +18,7 @@ on:
       - 'src/**'
       - 'bin/**'
       - 'tests/**'
+      - 'setup.cfg'
     types:
       - opened
       - reopened

AUTHORS.rst

@@ -19,6 +19,7 @@ Contributors
 - `Constantin Adam`_
 - `Ting Dai`_
 - `Leila Rashidi`_
+- `Kenneth Peeples`_
 
 .. _Xiaokui Shu: https://github.com/subbyte
 .. _Paul Coccoli: https://github.com/pcoccoli
@@ -30,3 +31,4 @@ Contributors
 .. _Constantin Adam: https://github.com/cmadam
 .. _Ting Dai: https://github.com/tingdai
 .. _Leila Rashidi: https://github.com/leila-rashidi
+.. _Kenneth Peeples: https://github.com/kpeeples

CHANGELOG.rst

@@ -9,6 +9,16 @@ The format is based on `Keep a Changelog`_.
 Unreleased
 ==========
 
+1.7.5 (2023-09-07)
+==================
+
+Added
+-----
+
+- Kestrel Docker container image in DockerHub
+- Documentation on how to use Kestrel Docker container
+- case insensitive option ``(?i)`` for Elasticserach via stix-shifter data source interface (stix-shifter v6.2.1)
+
 1.7.4 (2023-08-03)
 ==================
 

README.rst

@@ -109,6 +109,7 @@ Visit `Kestrel documentation`_ to learn Kestrel:
   - `How to connect to your data sources`_
   - `How to execute an analytic hunt step in Python/Docker`_
   - `How to use Kestrel via API`_
+  - `How to launch Kestrel as a Docker container`_
 
 Kestrel Huntbooks And Analytics
 ===============================
@@ -180,6 +181,7 @@ Connecting With The Community
 .. _How to execute an analytic hunt step in Python/Docker: https://kestrel.readthedocs.io/en/stable/installation/analytics.html
 .. _Language reference book: https://kestrel.readthedocs.io/en/stable/language/commands.html
 .. _How to use Kestrel via API: https://kestrel.readthedocs.io/en/stable/source/kestrel.session.html
+.. _How to launch Kestrel as a Docker container: https://kestrel.readthedocs.io/en/stable/deployment/
 .. _Kestrel documentation on talks: https://kestrel.readthedocs.io/en/stable/talks.html
 
 .. _Kestrel huntbook: https://github.com/opencybersecurityalliance/kestrel-huntbook

containers/docker/Dockerfile

@@ -0,0 +1,105 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+ARG OWNER=jupyter
+ARG BASE_CONTAINER=$OWNER/base-notebook
+FROM $BASE_CONTAINER
+
+LABEL maintainer="Jupyter Project <[email protected]>"
+
+# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
+# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+# Install all OS dependencies for fully functional notebook server
+# added curl and gnupg for analytics
+RUN apt-get update --yes && \
+    apt-get install --yes --no-install-recommends \
+    # Common useful utilities
+    git \
+    curl \
+    gnupg\
+    nano-tiny \
+    tzdata \
+    unzip \
+    vim-tiny \
+    # git-over-ssh
+    openssh-client \
+    # less is needed to run help in R
+    # see: https://github.com/jupyter/docker-stacks/issues/1588
+    less \
+    # nbconvert dependencies
+    # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex
+    texlive-xetex \
+    texlive-fonts-recommended \
+    texlive-plain-generic \
+    # Enable clipboard on Linux host systems
+    xclip && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# analytics prereq
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
+
+RUN sudo apt-get install --yes git-lfs
+
+# Install kestrel-LANG
+RUN pip install kestrel-lang
+
+# Create alternative for nano -> nano-tiny
+RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10
+
+RUN mkdir -p /opt/kaas && chown ${NB_UID}:${NB_UID} /opt/kaas
+
+# Switch back to jovyan to avoid accidental container runs as root
+USER ${NB_UID}
+
+# Install kestrel-JUPYTER
+RUN pip install kestrel-jupyter
+
+# setup Jupyter
+RUN python -m kestrel_jupyter_kernel.setup
+
+RUN mv "${HOME}/.local/share/jupyter/kernels/kestrel"* "${CONDA_DIR}/share/jupyter/kernels/" && \
+    chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \
+    rm -rf "${HOME}/.local"
+
+# Add R mimetype option to specify how the plot returns from R to the browser
+RUN curl https://raw.githubusercontent.com/kpeeples/kestrel-as-a-service/main/dockerhub/Rprofile.site -o ${HOME}/Rprofile.site
+RUN chown ${NB_UID}:${NB_GID} ${HOME}/Rprofile.site && mkdir -p /opt/conda/lib/R/etc && mv Rprofile.site /opt/conda/lib/R/etc/Rprofile.site
+#COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/
+
+# Install kestrel-ANALYTICS
+RUN git clone https://github.com/opencybersecurityalliance/data-bucket-kestrel.git /opt/kaas/data-bucket-kestrel
+
+RUN mkdir -p /opt/kaas/analytic-tutorials && \
+    mkdir -p /opt/kaas/huntbooks && \
+    mkdir -p /opt/kaas/tutorial && \
+    mkdir -p /opt/kaas/blackhat22
+
+# for generic huntbooks like BH22
+RUN cp -r /opt/kaas/data-bucket-kestrel/stix-bundles /opt/kaas/analytic-tutorials
+
+# for the tutorial
+RUN cp /opt/kaas/data-bucket-kestrel/stix-bundles/lab101.json /opt/kaas/analytic-tutorials
+
+RUN git clone https://github.com/opencybersecurityalliance/kestrel-analytics.git /opt/kaas/kestrel-analytics
+
+RUN cp /opt/kaas/data-bucket-kestrel/GeoLite2/GeoLite2-City.mmdb /opt/kaas/kestrel-analytics/analytics/piniponmap
+
+RUN rm -rf "/opt/kaas/data-bucket-kestrel"
+
+# install kestrel-HUNTBOOK
+RUN git clone https://github.com/opencybersecurityalliance/kestrel-huntbook.git /opt/kaas/kestrel-huntbook
+
+RUN mkdir -p /home/jovyan/.config/kestrel && \
+    mv /opt/kaas/kestrel-huntbook/config/stixshifter.yaml /home/jovyan/.config/kestrel/ && \
+    ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/huntbooks/stixshifter.yaml && \
+    ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/tutorial/stixshifter.yaml && \
+    ln -s /home/jovyan/.config/kestrel/stixshifter.yaml /opt/kaas/kestrel-huntbook/blackhat22/stixshifter.yaml
+
+RUN rmdir /opt/kaas/kestrel-huntbook/config
+
+RUN cp /opt/kaas/kestrel-analytics/pythonanalytics_sample.yaml /home/jovyan/.config/kestrel/pythonanalytics.yaml
+
+#RUN jupyter trust */*.ipynb

containers/docker/Rprofile.site

@@ -0,0 +1,4 @@
+# Add R mimetype to specify how the plot returns from R to the browser.
+# https://notebook.community/andrie/jupyter-notebook-samples/Changing%20R%20plot%20options%20in%20Jupyter
+
+options(jupyter.plot_mimetypes = c('text/plain', 'image/png', 'image/jpeg', 'image/svg+xml', 'application/pdf'))

containers/oci/README.rst

@@ -0,0 +1 @@
+This is a placeholder for any future oci formats

docs/deployment/dockerhub.rst

@@ -0,0 +1,26 @@
+Besides Python package (PyPI), Kestrel is also released into Docker container
+image on DockerHub.
+
+The image provides a full Kestrel runtime composed of the basic Kestrel
+runtime, `kestrel-jupyter`_ package, open-source Kestrel analytics in the
+`kestrel-analytics repo`_, and open-source Kestrel huntbooks and tutorials in
+the `kestrel-huntbook repo`_.
+
+The image is based on the `docker-stacks`_ Jupyter image, maintained by
+`Kenneth Peeples`_, and currently located under `Kenneth's DockerHub account`_.
+
+To launch the Kestrel container (opening Jupyter on host port 8888):
+
+.. code-block:: console
+
+    $ docker run -d -p 8888:8888 kpeeples/kaas-baseline:latest
+
+To have Kestrel syntax highlighting support, use the Jupyter Notebook URL (``http://hostname:8888/tree``) instead of Jupyter Lab (``http://hostname:8888/lab``) for Kestrel huntbooks.
+
+
+.. _kestrel-jupyter: https://github.com/opencybersecurityalliance/kestrel-jupyter
+.. _kestrel-analytics repo: https://github.com/opencybersecurityalliance/kestrel-analytics
+.. _kestrel-huntbook repo: https://github.com/opencybersecurityalliance/kestrel-huntbook
+.. _docker-stacks: https://github.com/jupyter/docker-stacks
+.. _Kenneth Peeples: https://github.com/kpeeples
+.. _Kenneth's DockerHub account: https://hub.docker.com/repository/docker/kpeeples/kaas-baseline

docs/deployment/index.rst

@@ -0,0 +1,13 @@
+====================
+Container Deployment
+====================
+
+Docker (at Dockerhub)
+=====================
+
+.. include:: dockerhub.rst
+
+OCI
+===
+
+.. include:: oci.rst

docs/deployment/oci.rst

@@ -0,0 +1,3 @@
+Placeholder for future `Open Container Initiative`_ (OCI)
+
+.. _Open Container Initiative: https://opencontainers.org/

docs/index.rst

@@ -23,6 +23,7 @@ talks, community entrances, and more.
    configuration
    debug
    runtime
+   deployment/index
    theory
    talks
    contributing

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kestrel-lang
-version = 1.7.4
+version = 1.7.5
 description = Kestrel Threat Hunting Language
 long_description = file:README.rst
 long_description_content_type = text/x-rst
@@ -29,18 +29,18 @@ scripts =
     bin/stix-shifter-diag
 python_requires = >= 3.8
 install_requires =
-    typeguard>=4.0.0
-    pyyaml>=6.0
+    typeguard>=4.1.3
+    pyyaml>=6.0.1
     lxml>=4.9.3
-    lark>=1.1.5
+    lark>=1.1.7
     pandas>=2.0.0
-    pyarrow>=5.0.0
-    docker>=5.0.0
+    pyarrow>=13.0.0
+    docker>=6.1.3
     requests>=2.31.0
-    nest-asyncio>=1.5.6
-    stix-shifter==6.0.3
-    stix-shifter-utils==6.0.3
-    firepit>=2.3.25
+    nest-asyncio>=1.5.7
+    stix-shifter==6.2.1
+    stix-shifter-utils==6.2.1
+    firepit>=2.3.27
 tests_require =
     pytest