diff --git a/.ci/01_before_install.sh b/.ci/01_before_install.sh index 79d0c3ac1..8c3b7d89a 100755 --- a/.ci/01_before_install.sh +++ b/.ci/01_before_install.sh @@ -1,17 +1,18 @@ #!/bin/sh +set -e STARTMSG="[before_install]" # Install Requirements echo echo "$STARTMSG Install requirements..." - [ ! -z "$(which apk)" ] && apk add --no-cache make bash sudo git curl coreutils grep python3 + [ ! -z "$(which apk)" ] && apk add --no-cache bash sudo git curl coreutils grep py-pip python-dev libffi-dev openssl-dev gcc libc-dev make [ ! -z "$(which apt-get)" ] && apt-get update; - [ ! -z "$(which apt-get)" ] && apt-get install make bash sudo git curl coreutils grep python3 + [ ! -z "$(which apt-get)" ] && apt-get install make bash sudo git curl coreutils grep python3 gcc # Upgrade Docke [ ! -z "$(which apt-get)" ] && apt-get install --only-upgrade docker-ce -y # Install docker-compose # https://stackoverflow.com/questions/42295457/using-docker-compose-in-a-gitlab-ci-pipeline - [ -z "$(which docker-compose)" ] && pip3 install --no-cache-dir docker-compose + [ -z "$(which docker-compose)" ] && pip install docker-compose # Show version of docker-compose: docker-compose -v diff --git a/.ci/02_script.sh b/.ci/02_script.sh index 93a7d90e6..c27f5ce31 100755 --- a/.ci/02_script.sh +++ b/.ci/02_script.sh @@ -1,4 +1,6 @@ #!/bin/sh +set -e + STARTMSG="[02_script]" # change directory for make usage @@ -31,9 +33,9 @@ echo "$DOCKER_LOGIN_OUTPUT" # Build config and deploy environent - echo "$STARTMSG build configuration..." && $makefile_main build-config REPOURL=$REGISTRY_URL - echo "$STARTMSG pull images..." && docker-compose -f current/docker-compose.yml -f current/docker-compose.override.yml pull - echo "$STARTMSG start environment..." && docker-compose -f current/docker-compose.yml -f current/docker-compose.override.yml up -d + echo "$STARTMSG Build Configuration..." && $makefile_main build-config REPOURL="$REGISTRY_URL" + echo "$STARTMSG Pull Images..." && docker-compose -f current/docker-compose.yml -f current/docker-compose.override.yml pull + echo "$STARTMSG Start Environment..." && docker-compose -f current/docker-compose.yml -f current/docker-compose.override.yml up -d ########################################################### # ATTENTION ATTENTION ATTENTION # If you want to use docker-in-docker (dind) you cant start docker container on another filesystem!!!! You need to do it from the docker-compose directly!!! @@ -55,6 +57,7 @@ then echo "$STARTMSG show running docker container..." && docker ps fi +$makefile_main configure # show config folders ls -laR config/ diff --git a/.ci/10_scanning.sh b/.ci/10_scanning.sh index 3d9a3a610..edaf7fabd 100755 --- a/.ci/10_scanning.sh +++ b/.ci/10_scanning.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e diff --git a/.ci/20_tag_2_registry.sh b/.ci/20_tag_2_registry.sh new file mode 100755 index 000000000..1a8682b2d --- /dev/null +++ b/.ci/20_tag_2_registry.sh @@ -0,0 +1,81 @@ +#!/bin/sh +set -e + +STARTMSG="[tag]" + +# first_version=5.100.2 +# second_version=5.1.2 +# if version_gt $first_version $second_version; then +# echo "$first_version is greater than $second_version !" +# fi' +version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + +func_tag() { + DOCKER_REPO="$1" + TAG="$2" + + # add -dev + [ -z "$(echo "$TAG"| grep dev)" ] && TAG="$TAG-dev" + IMAGE_ID="$(docker images --format "{{.Repository}}:{{.Tag}}:{{.ID}}"|grep "$DOCKER_REPO:$TAG"|cut -d : -f 3|head -n 1;)" + IMAGE_TAGS="$(docker images --format "{{.Repository}}:{{.Tag}}:{{.ID}}"|grep "$IMAGE_ID"|cut -d : -f 2;)" + for i in $IMAGE_TAGS + do + k="$(echo "$i"|sed 's,-dev$,,')" + echo "$STARTMSG Retag: $DOCKER_REPO:$i with $DOCKER_REPO:$k" + docker tag "$DOCKER_REPO:$i" "$DOCKER_REPO:$k" + echo "$STARTMSG Remove: $DOCKER_REPO:$i" + docker image rm "$DOCKER_REPO:$i" + done +} + + +# change directory for make usage +[ -z "$1" ] && echo "$STARTMSG No parameter with the Docker registry URL. Exit now." && exit 1 +[ "$1" = "NOT2PUSH" ] && echo "$STARTMSG The NOT2PUSH slug is only for local build and retag not for pushin to docker registries. Exit now." && exit 1 +[ -z "$2" ] && echo "$STARTMSG No parameter with the Docker registry username. Exit now." && exit 1 +[ -z "$3" ] && echo "$STARTMSG No parameter with the Docker registry password. Exit now." && exit 1 + +REGISTRY_URL="$1" +REGISTRY_USER="$2" +REGISTRY_PW="$3" + + +# Pull all latest tagged container + echo + echo "$STARTMSG Pull all latest-dev container..." + make pull-latest REPOURL="$REGISTRY_URL" + + +# prepare retagging +SERVER_TAG="$(docker ps -f name=server --format '{{.Image}}'|cut -d : -f 2)" +PROXY_TAG="$(docker ps -f name=proxy --format '{{.Image}}'|cut -d : -f 2)" +ROBOT_TAG="$(docker ps -f name=robot --format '{{.Image}}'|cut -d : -f 2)" +MODULES_TAG="$(docker ps -f name=modules --format '{{.Image}}'|cut -d : -f 2)" +DB_TAG=$(docker ps -f name=db --format '{{.Image}}'|cut -d : -f 2) +REDIS_TAG=$(docker ps -f name=redis --format '{{.Image}}'|cut -d : -f 2) + + +# Login to Docker registry +[ "$REGISTRY_URL" != "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" "$REGISTRY_URL" --password-stdin)" +[ "$REGISTRY_URL" = "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" --password-stdin)" +echo "$DOCKER_LOGIN_OUTPUT" +DOCKER_LOGIN_STATE="$(echo "$DOCKER_LOGIN_OUTPUT" | grep 'Login Succeeded')" + +if [ ! -z "$DOCKER_LOGIN_STATE" ]; then + # retag all existing tags dev 2 public repo + #$makefile_travis tag REPOURL=$REGISTRY_URL server_tag=${server_tag} proxy_tag=${proxy_tag} robot_tag=${robot_tag} modules_tag=${modules_tag} db_tag=${modules_tag} redis_tag=${modules_tag} postfix_tag=${postfix_tag} + func_tag "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAG" + func_tag "$REGISTRY_URL/misp-dockerized-proxy" "$PROXY_TAG" + func_tag "$REGISTRY_URL/misp-dockerized-robot" "$ROBOT_TAG" + func_tag "$REGISTRY_URL/misp-dockerized-misp-modules" "$MODULES_TAG" + if $version_gt "$VERSION" 1.1.0 ; then + func_tag "$REGISTRY_URL/misp-dockerized-redis" "$REDIS_TAG" + fi + #func_tag "$REGISTRY_URL/misp-dockerized-db" "$DB_TAG" + echo "###########################################" && docker images && echo "###########################################" +else + echo "$DOCKER_LOGIN_OUTPUT" + exit +fi + +echo "$STARTMSG $0 is finished." diff --git a/.ci/30_push_2_registry.sh b/.ci/30_push_2_registry.sh index 0ff0596f5..113280764 100755 --- a/.ci/30_push_2_registry.sh +++ b/.ci/30_push_2_registry.sh @@ -1,38 +1,26 @@ #!/bin/sh +set -e + STARTMSG="[push]" -func_push() { - DOCKER_REPO="$1" - TAG="$2" - TAG="$(echo "$TAG"|sed 's,-dev$,,')" +# first_version=5.100.2 +# second_version=5.1.2 +# if version_gt $first_version $second_version; then +# echo "$first_version is greater than $second_version !" +# fi' +version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } - IMAGE_ID="$(docker images --format "{{.Repository}}:{{.Tag}}:{{.ID}}"|grep "$DOCKER_REPO:$TAG"|cut -d : -f 3|head -n 1;)" - IMAGE_TAGS="$(docker images --format "{{.Repository}}:{{.Tag}}:{{.ID}}"|grep "$IMAGE_ID"|cut -d : -f 2;)" - for i in $IMAGE_TAGS - do - docker push "$DOCKER_REPO:$i" - done -} -func_tag() { +func_push() { DOCKER_REPO="$1" - TAG="$2" - - # add -dev - [ -z "$(echo "$TAG"| grep dev)" ] && TAG="$TAG-dev" - IMAGE_ID="$(docker images --format "{{.Repository}}:{{.Tag}}:{{.ID}}"|grep "$DOCKER_REPO:$TAG"|cut -d : -f 3|head -n 1;)" - IMAGE_TAGS="$(docker images --format "{{.Repository}}:{{.Tag}}:{{.ID}}"|grep "$IMAGE_ID"|cut -d : -f 2;)" - for i in $IMAGE_TAGS + TAGS="$2" + for i in $TAGS do - k="$(echo "$i"|sed 's,-dev$,,')" - echo "$STARTMSG Retag: $DOCKER_REPO:$i with $DOCKER_REPO:$k" - docker tag "$DOCKER_REPO:$i" "$DOCKER_REPO:$k" - echo "$STARTMSG Remove: $DOCKER_REPO:$i" - docker image rm "$DOCKER_REPO:$i" + [ ! -z "$(echo $i | grep 'dev')" ] && continue + docker push "$DOCKER_REPO:$i" done } - # change directory for make usage [ -z "$1" ] && echo "$STARTMSG No parameter with the Docker registry URL. Exit now." && exit 1 [ "$1" = "NOT2PUSH" ] && echo "$STARTMSG The NOT2PUSH slug is only for local build and retag not for pushin to docker registries. Exit now." && exit 1 @@ -43,20 +31,13 @@ REGISTRY_URL="$1" REGISTRY_USER="$2" REGISTRY_PW="$3" +SERVER_TAGS="$(docker images --no-trunc --format '{{.Tag}}={{.ID}}' | grep $(docker inspect misp-server -f '{{.Image}}')|cut -d = -f 1)" +PROXY_TAGS="$(docker images --no-trunc --format '{{.Tag}}={{.ID}}' | grep $(docker inspect misp-proxy -f '{{.Image}}')|cut -d = -f 1)" +ROBOT_TAGS="$(docker images --no-trunc --format '{{.Tag}}={{.ID}}' | grep $(docker inspect misp-robot -f '{{.Image}}')|cut -d = -f 1)" +MODULES_TAGS="$(docker images --no-trunc --format '{{.Tag}}={{.ID}}' | grep $(docker inspect misp-modules -f '{{.Image}}')|cut -d = -f 1)" +#DB_TAGS=$(docker ps -f name=db --format '{{.Image}}'|cut -d : -f 2) +REDIS_TAGS="$(docker images --no-trunc --format '{{.Tag}}={{.ID}}' | grep $(docker inspect misp-redis -f '{{.Image}}')|cut -d = -f 1)" -# Pull all latest tagged container - echo - echo "$STARTMSG Pull all latest-dev container..." - make pull-latest REPOURL="$REGISTRY_URL" - - -# prepare retagging -SERVER_TAG="$(docker ps -f name=server --format '{{.Image}}'|cut -d : -f 2)" -PROXY_TAG="$(docker ps -f name=proxy --format '{{.Image}}'|cut -d : -f 2)" -ROBOT_TAG="$(docker ps -f name=robot --format '{{.Image}}'|cut -d : -f 2)" -MODULES_TAG="$(docker ps -f name=modules --format '{{.Image}}'|cut -d : -f 2)" -DB_TAG=$(docker ps -f name=db --format '{{.Image}}'|cut -d : -f 2) -REDIS_TAG=$(docker ps -f name=redis --format '{{.Image}}'|cut -d : -f 2) # Login to Docker registry @@ -66,23 +47,25 @@ echo "$DOCKER_LOGIN_OUTPUT" DOCKER_LOGIN_STATE="$(echo "$DOCKER_LOGIN_OUTPUT" | grep 'Login Succeeded')" if [ ! -z "$DOCKER_LOGIN_STATE" ]; then - # retag all existing tags dev 2 public repo - #$makefile_travis tag REPOURL=$REGISTRY_URL server_tag=${server_tag} proxy_tag=${proxy_tag} robot_tag=${robot_tag} modules_tag=${modules_tag} db_tag=${modules_tag} redis_tag=${modules_tag} postfix_tag=${postfix_tag} - func_tag "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAG" - func_tag "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAG" - func_tag "$REGISTRY_URL/misp-dockerized-robot" "$ROBOT_TAG" - func_tag "$REGISTRY_URL/misp-dockerized-misp-modules" "$MODULES_TAG" - #func_tag "$REGISTRY_URL/misp-dockerized-db" "$DB_TAG" - func_tag "$REGISTRY_URL/misp-dockerized-redis" "$REDIS_TAG" - echo "###########################################" && docker images && echo "###########################################" +# # retag all existing tags dev 2 public repo +# #$makefile_travis tag REPOURL=$REGISTRY_URL server_tag=${server_tag} proxy_tag=${proxy_tag} robot_tag=${robot_tag} modules_tag=${modules_tag} db_tag=${modules_tag} redis_tag=${modules_tag} postfix_tag=${postfix_tag} +# func_tag "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAG" +# func_tag "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAG" +# func_tag "$REGISTRY_URL/misp-dockerized-robot" "$ROBOT_TAG" +# func_tag "$REGISTRY_URL/misp-dockerized-misp-modules" "$MODULES_TAG" +# #func_tag "$REGISTRY_URL/misp-dockerized-db" "$DB_TAG" +# func_tag "$REGISTRY_URL/misp-dockerized-redis" "$REDIS_TAG" +# echo "###########################################" && docker images && echo "###########################################" # Push all Docker images #$makefile_travis push REPOURL=$REGISTRY_URL server_tag=${server_tag} proxy_tag=${proxy_tag} robot_tag=${robot_tag} modules_tag=${modules_tag} postfix_tag=${postfix_tag} - func_push "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAG" - func_push "$REGISTRY_URL/misp-dockerized-proxy" "$PROXY_TAG" - func_push "$REGISTRY_URL/misp-dockerized-robot" "$ROBOT_TAG" - func_push "$REGISTRY_URL/misp-dockerized-misp-modules" "$MODULES_TAG" - #func_push "$REGISTRY_URL/misp-dockerized-db" "$DB_TAG" - func_push "$REGISTRY_URL/misp-dockerized-redis" "$REDIS_TAG" + func_push "$REGISTRY_URL/misp-dockerized-server" "$SERVER_TAGS" + func_push "$REGISTRY_URL/misp-dockerized-proxy" "$PROXY_TAGS" + func_push "$REGISTRY_URL/misp-dockerized-robot" "$ROBOT_TAGS" + func_push "$REGISTRY_URL/misp-dockerized-misp-modules" "$MODULES_TAGS" + if $version_gt "$VERSION" 1.1.0 ; then + func_push "$REGISTRY_URL/misp-dockerized-redis" "$REDIS_TAGS" + fi + #func_push "$REGISTRY_URL/misp-dockerized-db" "$DB_TAGS" else echo "$DOCKER_LOGIN_OUTPUT" exit diff --git a/.ci/Makefile b/.ci/Makefile index db4b5ce27..27526ecf8 100644 --- a/.ci/Makefile +++ b/.ci/Makefile @@ -24,6 +24,9 @@ test: @if [ ! -d reports ]; then mkdir reports; fi @docker cp misp-robot:/srv/MISP-dockerized-testbench/reports/. reports/ +tag: + @bash 20_tag_2_registry.sh $(REPOURL) $(USER) $(PW) + push: @bash 30_push_2_registry.sh $(REPOURL) $(USER) $(PW) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0e577a16..41b97ab68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ services: stages: - build +- test - scan - deploy @@ -26,16 +27,19 @@ before_script: # Install dependencies - .ci/01_before_install.sh + + # On all other pull requests and branches exclude dev and master: Build Environment, but do nothing more. No tests. Tests with our Test Framework are only done in the dev branch. .env_test_only: stage: build except: - master - dev - - 1.1.0 + - 1.2.0 script: + # Test - make -C .ci script REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW TEST=no_test VERSION=$CURRENT_VERSION - # Clean Up + # Clean Up - echo "##### clean up..." && make -C $FOLDER delete .env_test_only_hub.docker.com: @@ -43,22 +47,28 @@ before_script: except: - master - dev - - 1.1.0 + - 1.2.0 script: + # Test - make -C .ci script REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD TEST=no_test VERSION=$CURRENT_VERSION - # Clean Up + # Clean Up - echo "##### clean up..." && make -C $FOLDER delete + + # On dev Branch: Build Environment and Test it with our own Test Framework .long_test: - stage: build + stage: test only: - dev - - 1.1.0 + - 1.2.0 allow_failure: true script: + # Test - make -C .ci script REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW TEST=long_test VERSION=$CURRENT_VERSION - # Clean Up + # Tag + - make -C .ci tag REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW + # Clean Up - echo "##### clean up..." && make -C $FOLDER delete # https://docs.gitlab.com/ee/ci/yaml/README.html#artifacts-reports artifacts: @@ -69,13 +79,16 @@ before_script: # On dev Branch: Build Environment and Test it with our own Test Framework .long_test_hub.docker.com: - stage: build + stage: test only: - dev - - 1.1.0 + - 1.2.0 script: + # Test - make -C .ci script REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD TEST=long_test VERSION=$CURRENT_VERSION - # Clean Up + # Tag + - make -C .ci tag REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD + # Clean Up - echo "##### clean up..." && make -C $FOLDER delete # https://docs.gitlab.com/ee/ci/yaml/README.html#artifacts-reports artifacts: @@ -85,6 +98,7 @@ before_script: junit: .ci/reports/*.xml + # On MASTER Branch: Build Environment + retag Images to remove "-dev" tag .push_productive_images: stage: deploy @@ -95,6 +109,8 @@ before_script: # Check internal Registry # Test - make -C .ci script REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW TEST=long_test VERSION=$CURRENT_VERSION + # Tag + - make -C .ci tag REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW # Push - make -C .ci push REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW # Clean Up @@ -115,6 +131,8 @@ before_script: # Check hub.docker.com Registry # Test - make -C .ci script REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD TEST=long_test VERSION=$CURRENT_VERSION + # Tag + - make -C .ci tag REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD # Push - make -C .ci push REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD # Clean Up @@ -125,16 +143,10 @@ before_script: - ./.ci/reports/*.xml reports: junit: ./.ci/reports/*.xml - - - -# deactivated: -# https://docs.gitlab.com/ee/ci/examples/container_scanning.html -# https://docs.gitlab.com/ee/ci/yaml/README.html#artifactsreportscontainer_scanning -# https://docs.gitlab.com/ee/user/project/merge_requests/container_scanning.html -# https://docs.gitlab.com/ee/ci/yaml/#artifactsreportscontainer_scanning-ultimate + + + .container_scanning: - # deactivated: # https://docs.gitlab.com/ee/ci/examples/container_scanning.html # https://docs.gitlab.com/ee/ci/yaml/README.html#artifactsreportscontainer_scanning # https://docs.gitlab.com/ee/user/project/merge_requests/container_scanning.html @@ -165,5 +177,6 @@ include: #- '1.0.0/.gitlab-ci.yml' #- '1.0.1/.gitlab-ci.yml' #- '1.0.2/.gitlab-ci.yml' - #- '1.0.3/.gitlab-ci.yml' - - '1.1.0/.gitlab-ci.yml' + - '1.0.3/.gitlab-ci.yml' + #- '1.1.0/.gitlab-ci.yml' + - '1.1.1/.gitlab-ci.yml' diff --git a/.scripts/requirements.sh b/.scripts/requirements.sh index 201e8018c..28c57e245 100755 --- a/.scripts/requirements.sh +++ b/.scripts/requirements.sh @@ -149,7 +149,7 @@ if [ ! -f ./config/ssl/key.pem -a ! -f ./config/ssl/cert.pem ]; then echo " To change: " echo " 1. Please save your certificate in: $PWD/config/ssl/cert.pem" echo " 2. Please save your private keyfile in: $PWD/config/ssl/key.pem" - echo " 3. do: make config-ssl" + echo " 3. do: make -C current config-ssl" echo echo fi @@ -161,7 +161,7 @@ if [ ! -f ./config/smime/key.pem -a ! -f ./config/smime/cert.pem ]; then echo " To change: " echo " 1. Please save your certificate in: $PWD/config/smime/cert.pem" echo " 2. Please save your private key in: $PWD/config/smime/key.pem" - echo " 3. Do: make config-smime" + echo " 3. Do: make -C current config-smime" echo fi @@ -172,7 +172,7 @@ if [ ! -f ./config/pgp/private.key -a ! -f ./config/pgp/public.key ]; then echo " To change: " echo " 1. Please save your public key in: $PWD/config/pgp/public.key" echo " 2. Please save your private key in: $PWD/config/pgp/private.key" - echo " 3. Do: make config-pgp" + echo " 3. Do: make -C current config-pgp" echo echo fi diff --git a/.travis.yml b/.travis.yml index d7ce4836d..ab6739f7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,9 +26,11 @@ env: #- VERSION=0.3.4 #- VERSION=1.0.0 #- VERSION=1.0.1 - # - VERSION=1.0.2 + #- VERSION=1.0.2 - VERSION=1.0.3 - - VERSION=1.1.0 + #- VERSION=1.1.0 + - VERSION=1.1.1 + before_install: # Include Variables diff --git a/1.1.1/.gitlab-ci.yml b/1.1.1/.gitlab-ci.yml new file mode 100644 index 000000000..a30822d43 --- /dev/null +++ b/1.1.1/.gitlab-ci.yml @@ -0,0 +1,77 @@ +env_test_only 1.1.1: + variables: + CURRENT_VERSION: 1.1.1 + extends: .env_test_only + # only: + # changes: + # - 1.1.1/* + +env_test_only_hub.docker.com 1.1.1: + variables: + CURRENT_VERSION: 1.1.1 + extends: .env_test_only_hub.docker.com + # only: + # changes: + # - 1.1.1/* + + + +long_test 1.1.1: + variables: + CURRENT_VERSION: 1.1.1 + extends: .long_test + # only: + # changes: + # - 1.1.1/* + +long_test_hub.docker.com 1.1.1: + variables: + CURRENT_VERSION: 1.1.1 + extends: .long_test_hub.docker.com + # only: + # changes: + # - 1.1.1/* + + + +push_productive_images 1.1.1: + variables: + CURRENT_VERSION: 1.1.1 + extends: .push_productive_images + # only: + # changes: + # - 1.1.1/* + +push_productive_images_hub.docker.com 1.1.1: + variables: + CURRENT_VERSION: 1.1.1 + extends: .push_productive_images_hub.docker.com + # only: + # changes: + # - 1.1.1/* + + + +# scan proxy: +# variables: +# CI_CONTAINER_NAME: misp-dockerized-proxy +# #CI_APPLICATION_TAG: +# extends: .container_scanning + +# scan robot: +# variables: +# CI_CONTAINER_NAME: misp-dockerized-robot +# #CI_APPLICATION_TAG: +# extends: .container_scanning + +# scan server: +# variables: +# CI_CONTAINER_NAME: misp-dockerized-server +# #CI_APPLICATION_TAG: +# extends: .container_scanning + +# scan modules: +# variables: +# CI_CONTAINER_NAME: misp-dockerized-misp-modules +# #CI_APPLICATION_TAG: +# extends: .container_scanning \ No newline at end of file diff --git a/1.1.1/Makefile b/1.1.1/Makefile new file mode 100644 index 000000000..a88fb5c48 --- /dev/null +++ b/1.1.1/Makefile @@ -0,0 +1,218 @@ +.PHONY: help \ + start requirements build-config deploy delete change-ssl disable-maintenance enable-maintenance\ + security configure config-db config-server config-proxy \ + backup-all backup-server backup-redis backup-db backup-proxy backup-robot \ + build-server build-proxy build-robot build-all \ + +# Shows Help and all Commands +help: + @echo "Please use one of the following options:\n \ + General: \n \ + make install | Initial Command for: requirements, build-config, deploy\n \ + make requirements | Check if server fullfill all requirements\n \ + make deploy | Deploy Docker container\n \ + make upgrade | Upgrade MISP-dockerized\n \ + make update | Update MISP-dockerized same as make install\n \ + make delete | Delete all docker container, volumes and images for MISP\n \ + make delete-unused | Delete all unused docker container, volumes and images \n \ + make security | Check docker security via misp-robot\n \ + make test | Test if MISP-environment is ready.\n \ + \n \ + make build-config REPOURL= | Build configuration\n \ + \n \ + \n Control Docker Instances\n\ + make start-all | Start all docker container\n \ + make stop-all | Stop all docker container \n \ + make restart-all | Restart all docker container \n \ + \n Configure: \n \ + make change-ssl | Change ssl certificate and key\n \ + make change-smime | Change S/MIME certificate and key\n \ + make change-pgp | Change PGP keys\n \ + make change-all | Change SSL, S/MIME and PGP Keys \n \ + \n Maintenance: \n \ + make enable-maintenance | Enable maintenance mode \n \ + make disable-maintenance | Disable maintenance mode \n \ + \n Backup: \n \ + make backup-all | Backup all misp volumes via misp-robot\n \ + make backup-server | Backup misp-server volumes via misp-robot\n \ + make backup-redis | Backup misp-redis volumes via misp-robot\n \ + make backup-db | Backup misp-db volumes via misp-robot\n \ + make backup-proxy | Backup misp-proxy volumes via misp-robot\n \ + make backup-robot | Backup misp-robot volumes via misp-robot\n \ + \n \ + make restore-all | Restore all via misp-robot\n \ + make restore-server | Restore misp-server volumes via misp-robot\n \ + make restore-redis | Restore misp-redis volumes via misp-robot\n \ + make restore-db | Restore misp-db volumes via misp-robot\n \ + make restore-proxy | Restore misp-proxy volumes via misp-robot\n \ + make restore-robot | Restore misp-robot volumes via misp-robot\n \ + \n Help: \n \ + make help | Show help\n" + +# Start +install: requirements build-config pull deploy + @echo + @echo " ########### MISP environment is ready ###########" + @echo "Please go to: $(shell cat config/config.env|grep HOSTNAME|cut -d = -f 2)" + @echo "Login credentials:" + @echo " Username: admin@admin.test" + @echo " Password: admin" + @echo + @echo "Do not forget to change your SSL certificate with: make change-ssl" + @echo "Do not forget to change your S/MIME certificate with: make change-smime" + @echo "Do not forget to change your PGP certificate with: make change-pgp'" + @echo " ##########################################################" + @echo + +#################### used as host #################### +# Check requirements +requirements: + @echo " ########### Checking Requirements ###########" + @scripts/requirements.sh + @sleep 2 + +support: + make -C ../ support + +# Build Configuration +build-config: + @echo " ########### Build Configuration ###########" + @scripts/build_config.sh $(REPOURL) + @sleep 2 + +# Start Docker environment +pull: + @echo " ########### Pull Environment ###########" + docker run \ + --name misp-robot-init \ + --rm \ + --network="host" \ + -v $(CURDIR):/srv/MISP-dockerized \ + -v $(CURDIR)/scripts:/srv/scripts:ro \ + -v ~/.docker:/root/.docker:ro \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + $(shell cat $(CURDIR)/config/config.env|grep DOCKER_REGISTRY|cut -d = -f 2)/misp-dockerized-robot:$(shell cat $(CURDIR)/config/config.env|grep ROBOT_CONTAINER_TAG|cut -d = -f 2) bash -c "docker-compose -f /srv/MISP-dockerized/docker-compose.yml -f /srv/MISP-dockerized/docker-compose.override.yml pull " +deploy: + @echo " ########### Deploy Environment ###########" + docker run \ + --name misp-robot-init \ + --rm \ + --network="host" \ + -v $(CURDIR):/srv/MISP-dockerized \ + -v $(CURDIR)/scripts:/srv/scripts:ro \ + -v ~/.docker:/root/.docker:ro \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + $(shell cat $(CURDIR)/config/config.env|grep DOCKER_REGISTRY|cut -d = -f 2)/misp-dockerized-robot:$(shell cat $(CURDIR)/config/config.env|grep ROBOT_CONTAINER_TAG|cut -d = -f 2) bash -c "docker-compose -f /srv/MISP-dockerized/docker-compose.yml -f /srv/MISP-dockerized/docker-compose.override.yml up -d " + +# delete all misp container, volumes and images +delete: copy-ssl + scripts/delete_all_misp_from_host.sh + +# stop all misp docker container +stop-all: + docker stop misp-server + docker stop misp-proxy + docker stop misp-modules + docker stop misp-robot + +# start all misp docker container +start-all: + docker start misp-server + docker start misp-proxy + docker start misp-modules + docker start misp-robot + +# restart all misp docker container +restart-all: stop-all start-all + +# upgrade to a new version +upgrade: + @echo " ########### Upgrade MISP-dockerized to a new version ###########" + @cd ../ + @UPGRADE.sh + +# Update current MISP to all new functions in this Version without a new version +update: install + + +test: + @make -C ../.ci test + @echo + @echo "#############################################" + @echo "For the report output: cat $(CURDIR)/.travis/reports/*.xml" + @echo "#############################################" + + +#################### used in misp-robot #################### +DOCKER_EXEC=docker exec -ti +#DOCKER_EXEC= "docker run -it --rm " + +# configure +configure: + @echo " ########### Configure Environment ########### " + make copy-smime + make copy-pgp + make config-ssl + + +change-ssl: config-ssl +config-ssl: copy-ssl + docker restart misp-server + docker restart misp-proxy + +# x509 SSL Certificate +copy-ssl: + -docker cp $(CURDIR)/config/ssl/cert.pem misp-proxy:/etc/nginx/ssl/ + -docker cp $(CURDIR)/config/ssl/key.pem misp-proxy:/etc/nginx/ssl/ + -docker cp misp-proxy:/etc/nginx/ssl/cert.pem $(CURDIR)/config/ssl/ + -docker cp misp-proxy:/etc/nginx/ssl/key.pem $(CURDIR)/config/ssl/ + +# S/MIME +change-smime: copy-smime +copy-smime: + #$(DOCKER_EXEC) misp-robot /bin/bash -c "ansible-playbook -i 'localhost,' -c local -t smime /etc/ansible/playbooks/robot-playbook/site.yml" + docker cp $(CURDIR)/config/smime/. misp-server:/var/www/MISP/.smime + +# GnuPGP +change-pgp: copy-pgp +copy-pgp: + #$(DOCKER_EXEC) misp-robot /bin/bash -c "ansible-playbook -i 'localhost,' -c local -t pgp /etc/ansible/playbooks/robot-playbook/site.yml" + docker cp $(CURDIR)/config/pgp/. misp-server:/var/www/MISP/.pgp + +# maintainence +enable-maintenance: + $(DOCKER_EXEC) misp-proxy /entrypoint_nginx.sh enable-maintenance + docker restart misp-proxy +disable-maintenance: + $(DOCKER_EXEC) misp-proxy /entrypoint_nginx.sh disable-maintenance + docker restart misp-proxy + +# backup all services +backup-all: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh backup all" +backup-server: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh backup server" +backup-redis: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh backup redis" +backup-db: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh backup mysql" +backup-proxy: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh backup proxy" +backup-robot: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh backup robot" +backup-ssl: copy-ssl + +# restore service +restore-all: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh restore" +restore-server: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh restore server" +restore-redis: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh restore redis" +restore-db: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh restore mysql" +restore-proxy: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh restore proxy" +restore-robot: + $(DOCKER_EXEC) misp-robot /bin/bash -c "scripts/backup_restore.sh restore robot" + diff --git a/1.1.1/docker-compose.yml b/1.1.1/docker-compose.yml new file mode 100644 index 000000000..2148da0e5 --- /dev/null +++ b/1.1.1/docker-compose.yml @@ -0,0 +1,164 @@ +version: '3.1' + +networks: + misp-backend: + driver: bridge + driver_opts: + com.docker.network.bridge.name: "mispbr0" + com.docker.network.enable_ipv6: "false" + ipam: + config: + - subnet: "192.168.47.0/28" + +services: + # ### MISP Database ### + # # LINK: https://hub.docker.com/_/mariadb/ + # misp-db: + # image: mariadb:10.3.10-bionic + # container_name: misp-db + # restart: unless-stopped + # volumes: + # - misp-vol-db-data:/var/lib/mysql/" + # healthcheck: + # test: ["CMD-SHELL", "mysql -u $MYSQL_USER -p$MYSQL_PASSWORD -h misp-db --execute 'show databases;'"] + # interval: 1m + # timeout: 15s + # retries: 5 + # networks: + # misp-backend: + # aliases: + # - misp-db + + ### MISP Redis Server ### + # LINK: https://docs.docker.com/samples/library/redis/ + # LINK: https://github.com/docker-library/docs/tree/master/redis + misp-redis: + image: dockerhub.dcso.de/misp-dockerized-redis:5-alpine3.9 + restart: unless-stopped + container_name: misp-redis + command: ["redis-server", "--appendonly", "yes"] #For Data persistence + healthcheck: + test: ["CMD-SHELL", "[ $$(redis-cli ping) == PONG ] || exit 1"] + interval: 1m30s + timeout: 15s + retries: 3 + volumes: + - misp-vol-redis-data:/data/ + networks: + misp-backend: + aliases: + - misp-redis + + ### MISP Modules ### + misp-modules: + image: dockerhub.dcso.de/misp-dockerized-misp-modules:2 + container_name: misp-modules + restart: unless-stopped + networks: + misp-backend: + aliases: + - misp-modules + + + ### MISP Apache Server ### + misp-server: + image: dockerhub.dcso.de/misp-dockerized-server:2.4.103-debian + container_name: misp-server + restart: unless-stopped + volumes: + ###### mips-server ###### + # Apache2 Configuration + - misp-vol-server-apache2-config-sites-enabled:/etc/apache2/sites-enabled + # Volume with Certificates + - misp-vol-ssl:/etc/apache2/ssl + # Volume with PGP Key + - misp-vol-pgp:/var/www/MISP/.gnupg + # Volume with S/MIME Certificate and Key + - misp-vol-smime:/var/www/MISP/.smime + ###### misp-redis ###### + - misp-vol-redis-data:/redis_data_dir + # MISP Configurations: + - misp-vol-server-MISP-app-Config:/var/www/MISP/app/Config + - misp-vol-server-MISP-cakeresque-config:/var/www/MISP/app/Plugin/CakeResque/Config + - misp-vol-server-MISP-tmp:/var/www/MISP/app/tmp + - misp-vol-server-MISP-attachments:/var/www/MISP/app/files + ###### misp-db ###### + - misp-vol-db-data:/var/lib/mysql + networks: + misp-backend: + aliases: + - misp-server + + ### Reverse Proxy ### + misp-proxy: + image: dockerhub.dcso.de/misp-dockerized-proxy:1 + container_name: misp-proxy + restart: unless-stopped + ports: + - "80:80" + - "443:443" + environment: + HTTP_PROXY: ${HTTP_PROXY} + HTTPS_PROXY: ${HTTPS_PROXY} + NO_PROXY: ${NO_PROXY} + volumes: + # NGINX Configuration + #- misp-vol-proxy-conf:/etc/nginx/conf.d + # SSL + - misp-vol-ssl:/etc/nginx/ssl + networks: + misp-backend: + aliases: + - misp-nginx + + + ### Robot ### + misp-robot: + image: dockerhub.dcso.de/misp-dockerized-robot:2 + container_name: misp-robot + restart: unless-stopped + tty: true + stdin_open: true + healthcheck: + disable: true + volumes: + ###### GLOBAL VOLUMES ###### + # Docker.sock File + - /var/run/docker.sock:/var/run/docker.sock:ro + # Volume with Certificates + - misp-vol-ssl:/srv/misp-ssl + # Volume with PGP Key + - misp-vol-pgp:/srv/misp-pgp + # Volume with S/MIME Certificate and Key + - misp-vol-smime:/srv/misp-smime + ###### mips-server ###### + - misp-vol-server-apache2-config-sites-enabled:/srv/misp-server/apache2/sites-enabled + - misp-vol-server-MISP-app-Config:/srv/misp-server/MISP/Config + - misp-vol-server-MISP-cakeresque-config:/srv/misp-server/MISP/CakeResque/Config + - misp-vol-server-MISP-tmp:/srv/misp-server/MISP/app/tmp + - misp-vol-server-MISP-attachments:/srv/misp-server/MISP/app/files + ###### misp-proxy ###### + - misp-vol-proxy-conf:/srv/misp-proxy/conf.d + ###### misp-redis ###### + - misp-vol-redis-data:/srv/misp-redis + ###### misp-db ###### + - misp-vol-db-data:/srv/misp-db + networks: + misp-backend: + aliases: + - misp-robot + + +### VOLUMES ### +volumes: + misp-vol-ssl: + misp-vol-pgp: + misp-vol-smime: + misp-vol-db-data: + misp-vol-redis-data: + misp-vol-server-apache2-config-sites-enabled: + misp-vol-server-MISP-app-Config: + misp-vol-server-MISP-cakeresque-config: + misp-vol-server-MISP-tmp: + misp-vol-server-MISP-attachments: + misp-vol-proxy-conf: diff --git a/1.1.1/example/crontab.txt b/1.1.1/example/crontab.txt new file mode 100644 index 000000000..e50aae79f --- /dev/null +++ b/1.1.1/example/crontab.txt @@ -0,0 +1,2 @@ +# Patch once a week the container +@weekly make -C /opt/MISP-dockerized deploy diff --git a/1.1.1/example/syslog_example/100-MISP-dockerized_syslog.conf b/1.1.1/example/syslog_example/100-MISP-dockerized_syslog.conf new file mode 100644 index 000000000..54a0cacce --- /dev/null +++ b/1.1.1/example/syslog_example/100-MISP-dockerized_syslog.conf @@ -0,0 +1,24 @@ +# Messages from MISP-dockerized containers + +if $syslogtag startswith ('dcso/misp-dockerized' or 'dockerhub.dcso.de/misp-dockerized' ) and ($syslogseverity-text == 'error' or $syslogseverity-text == 'warn' or $syslogseverity-text == 'crit' or $syslogseverity-text == 'emerg') then /var/log/MISP-dockerized_error.log +if $syslogtag startswith ('dcso/misp-dockerized' or 'dockerhub.dcso.de/misp-dockerized' ) and not ($syslogseverity-text == 'error' or $syslogseverity-text == 'warn' or $syslogseverity-text == 'crit' or $syslogseverity-text == 'emerg') then /var/log/MISP-dockerized.log + + +#Optional: + # # Dockerd Messages + # :syslogtag, startswith, "dockerd" { + # /var/log/docker.log + # stop + # } + + # # Docker Interface entries + # :msg, contains, "docker" { + # /var/log/docker.log + # stop + # } + + # # misp Interface Entries + # :msg, contains, "misp" { + # /var/log/docker.log + # stop + # } diff --git a/1.1.1/scripts/backup_restore.sh b/1.1.1/scripts/backup_restore.sh new file mode 100755 index 000000000..5336a79af --- /dev/null +++ b/1.1.1/scripts/backup_restore.sh @@ -0,0 +1,185 @@ +#!/bin/bash +#description :This script can be used to backup and retore the misp docker environment. +#============================================================================== + +if [[ ! ${1} =~ (backup|restore) ]]; then + echo "First parameter needs to be 'backup' or 'restore'" + exit 1 +fi + +if [[ ${1} == "backup" && ! ${2} =~ (server|redis|mysql|proxy|config|all) ]]; then + echo "Second parameter needs to be 'server', 'redis', 'mysql', 'proxy', 'config' or 'all'" + exit 1 +fi + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +COMPOSE_FILE=${SCRIPT_DIR}/../docker-compose.yml +#source ${SCRIPT_DIR}/../.env +source /srv/MISP-dockerized/config/config.env + + + +## DebuggL +#echo "Script Dir: ${SCRIPT_DIR}" +#echo "Compose File: ${COMPOSE_FILE}" +#echo "MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}" +BACKUP_LOCATION="/srv/MISP-dockerized/backup" + +function backup() { + DATE=$(date +"%Y-%m-%d-%H-%M-%S") + DOCKER_BACKUPDIR="${BACKUP_LOCATION}/misp-${DATE}" + mkdir -p ${DOCKER_BACKUPDIR} + chmod 755 ${DOCKER_BACKUPDIR} + echo "--- Start Backup ---" + while (( "$#" )); do + case "$1" in + server|all) + echo "Backup server at ${DOCKER_BACKUPDIR}" + tar -cvpzPf ${DOCKER_BACKUPDIR}/backup_server_data.tar.gz /srv/misp-server/MISP + tar -cvpzPf ${DOCKER_BACKUPDIR}/backup_server_config.tar.gz /srv/misp-server/apache2 + ;;& + redis|all) + echo "Backup redis at ${DOCKER_BACKUPDIR}" + docker exec $(docker ps -qf name=misp-server) redis-cli save + tar -cvpzPf ${DOCKER_BACKUPDIR}/backup_redis.tar.gz /srv/misp-redis + ;;& + proxy|all) + echo "Backup proxy at ${DOCKER_BACKUPDIR}" + tar -cvpzPf ${DOCKER_BACKUPDIR}/backup_proxy_data.tar.gz /srv/misp-proxy/conf.d + ;;& + mysql|all) + echo "Backup mysql at ${DOCKER_BACKUPDIR} - This could take a while" + mysqldump -u root -p${MYSQL_ROOT_PASSWORD} -h misp-server --all-databases | gzip > ${DOCKER_BACKUPDIR}/backup_mysql.gz & pid=$! + loading_animation ${pid} + ;;#& + #config|all) + # echo "Backup config files at ${DOCKER_BACKUPDIR}/misp-${DATE}" + # tar -cvpzf ${DOCKER_BACKUPDIR}/backup_config.tar.gz /srv/MISP-dockerized/config + #;; + esac + shift + done + echo "--- Done ---" +} + +function restore() { + RESTORE_LOCATION="${1}" + echo "--- Start Restore ---" + # echo "Restore location: ${RESTORE_LOCATION}" # Debug Output + shift + while (( "$#" )); do + case "$1" in + redis|all) + echo "Restore MISP Redis" #Debug + tar -xvzPf ${RESTORE_LOCATION}backup_redis.tar.gz + docker exec misp-server service redis-server restart + ;;& + server|all) + echo "Restore MISP Server" #Debug + tar -xvzPf ${RESTORE_LOCATION}backup_server_data.tar.gz + tar -xvzPf ${RESTORE_LOCATION}backup_server_config.tar.gz; + docker exec misp-server service apache2 restart + ;;& + mysql|all) + echo "Restore MISP DB - This could take a while" #Debug + echo "-> unpacking .sql file" + gunzip < ${RESTORE_LOCATION}backup_mysql.gz | cat > ${RESTORE_LOCATION}backup_mysql.sql & pid=$! + loading_animation ${pid} + echo "-> restore database" + mysql -u root -p${MYSQL_ROOT_PASSWORD} -h misp-server < ${RESTORE_LOCATION}backup_mysql.sql & pid=$! + loading_animation ${pid} + echo "-> clean up" + rm ${RESTORE_LOCATION}backup_mysql.sql + #echo "-> restarting MySQL" + #docker exec misp-server service mysql restart + ;;& + proxy|all) + echo "Restore MISP Proxy" #Debug + tar -xvzPf ${RESTORE_LOCATION}backup_proxy_data.tar.gz + ;;#& + #config|all) + # echo "Restore config files" + # tar -xvzf ${BACKUP_LOCATION}/misp-${DATE}/backup_config.tar.gz -C ${SCRIPT_DIR}/../. + #;;& + esac + shift + echo "--- Done ---" + done +} + +function loading_animation() { + pid=${1} + + spin='-\|/' + + i=0 + while kill -0 $pid 2>/dev/null + do + i=$(( (i+1) %4 )) + printf "\r${spin:$i:1} ...working" + sleep .1 + done + echo "" +} + +if [[ ${1} == "backup" ]]; then + backup ${@,,} +elif [[ ${1} == "restore" ]]; then + i=1 + declare -A FOLDER_SELECTION + if [[ $(find ${BACKUP_LOCATION}/misp-* -maxdepth 1 -type d 2> /dev/null| wc -l) -lt 1 ]]; then + echo "Selected backup location has no subfolders" + exit 1 + fi + for folder in $(ls -d ${BACKUP_LOCATION}/misp-*/); do + echo "[ ${i} ] - ${folder}" + FOLDER_SELECTION[${i}]="${folder}" + ((i++)) + done + echo + input_sel=0 + while [[ ${input_sel} -lt 1 || ${input_sel} -gt ${i} ]]; do + read -p "Select a restore point: " input_sel + done + i=1 + echo + declare -A FILE_SELECTION + RESTORE_POINT="${FOLDER_SELECTION[${input_sel}]}" + if [[ -z $(find "${FOLDER_SELECTION[${input_sel}]}" -maxdepth 1 -type f -regex ".*\(redis\|mysql\|server\|config\).*") ]]; then + echo "No datasets found" + exit 1 + fi + for file in $(ls -f "${FOLDER_SELECTION[${input_sel}]}"); do + if [[ ${file} =~ server_data ]]; then + echo "[ ${i} ] - Server directory" + FILE_SELECTION[${i}]="server" + ((i++)) + elif [[ ${file} =~ proxy_data ]]; then + echo "[ ${i} ] - Proxy directory" + FILE_SELECTION[${i}]="proxy" + ((i++)) + elif [[ ${file} =~ redis ]]; then + echo "[ ${i} ] - Redis DB" + FILE_SELECTION[${i}]="redis" + ((i++)) + elif [[ ${file} =~ mysql ]]; then + echo "[ ${i} ] - SQL DB" + FILE_SELECTION[${i}]="mysql" + ((i++)) + fi + #elif [[ ${file} =~ config ]]; then + # echo "[ ${i} ] - Config files " + # FILE_SELECTION[${i}]="config" + # ((i++)) + done + echo "[ ${i} ] - All" + FILE_SELECTION[${i}]="all" + + echo + input_sel=0 + while [[ ${input_sel} -lt 1 || ${input_sel} -gt ${i} ]]; do + read -p "Select a dataset to restore: " input_sel + done + echo "Restoring ${FILE_SELECTION[${input_sel}]} from ${RESTORE_POINT}..." + restore "${RESTORE_POINT}" ${FILE_SELECTION[${input_sel}]} +fi \ No newline at end of file diff --git a/1.1.1/scripts/build_config.sh b/1.1.1/scripts/build_config.sh new file mode 100755 index 000000000..15b02b063 --- /dev/null +++ b/1.1.1/scripts/build_config.sh @@ -0,0 +1,737 @@ +#!/bin/bash +#description :This script build the configuration for the MISP Container and their content. +#============================================================================== +STARTMSG="[build_config.sh]" +set -e + +# Available Parameters from outside: +# export CI=true +# export DEV=true +# export DOCKER_REGISTRY=custom.url + +PARAMETER_DOCKER_REGISTRY="$1" + +# check if this is an automate build not ask any questions +[ "$CI" = true ] && AUTOMATE_BUILD=true +[ "$DEV" = true ] && DEV_MODE=true + + + +# full path /scripts +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +# full path in +MISP_dockerized_repo=$(echo ${SCRIPTPATH%/*}) +# full path in the git repository +MISP_dockerized_repo=$(echo ${MISP_dockerized_repo%/*}) + +CONFIG_FILE="${MISP_dockerized_repo}/config/config.env" +DOCKER_COMPOSE_CONF="${MISP_dockerized_repo}/current/docker-compose.override.yml" +DOCKER_COMPOSE_FILE="${MISP_dockerized_repo}/current/docker-compose.yml" +BACKUP_PATH="${MISP_dockerized_repo}/backup" +ENABLE_FILE_DCSO_DOCKER_REGISTRY="${MISP_dockerized_repo}/config/use_secure_DCSO_Docker_Registry.enable" +ENABLE_FILE_SMIME="${MISP_dockerized_repo}/config/smime/smime.enable" +ENABLE_FILE_PGP="${MISP_dockerized_repo}/config/pgp/pgp.enable" +###################################################################### +# Function to import configuration +function import_config(){ + echo -n "$STARTMSG Check and import existing configuration..." + [ -f $CONFIG_FILE ] && source $CONFIG_FILE + echo "done" +} +# Function to set default values +function check_if_vars_exists() { + echo -n "$STARTMSG Check if all vars exists..." + # Default Variables for the config: + # Hostname + [ -z "${myHOSTNAME+x}" ] && myHOSTNAME="`hostname -f`" && QUERY_myHOSTNAME="yes" + [ -z "${MISP_FQDN}" ] && MISP_FQDN="$myHOSTNAME" + [ -z "${MISP_URL}" ] && MISP_URL="https://${myHOSTNAME}" + # Network + [ -z "$DOCKER_NETWORK" ] && DOCKER_NETWORK="192.168.47.0/28" && QUERY_NETWORK="yes" + [ -z "$BRIDGE_NAME" ] && BRIDGE_NAME="mispbr0" && QUERY_NETWORK="yes" + # PROXY + [ -z "$QUESTION_USE_PROXY" ] && QUESTION_USE_PROXY="no" && QUERY_PROXY="yes" + [ -z "${HTTP_PROXY+x}" ] && HTTP_PROXY="" && QUERY_PROXY="yes" + [ -z "${HTTPS_PROXY+x}" ] && HTTPS_PROXY="" && QUERY_PROXY="yes" + [ -z "$NO_PROXY" ] && NO_PROXY="0.0.0.0" && QUERY_PROXY="yes" + # DB + [ -z "$QUESTION_OWN_DB" ] && QUESTION_OWN_DB="no" && QUERY_DB="yes" + [ -z "$MYSQL_HOST" ] && MYSQL_HOST="localhost" && QUERY_DB="yes" + [ -z "$MYSQL_PORT" ] && MYSQL_PORT="3306" && QUERY_DB="yes" + [ -z "$MYSQL_DATABASE" ] && MYSQL_DATABASE="misp" && QUERY_DB="yes" + [ -z "$MYSQL_USER" ] && MYSQL_USER="misp" && QUERY_DB="yes" + [ -z "$MYSQL_PASSWORD" ] && MYSQL_PASSWORD="$( $DOCKER_COMPOSE_CONF +version: '3.1' + +networks: + misp-backend: + driver_opts: + com.docker.network.bridge.name: "${BRIDGE_NAME}" + ipam: + config: + - subnet: "${DOCKER_NETWORK}" + +services: + # misp-db: + # ${IMAGE_MISP_DB} + # environment: + # MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + # MYSQL_DATABASE: ${MYSQL_DATABASE} + # MYSQL_USER: ${MYSQL_USER} + # MYSQL_PASSWORD: ${MYSQL_PASSWORD} + + misp-redis: + ${IMAGE_MISP_REDIS} + + misp-modules: + ${IMAGE_MISP_MODULES} + environment: + REDIS_BACKEND: ${REDIS_FQDN} + REDIS_PORT: "${REDIS_PORT}" + REDIS_PW: "${REDIS_PW}" + REDIS_DATABASE: "245" + http_proxy: ${HTTP_PROXY} + https_proxy: ${HTTPS_PROXY} + no_proxy: ${NO_PROXY} + ${LOG_SETTINGS} + + misp-server: + ${IMAGE_MISP_SERVER} + # ports: + # - "8080:80" # DEBUG only + # - "8443:443" # DEBUG only + environment: + # DB + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: ${MYSQL_DATABASE} + MYSQL_USER: ${MYSQL_USER} + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + # REDIS + REDIS_FQDN: "${REDIS_FQDN}" + REDIS_PORT: "${REDIS_PORT}" + REDIS_PW: "${REDIS_PW}" + # PROXY + http_proxy: ${HTTP_PROXY} + https_proxy: ${HTTPS_PROXY} + no_proxy: ${NO_PROXY} + # POSTFIX + SENDER_ADDRESS: ${SENDER_ADDRESS} + DOMAIN: ${DOMAIN} + HTTP_SERVERADMIN: ${HTTP_SERVERADMIN} + RELAYHOST: ${RELAYHOST} + RELAY_USER: ${RELAY_USER} + RELAY_PASSWORD: ${RELAY_PASSWORD} + DOCKER_NETWORK: ${DOCKER_NETWORK} + DEBUG_PEER: ${DEBUG_PEER} + # MISP + MISP_FQDN: ${MISP_FQDN} + MISP_URL: ${MISP_URL} + MISP_HTTPS_PORT: ${HTTPS_PORT} + MISP_prefix: ${MISP_prefix} + MISP_encoding: ${MISP_encoding} + MISP_SALT: ${MISP_SALT} + ADD_ANALYZE_COLUMN: "${ADD_ANALYZE_COLUMN}" + USE_PGP: "${USE_PGP}" + USE_SMIME: "${USE_SMIME}" + # Cron + CRON_INTERVAL: "${CRON_INTERVAL}" + CRON_USER_ID: "${CRON_USER_ID}" + # PHP + PHP_MEMORY_LIMIT: "${PHP_MEMORY_LIMIT}" + PHP_MAX_EXECUTION_TIME: "${PHP_MAX_EXECUTION_TIME}" + PHP_POST_MAX_SIZE: "${PHP_POST_MAX_SIZE}" + PHP_UPLOAD_MAX_FILESIZE: "${PHP_UPLOAD_MAX_FILESIZE}" + ${LOG_SETTINGS} + + misp-proxy: + ${IMAGE_MISP_PROXY} + environment: + MISP_FQDN: ${MISP_FQDN} + HTTP_SERVERADMIN: ${HTTP_SERVERADMIN} + http_proxy: ${HTTP_PROXY} + https_proxy: ${HTTPS_PROXY} + no_proxy: ${NO_PROXY} + IP: ${HTTP_ALLOWED_IP} + PHP_UPLOAD_MAX_FILESIZE: "${PHP_UPLOAD_MAX_FILESIZE}" + PHP_MAX_EXECUTION_TIME: "${PHP_MAX_EXECUTION_TIME}" + ${LOG_SETTINGS} + + misp-robot: + ${IMAGE_MISP_ROBOT} + environment: + http_proxy: ${HTTP_PROXY} + https_proxy: ${HTTPS_PROXY} + no_proxy: ${NO_PROXY} + MISP_FQDN: ${MISP_FQDN} + volumes: + # Github Repository + - ${MISP_dockerized_repo}:/srv/MISP-dockerized + #- ${MISP_dockerized_repo}/current/playbooks:/etc/ansible/playbooks/robot-playbook:ro + ${LOG_SETTINGS} + +EOF +############################################### + +##################################### +# ALL Variables +cat << EOF > $CONFIG_FILE +#description :This file is the global configuration file +#================================================= +# ------------------------------ +# Network Configuration +# ------------------------------ +DOCKER_NETWORK="${DOCKER_NETWORK}" +BRIDGE_NAME="${BRIDGE_NAME}" +# ------------------------------ +# For more than one container +# ------------------------------ +myHOSTNAME="${myHOSTNAME}" +# Logging +USE_SYSLOG="${USE_SYSLOG}" +SYSLOG_REMOTE_HOST="${SYSLOG_REMOTE_HOST}" +# Docker Registry Environment Variables +DOCKER_REGISTRY=${DOCKER_REGISTRY} +#POSTFIX_CONTAINER_TAG=${POSTFIX_CONTAINER_TAG} +#MISP_CONTAINER_TAG=${MISP_CONTAINER_TAG} +#PROXY_CONTAINER_TAG=${PROXY_CONTAINER_TAG} +#ROBOT_CONTAINER_TAG=${ROBOT_CONTAINER_TAG} +#MISP_MODULES_CONTAINER_TAG=${MISP_MODULES_CONTAINER_TAG} +#REDIS_CONTAINER_TAG=${REDIS_CONTAINER_TAG} +#DB_CONTAINER_TAG=${DB_CONTAINER_TAG} +# Proxy +QUESTION_USE_PROXY="${QUESTION_USE_PROXY}" +HTTP_PROXY="${HTTP_PROXY}" +HTTPS_PROXY="${HTTPS_PROXY}" +NO_PROXY="${NO_PROXY}" +# ------------------------------ +# misp-db +# ------------------------------ +# ALL DB SETTINGS REQUIRED WITHOUT ""!!! +QUESTION_OWN_DB="${QUESTION_OWN_DB}" +MYSQL_HOST="${MYSQL_HOST}" +MYSQL_PORT="${MYSQL_PORT}" +MYSQL_DATABASE="${MYSQL_DATABASE}" +MYSQL_USER="${MYSQL_USER}" +MYSQL_PASSWORD="${MYSQL_PASSWORD}" +MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD}" +# ------------------------------ +# misp-proxy +# ------------------------------ +HTTP_PORT="${HTTP_PORT}" +HTTPS_PORT="${HTTPS_PORT}" +client_max_body_size="${client_max_body_size}" +ALLOW_ALL_IPs="${ALLOW_ALL_IPs}" +HTTP_ALLOWED_IP="${HTTP_ALLOWED_IP}" +HTTP_SERVERADMIN="${HTTP_SERVERADMIN}" +# ------------------------------ +# misp-redis +# ------------------------------ +REDIS_FQDN="${REDIS_FQDN}" +REDIS_PW="${REDIS_PW}" +REDIS_PORT="${REDIS_PORT}" +USE_EXTERNAL_REDIS="${USE_EXTERNAL_REDIS}" +# ------------------------------ +# misp-server +# ------------------------------ +MISP_FQDN="${MISP_FQDN}" +MISP_URL="${MISP_URL}" +MISP_HTTPS_PORT="${HTTPS_PORT}" +MISP_TAG="${MISP_TAG}" +MISP_prefix="${MISP_prefix}" +MISP_encoding="${MISP_encoding}" +MISP_SALT="${MISP_SALT}" +ADD_ANALYZE_COLUMN="${ADD_ANALYZE_COLUMN}" +# PGP / SMIME +USE_PGP="${USE_PGP}" +USE_SMIME="${USE_SMIME}" +# Cron +CRON_INTERVAL="${CRON_INTERVAL}" +CRON_USER_ID="${CRON_USER_ID}" +# PHP +PHP_MEMORY_LIMIT="${PHP_MEMORY_LIMIT}" +PHP_MAX_EXECUTION_TIME="${PHP_MAX_EXECUTION_TIME}" +PHP_POST_MAX_SIZE="${PHP_POST_MAX_SIZE}" +PHP_UPLOAD_MAX_FILESIZE="${PHP_UPLOAD_MAX_FILESIZE}" +# ------------------------------ +# misp-postfix +# ------------------------------ +DOMAIN="${DOMAIN}" +RELAYHOST="${RELAYHOST}" +RELAY_USER="${RELAY_USER}" +RELAY_PASSWORD="${RELAY_PASSWORD}" +SENDER_ADDRESS="${SENDER_ADDRESS}" +QUESTION_DEBUG_PEERS="${QUESTION_DEBUG_PEERS}" +DEBUG_PEER="${DEBUG_PEER}" +################################################################## + +EOF +############################################### + +echo "...done" +echo +echo "$STARTMSG To change the configuration, delete the corresponding line in:" +echo "$CONFIG_FILE" +sleep 2 +########################################## diff --git a/1.1.1/scripts/delete_all_misp_from_host.sh b/1.1.1/scripts/delete_all_misp_from_host.sh new file mode 100755 index 000000000..edc9c063f --- /dev/null +++ b/1.1.1/scripts/delete_all_misp_from_host.sh @@ -0,0 +1,17 @@ +#!/bin/bash +#description :This script remove all misp docker container, their volumes and the /opt/misp path. +#============================================================================== + +echo "This will remove all container, volumes and corresponding images." +[ CI == "true" ] || read -p "Are you sure? (y): " USER_GO +[ CI == "true" ] && USER_GO="y" +if [ "$USER_GO" == "y" ]; then + echo '### stop and remove all container ###' + docker rm -f $(docker ps -aqf name=misp*) + echo '### remove all volumes ###' + docker volume rm $(docker volume ls -qf name=misp*) + echo '### remove MISP images ###' + #docker image rm $(docker image ls --format '{{.Repository}}:{{.ID}}' | grep misp | sed 's/^[^:]*://g') + echo '### remove MISP Network' + docker network rm $(docker network ls --format '{{.Name}}:{{.ID}}' | grep misp | sed 's/^[^:]*://g') +fi diff --git a/1.1.1/scripts/deploy.sh b/1.1.1/scripts/deploy.sh new file mode 100755 index 000000000..7aacf019b --- /dev/null +++ b/1.1.1/scripts/deploy.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +docker run \ + --name misp-robot-init \ + --rm \ + --network="host" \ + -v $PWD:/srv/MISP-dockerized \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + $(cat $PWD/config/config.env|grep DOCKER_REGISTRY|cut -d = -f 2)/misp-dockerized-robot:$(cat $PWD/config/config.env|grep ROBOT_CONTAINER_TAG|cut -d = -f 2) bash -c "docker-compose -f /srv/MISP-dockerized/docker-compose.yml -f /srv/MISP-dockerized/docker-compose.override.yml up -d " + docker cp $PWD/config/ssl/. misp-proxy:/etc/nginx/ssl/ \ No newline at end of file diff --git a/1.1.1/scripts/pull.sh b/1.1.1/scripts/pull.sh new file mode 100755 index 000000000..24384ce29 --- /dev/null +++ b/1.1.1/scripts/pull.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +docker run \ + -ti \ + --name misp-robot-init \ + --rm \ + --network="host" \ + -v $PWD:/srv/MISP-dockerized \ + -v ~/.docker:/root/.docker:ro \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + $(cat $PWD/config/config.env|grep DOCKER_REGISTRY|cut -d = -f 2)/misp-dockerized-robot:$(cat $PWD/config/config.env|grep ROBOT_CONTAINER_TAG|cut -d = -f 2) bash -c "docker-compose -f /srv/MISP-dockerized/docker-compose.yml -f /srv/MISP-dockerized/docker-compose.override.yml pull " + #$(cat $(CURDIR)/config/config.env|grep DOCKER_REGISTRY|cut -d = -f 2)/misp-dockerized-robot:$(shell cat $(CURDIR)/config/config.env|grep ROBOT_CONTAINER_TAG|cut -d = -f 2) bash -c "docker-compose -f /srv/MISP-dockerized/docker-compose.yml -f /srv/MISP-dockerized/docker-compose.override.yml pull " \ No newline at end of file diff --git a/1.1.1/scripts/requirements.sh b/1.1.1/scripts/requirements.sh new file mode 100755 index 000000000..6988b0b27 --- /dev/null +++ b/1.1.1/scripts/requirements.sh @@ -0,0 +1,191 @@ +#!/bin/bash + +# check if this is an automate build not ask any questions +[ "$CI" = "true" ] && AUTOMATE_BUILD="true" + +# Variables +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +STATUS="OK" +DOCKER_SOCK="/var/run/docker.sock" + + +# Load Variables from Configuration +[ -e "$SCRIPTPATH/../.env" ] && source $SCRIPTPATH/../config/config.env + +# to add options to the echo command + echo () { + command echo -e "$@" + } + +#################### Start Script ############################## + +# +# Check DOCKER +# + if [ -z "$(which docker)" ] + then + STATUS="FAIL" + echo "[FAIL] Docker is not installed. \tPlease install it first!" + else + echo "[OK] Docker is installed. \t\tOutput: $(docker -v)" + fi + +# +# Check GIT +# + if [ -z "$(which git)" ] + then + STATUS="FAIL" + echo -e "[FAIL] Git is not installed. \t\t\tPlease install it first!" + else + echo -e "[OK] Git is installed. \t\t\tOutput: $(git --version)" + fi + +# +# CHECK required URLs +# +echo +# check_URL: check a url with 2 parameters: URL_BASE=github.com & URL_PROTOCOL=http/https +function check_URL(){ + #set -xv + URL="$1" + [ "$USE_PROXY" == "yes" ] && PROXY=" -x $HTTP_PROXY" + OPTIONS="-vs --connect-timeout 60 -m 30 $PROXY" + COMMAND="$(curl $OPTIONS $URL 2>&1|grep 'Connected to')" + + if [ -z "$COMMAND" ] + then + echo "[WARN] Check: $URL" + echo " Result: Connection not available." + #[ "$AUTOMATE_BUILD" == "true" ] || read -r -p " continue with ENTER" + else + echo "[OK] Check: $URL" + echo " Result: $COMMAND." + fi +} + +check_URL https://misp.dcso.de +check_URL https://dockerhub.dcso.de/v2/ +check_URL https://github.com/DCSO/misp-dockerized +check_URL https://github.com/misp/misp + +############################### USER CHECKS ######################### +echo "" # Empty Line for a better overview. + +# +# Check user part of docker group +# + if [ $(whoami) != "root" ] + then + # if user is not root then check if it is in docker group + if [ -z "$(cat /etc/group|grep docker|grep `whoami`)" ] + then + STATUS="FAIL" + # user not part of docker group + echo "[FAIL] User '$(whoami)' is not part of the 'docker' group. -> Try: sudo usermod -aG docker $(whoami)" + else + # user is in docker group + echo "[OK] User '$(whoami)' is part of the 'docker' group." + fi + else + echo "[OK] User '$(whoami)' is root." + fi +# +# Check docker.sock +# + if [ ! -z "$(docker ps 2>&1|grep 'permission denied')" ] + then + STATUS="FAIL" + echo "[FAIL] User '$(whoami)' has not access to Docker daemon." + else + # user is in docker group + echo "[OK] User '$(whoami)' has access to Docker daemon." + fi + +############################### FILE CHECKS ######################### + +# +# Check Write permissions +# +echo +[ ! -d ./config/ssl ] && echo -n "Create config and config/ssl directory..." && mkdir -p ./config/ssl && echo "finished." +[ ! -d ./config/smime ] && echo -n "Create config/smime directory..." && mkdir ./config/smime && echo "finished." +[ ! -d ./config/pgp ] && echo -n "Create config/pgp directory..." && mkdir ./config/pgp && echo "finished." +[ ! -d ./backup ] && echo -n "Create backup directory..." && mkdir ./backup && echo "finished." + +function check_folder(){ + FOLDER="$1" + if [ ! -e "$FOLDER" ] + then + STATUS="FAIL" + echo "[FAIL] Can not create '$FOLDER' folder." + else + # user is in docker group + echo "[OK] Folder $FOLDER exists." + touch $FOLDER/test + if [ ! -e $FOLDER/test ] + then + STATUS="FAIL" + echo "[FAIL] No write permissions in '$FOLDER'. Please ensure that user '${whoami}' has write permissions.'" + else + echo "[OK] Testfile in '$FOLDER' can be created." + rm $FOLDER/test + fi + fi +} + +check_folder "config" +check_folder "config/ssl" +check_folder "config/pgp" +check_folder "config/smime" +check_folder "backup" + + + +############################### SSL CERT CHECKS ######################### +echo +if [ ! -f ./config/ssl/key.pem -a ! -f ./config/ssl/cert.pem ]; then + echo "[WARN] No SSL certificate found. We create a self-signed certificate in the volume." + echo " To change the SSL certificate and private key later: " + echo " 1. Please save your certificate in: $PWD/config/ssl/cert.pem" + echo " 2. Please save your private keyfile in: $PWD/config/ssl/key.pem" + echo " 3. do: make config-ssl" + echo + echo +fi + +############################### SMIME CHECKS ######################### +echo +if [ ! -f ./config/smime/key.pem -a ! -f ./config/smime/cert.pem ]; then + echo "[WARN] No S/MIME certificate found." + echo " 1. Please save your certificate in: $PWD/config/smime/cert.pem" + echo " 2. Please save your private key in: $PWD/config/smime/key.pem" + echo " 3. Do: make config-smime" + echo +fi + +############################### PGP CHECKS ######################### +echo +if [ ! -f ./config/pgp/private.key -a ! -f ./config/pgp/public.key ]; then + echo "[WARN] No PGP key found." + echo " To replace the PGP public and private file later: " + echo " 1. Please save your public key in: $PWD/config/pgp/public.key" + echo " 2. Please save your private key in: $PWD/config/pgp/private.key" + echo " 3. Do: make config-pgp" + echo + echo +fi + +############################### END Result ######################### +echo "End result:" +if [ $STATUS == "FAIL" ] + then + echo "[$STATUS] At least one error is occured." + echo + exit 1 + else + echo "[$STATUS] No error is occured." + echo + exit 0 +fi +########################################################################## \ No newline at end of file diff --git a/Makefile b/Makefile index 57f38ee11..7c74421ba 100644 --- a/Makefile +++ b/Makefile @@ -25,3 +25,16 @@ delete: support: bash .scripts/support.sh + +change-ssl: + -make -C current change-ssl + +change-smime: + -make -C current change-smime + +change-pgp: + -make -C current change-pgp + +backup: + make -C current backup-all +