From 06db66617b7c6935df097b59c94a6085bad3de6a Mon Sep 17 00:00:00 2001 From: schneidermatic Date: Sun, 5 May 2019 17:26:26 +0200 Subject: [PATCH] The following files were added/modified for elastic 'v7.0.1'. On branch master Your branch is up to date with 'origin/master'. Changes to be committed: modified: .env modified: .gitignore modified: config/apm-server/apm-server.yml modified: config/auditbeat/auditbeat.yml modified: config/elasticsearch/elasticsearch.p12 modified: config/elasticsearch/elasticsearch.yml modified: config/filebeat/filebeat.yml modified: config/heartbeat/heartbeat.yml new file: config/journalbeat/journalbeat.yml modified: config/kibana/kibana.p12 modified: config/kibana/kibana.yml modified: config/logstash/logstash.p12 modified: config/logstash/logstash.yml modified: config/logstash/pipeline/logstash.conf modified: config/metricbeat/metricbeat.yml modified: config/packetbeat/packetbeat.yml modified: config/ssl/instances.yml modified: docker-compose.setup.yml modified: docker-compose.yml new file: scripts/set-pwd.sh modified: scripts/setup-beat.sh modified: scripts/setup-elasticsearch.sh modified: scripts/setup-kibana.sh modified: scripts/setup-logstash.sh modified: scripts/setup.sh modified: setup.yml --- .env | 4 +- .gitignore | 13 ++-- config/apm-server/apm-server.yml | 4 ++ config/auditbeat/auditbeat.yml | 5 +- config/elasticsearch/elasticsearch.p12 | Bin config/elasticsearch/elasticsearch.yml | 4 +- config/filebeat/filebeat.yml | 31 ++++---- config/heartbeat/heartbeat.yml | 4 +- config/journalbeat/journalbeat.yml | 30 ++++++++ config/kibana/kibana.p12 | Bin config/kibana/kibana.yml | 2 +- config/logstash/logstash.p12 | Bin config/logstash/logstash.yml | 5 +- config/logstash/pipeline/logstash.conf | 0 config/metricbeat/metricbeat.yml | 2 + config/packetbeat/packetbeat.yml | 4 +- config/ssl/instances.yml | 0 docker-compose.setup.yml | 23 +++++- docker-compose.yml | 83 +++++++++++++++------ scripts/set-pwd.sh | 8 +++ scripts/setup-beat.sh | 42 +++++++++-- scripts/setup-elasticsearch.sh | 95 ++++++++++++------------- scripts/setup-kibana.sh | 12 +++- scripts/setup-logstash.sh | 6 +- scripts/setup.sh | 29 ++++++-- setup.yml | 2 +- 26 files changed, 289 insertions(+), 119 deletions(-) mode change 100644 => 100755 config/elasticsearch/elasticsearch.p12 mode change 100644 => 100755 config/elasticsearch/elasticsearch.yml create mode 100644 config/journalbeat/journalbeat.yml mode change 100644 => 100755 config/kibana/kibana.p12 mode change 100644 => 100755 config/kibana/kibana.yml mode change 100644 => 100755 config/logstash/logstash.p12 mode change 100644 => 100755 config/logstash/logstash.yml mode change 100644 => 100755 config/logstash/pipeline/logstash.conf mode change 100644 => 100755 config/ssl/instances.yml mode change 100644 => 100755 docker-compose.setup.yml mode change 100644 => 100755 docker-compose.yml create mode 100755 scripts/set-pwd.sh mode change 100644 => 100755 setup.yml diff --git a/.env b/.env index c342d84..0f23a08 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ -TAG=6.6.0 -ELASTIC_VERSION=6.6.0 +TAG=7.0.1 +ELASTIC_VERSION=7.0.1 diff --git a/.gitignore b/.gitignore index a990b5d..61679f3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,10 @@ config/ssl/ca/ca.key config/ssl/docker-cluster-ca.zip config/ssl/docker-cluster.zip -config/auditbeat/auditbeat.keystore -config/filebeat/filebeat.keystore -config/heartbeat/heartbeat.keystore -config/metricbeat/metricbeat.keystore -config/packetbeat/packetbeat.keystore -config/apm-server/apm-server.keystore \ No newline at end of file +config/apm-server/beats.keystore +config/auditbeat/beats.keystore +config/filebeat/beats.keystore +config/heartbeat/beats.keystore +config/journalbeat/beats.keystore +config/metricbeat/beats.keystore +config/packetbeat/beats.keystore \ No newline at end of file diff --git a/config/apm-server/apm-server.yml b/config/apm-server/apm-server.yml index faba2ef..68b5374 100644 --- a/config/apm-server/apm-server.yml +++ b/config/apm-server/apm-server.yml @@ -1,6 +1,8 @@ apm-server.frontend.enabled: true apm-server.host: "0.0.0.0:8200" +keystore.path: "/usr/share/apm-server/data/beats.keystore" + output.elasticsearch: hosts: ['elasticsearch:9200'] protocol: "https" @@ -16,3 +18,5 @@ setup.kibana: protocol: "http" ssl.enabled: false ssl.certificate_authorities: ["/usr/share/apm-server/certs/ca/ca.crt"] + +xpack.monitoring.enabled: true diff --git a/config/auditbeat/auditbeat.yml b/config/auditbeat/auditbeat.yml index 98f1613..49d1c7b 100644 --- a/config/auditbeat/auditbeat.yml +++ b/config/auditbeat/auditbeat.yml @@ -13,12 +13,14 @@ auditbeat.modules: - /usr/sbin - /etc +keystore.path: "/usr/share/auditbeat/data/beats.keystore" + output.elasticsearch: hosts: ['elasticsearch:9200'] protocol: "https" username: elastic - # Read PW from auditbeat.keystore password: "${ELASTIC_PASSWORD}" + # Read PW from auditbeat.keystore ssl.certificate_authorities: ["/usr/share/auditbeat/certs/ca/ca.crt"] setup.kibana: @@ -30,3 +32,4 @@ setup.kibana: ssl.certificate_authorities: ["/usr/share/auditbeat/certs/ca/ca.crt"] xpack.monitoring.enabled: true +xpack.monitoring.elasticsearch: diff --git a/config/elasticsearch/elasticsearch.p12 b/config/elasticsearch/elasticsearch.p12 old mode 100644 new mode 100755 diff --git a/config/elasticsearch/elasticsearch.yml b/config/elasticsearch/elasticsearch.yml old mode 100644 new mode 100755 index 0f77d84..4b5c7ca --- a/config/elasticsearch/elasticsearch.yml +++ b/config/elasticsearch/elasticsearch.yml @@ -1,3 +1,4 @@ + cluster.name: "docker-cluster" network.host: 0.0.0.0 transport.host: 0.0.0.0 @@ -5,12 +6,13 @@ transport.host: 0.0.0.0 # minimum_master_nodes need to be explicitly set when bound on a public IP # set to 1 to allow single node clusters # Details: https://github.com/elastic/elasticsearch/pull/17288 +cluster.initial_master_nodes: ["es-cluster01"] discovery.zen.minimum_master_nodes: 1 xpack.license.self_generated.type: trial xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.http.ssl.verification_mode: certificate -xpack.security.http.ssl.key: certs/elasticsearch/elasticsearch.key +xpack.security.http.ssl.key: certs/elasticsearch/elasticsearch.key xpack.security.http.ssl.certificate: certs/elasticsearch/elasticsearch.crt xpack.security.http.ssl.certificate_authorities: [ "certs/ca/ca.crt" ] diff --git a/config/filebeat/filebeat.yml b/config/filebeat/filebeat.yml index 78ccd0a..661e300 100644 --- a/config/filebeat/filebeat.yml +++ b/config/filebeat/filebeat.yml @@ -1,28 +1,33 @@ -filebeat.config: - prospectors: - path: ${path.config}/prospectors.d/*.yml - reload.enabled: false - modules: - path: ${path.config}/modules.d/*.yml - reload.enabled: false +filebeat.inputs: +- type: log + enabled: true + paths: + - ${path.config}/prospectors.d/*.yml + +filebeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false processors: -- add_cloud_metadata: + - add_cloud_metadata: + +keystore.path: "/usr/share/filebeat/data/beats.keystore" output.elasticsearch: - hosts: ['elasticsearch:9200'] - protocol: "https" username: elastic - # Read PW from filebeat.keystore password: "${ELASTIC_PASSWORD}" + hosts: ["elasticsearch:9200"] + protocol: https + # Read PW from auditbeat.keystore ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"] setup.kibana: - host: "http://kibana:5601" username: elastic password: "${ELASTIC_PASSWORD}" - protocol: "http" + protocol: http + host: "http://kibana:5601" ssl.enabled: false ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"] xpack.monitoring.enabled: true +xpack.monitoring.elasticsearch: diff --git a/config/heartbeat/heartbeat.yml b/config/heartbeat/heartbeat.yml index bbdba46..46e99f4 100644 --- a/config/heartbeat/heartbeat.yml +++ b/config/heartbeat/heartbeat.yml @@ -14,7 +14,9 @@ heartbeat.monitors: - kibana processors: -- add_cloud_metadata: + - add_cloud_metadata: + +keystore.path: "/usr/share/heartbeat/data/beats.keystore" output.elasticsearch: hosts: ['elasticsearch:9200'] diff --git a/config/journalbeat/journalbeat.yml b/config/journalbeat/journalbeat.yml new file mode 100644 index 0000000..b45f4ea --- /dev/null +++ b/config/journalbeat/journalbeat.yml @@ -0,0 +1,30 @@ +journalbeat.inputs: +- paths: [] + seek: cursor + +setup.template.settings: + index.number_of_shards: 1 + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + +keystore.path: "/usr/share/journalbeat/data/beats.keystore" + +output.elasticsearch: + hosts: ['elasticsearch:9200'] + protocol: "https" + username: elastic + # Read PW from beats.keystore + password: "${ELASTIC_PASSWORD}" + ssl.certificate_authorities: ["/usr/share/journalbeat/certs/ca/ca.crt"] + +setup.kibana: + host: "http://kibana:5601" + username: elastic + password: "${ELASTIC_PASSWORD}" + protocol: "http" + ssl.enabled: false + ssl.certificate_authorities: ["/usr/share/journalbeat/certs/ca/ca.crt"] + +xpack.monitoring.enabled: true diff --git a/config/kibana/kibana.p12 b/config/kibana/kibana.p12 old mode 100644 new mode 100755 diff --git a/config/kibana/kibana.yml b/config/kibana/kibana.yml old mode 100644 new mode 100755 index 37857aa..d40c828 --- a/config/kibana/kibana.yml +++ b/config/kibana/kibana.yml @@ -2,7 +2,7 @@ server.name: kibana server.host: "0" -elasticsearch.url: https://elasticsearch:9200 +elasticsearch.hosts: [ "https://elasticsearch:9200" ] # elasticsearch.password is stored in `kibana.keystore` elasticsearch.username: kibana elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca/ca.crt"] diff --git a/config/logstash/logstash.p12 b/config/logstash/logstash.p12 old mode 100644 new mode 100755 diff --git a/config/logstash/logstash.yml b/config/logstash/logstash.yml old mode 100644 new mode 100755 index b53754f..6c1e48b --- a/config/logstash/logstash.yml +++ b/config/logstash/logstash.yml @@ -1,7 +1,6 @@ http.host: 0.0.0.0 # read password from logstash.keystore xpack.monitoring.elasticsearch.password: ${ELASTIC_PASSWORD} -xpack.monitoring.elasticsearch.url: https://elasticsearch:9200 +xpack.monitoring.elasticsearch.hosts: ["https://elasticsearch:9200"] xpack.monitoring.elasticsearch.username: logstash_system -xpack.monitoring.elasticsearch.ssl.ca: /usr/share/logstash/config/certs/ca/ca.crt - +xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/ca/ca.crt diff --git a/config/logstash/pipeline/logstash.conf b/config/logstash/pipeline/logstash.conf old mode 100644 new mode 100755 diff --git a/config/metricbeat/metricbeat.yml b/config/metricbeat/metricbeat.yml index bfc3a2e..b7ad743 100644 --- a/config/metricbeat/metricbeat.yml +++ b/config/metricbeat/metricbeat.yml @@ -5,6 +5,8 @@ metricbeat.config.modules: processors: - add_cloud_metadata: +keystore.path: "/usr/share/metricbeat/data/beats.keystore" + output.elasticsearch: hosts: ['elasticsearch:9200'] protocol: "https" diff --git a/config/packetbeat/packetbeat.yml b/config/packetbeat/packetbeat.yml index 48c2248..bf5d7cf 100644 --- a/config/packetbeat/packetbeat.yml +++ b/config/packetbeat/packetbeat.yml @@ -35,7 +35,9 @@ packetbeat.protocols.cassandra: processors: -- add_cloud_metadata: + - add_cloud_metadata: + +keystore.path: "/usr/share/packetbeat/data/beats.keystore" output.elasticsearch: hosts: ['elasticsearch:9200'] diff --git a/config/ssl/instances.yml b/config/ssl/instances.yml old mode 100644 new mode 100755 diff --git a/docker-compose.setup.yml b/docker-compose.setup.yml old mode 100644 new mode 100755 index 50acaea..76e63ae --- a/docker-compose.setup.yml +++ b/docker-compose.setup.yml @@ -54,7 +54,7 @@ services: command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s auditbeat'] environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] networks: ['stack'] - depends_on: ['kibana'] + depends_on: ['elasticsearch', 'kibana'] setup_filebeat: image: docker.elastic.co/beats/filebeat:${TAG} @@ -65,10 +65,11 @@ services: - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - './config/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml' - './config/ssl/ca/ca.crt:/usr/share/filebeat/certs/ca/ca.crt' + command: ['/bin/bash', '-c', 'chown root:filebeat /usr/share/filebeat/filebeat.yml'] command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s filebeat'] environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] networks: ['stack'] - depends_on: ['kibana'] + depends_on: ['elasticsearch', 'kibana'] setup_heartbeat: image: docker.elastic.co/beats/heartbeat:${TAG} @@ -113,6 +114,22 @@ services: networks: ['stack'] depends_on: ['kibana'] + setup_journalbeat: + image: docker.elastic.co/beats/journalbeat:${TAG} + container_name: setup_journalbeat + user: root + pid: host + cap_add: ['AUDIT_CONTROL', 'AUDIT_READ'] + volumes: + - './config:/config' + - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/journalbeat/journalbeat.yml:/usr/share/journalbeat/journalbeat.yml' + - './config/ssl/ca/ca.crt:/usr/share/journalbeat/certs/ca/ca.crt' + command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s journalbeat'] + environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] + networks: ['stack'] + depends_on: ['elasticsearch', 'kibana'] + setup_apm_server: image: docker.elastic.co/apm/apm-server:${TAG} container_name: setup_apm_server @@ -125,4 +142,4 @@ services: - './config/ssl/ca/ca.crt:/usr/share/apm-server/certs/ca/ca.crt' environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] networks: ['stack'] - depends_on: ['kibana'] + depends_on: ['kibana'] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 index 346dd61..313b486 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: # '.env' file in this folder. It can be overridden with any normal # technique for setting environment variables, for example: # - # TAG=6.0.0-beta1 docker-compose up + # TAG=7.0.0 docker-compose up # # REF: https://docs.docker.com/compose/compose-file/#variable-substitution # @@ -31,7 +31,17 @@ services: target: /usr/share/elasticsearch/config/certs/elasticsearch/elasticsearch.key - source: elasticsearch.crt target: /usr/share/elasticsearch/config/certs/elasticsearch/elasticsearch.crt - ports: ['9200:9200'] + ports: + - '9200:9200' + - '9300:9300' + environment: + - node.name=es01 + # - discovery.type=single-node + - discovery.seed_hosts=es01 + - cluster.initial_master_nodes=es01 + - cluster.name=es-cluster01 + # - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" networks: ['stack'] volumes: - 'es_data:/usr/share/elasticsearch/data' @@ -94,7 +104,7 @@ services: - source: auditbeat.yml target: /usr/share/auditbeat/auditbeat.yml - source: auditbeat.keystore - target: /usr/share/auditbeat/auditbeat.keystore + target: /usr/share/auditbeat/data/beats.keystore - source: ca.crt target: /usr/share/auditbeat/certs/ca/ca.crt # Auditbeat must run in the main process namespace. @@ -112,15 +122,17 @@ services: filebeat: image: docker.elastic.co/beats/filebeat:${TAG} container_name: filebeat - command: --strict.perms=false -e # -e flag to log to stderr and disable syslog/file output + command: -e --strict.perms=false # -e flag to log to stderr and disable syslog/file output # If the host system has logs at "/var/log", mount them at "/mnt/log" # inside the container, where Filebeat can find them. # volumes: ['/var/log:/mnt/log:ro'] + cap_add: ['AUDIT_CONTROL', 'AUDIT_READ'] + #environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] secrets: - source: filebeat.yml target: /usr/share/filebeat/filebeat.yml - source: filebeat.keystore - target: /usr/share/filebeat/filebeat.keystore + target: /usr/share/filebeat/data/beats.keystore - source: ca.crt target: /usr/share/filebeat/certs/ca/ca.crt volumes: @@ -137,11 +149,12 @@ services: image: docker.elastic.co/beats/heartbeat:${TAG} container_name: heartbeat command: --strict.perms=false -e # -e flag to log to stderr and disable syslog/file output + cap_add: ['AUDIT_CONTROL', 'AUDIT_READ'] secrets: - source: heartbeat.yml target: /usr/share/heartbeat/heartbeat.yml - source: heartbeat.keystore - target: /usr/share/heartbeat/heartbeat.keystore + target: /usr/share/heartbeat/data/beats.keystore - source: ca.crt target: /usr/share/heartbeat/certs/ca/ca.crt volumes: @@ -170,7 +183,7 @@ services: - source: metricbeat.yml target: /usr/share/metricbeat/metricbeat.yml - source: metricbeat.keystore - target: /usr/share/metricbeat/metricbeat.keystore + target: /usr/share/metricbeat/data/beats.keystore - source: ca.crt target: /usr/share/metricbeat/certs/ca/ca.crt volumes: @@ -204,7 +217,7 @@ services: - source: packetbeat.yml target: /usr/share/packetbeat/packetbeat.yml - source: packetbeat.keystore - target: /usr/share/packetbeat/packetbeat.keystore + target: /usr/share/packetbeat/data/beats.keystore - source: ca.crt target: /usr/share/packetbeat/certs/ca/ca.crt volumes: @@ -216,6 +229,29 @@ services: timeout: 15s retries: 5 + journalbeat: + image: docker.elastic.co/beats/journalbeat:${TAG} + container_name: journalbeat + command: -e --strict.perms=false # -e flag to log to stderr and disable syslog/file output + cap_add: ['AUDIT_CONTROL', 'AUDIT_READ'] + secrets: + - source: journalbeat.yml + target: /usr/share/journalbeat/journalbeat.yml + - source: journalbeat.keystore + target: /usr/share/journalbeat/data/beats.keystore + - source: ca.crt + target: /usr/share/journalbeat/certs/ca/ca.crt + # Journalbeat must run in the main process namespace. + volumes: + - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + networks: ['stack'] + depends_on: ['elasticsearch', 'kibana'] + healthcheck: + test: journalbeat --strict.perms=false test config + interval: 30s + timeout: 15s + retries: 5 + apm-server: image: docker.elastic.co/apm/apm-server:${TAG} container_name: apm_server @@ -226,7 +262,7 @@ services: - source: apm-server.yml target: /usr/share/apm-server/apm-server.yml - source: apm-server.keystore - target: /usr/share/apm-server/apm-server.keystore + target: /usr/share/apm-server/data/beats.keystore - source: ca.crt target: /usr/share/apm-server/certs/ca/ca.crt volumes: @@ -238,6 +274,7 @@ services: timeout: 10s retries: 5 + networks: {stack: {}} # use docker volume to persist ES data outside of a container. volumes: @@ -246,12 +283,6 @@ volumes: secrets: ca.crt: file: ./config/ssl/ca/ca.crt - logstash.yml: - file: ./config/logstash/logstash.yml - logstash.keystore: - file: ./config/logstash/logstash.keystore - logstash.conf: - file: ./config/logstash/pipeline/logstash.conf elasticsearch.yml: file: ./config/elasticsearch/elasticsearch.yml elasticsearch.keystore: @@ -270,27 +301,37 @@ secrets: file: ./config/kibana/kibana.key kibana.crt: file: ./config/kibana/kibana.crt + logstash.yml: + file: ./config/logstash/logstash.yml + logstash.keystore: + file: ./config/logstash/logstash.keystore + logstash.conf: + file: ./config/logstash/pipeline/logstash.conf auditbeat.yml: file: ./config/auditbeat/auditbeat.yml auditbeat.keystore: - file: ./config/auditbeat/auditbeat.keystore + file: ./config/auditbeat/beats.keystore filebeat.yml: file: ./config/filebeat/filebeat.yml filebeat.keystore: - file: ./config/filebeat/filebeat.keystore + file: ./config/filebeat/beats.keystore heartbeat.yml: file: ./config/heartbeat/heartbeat.yml heartbeat.keystore: - file: ./config/heartbeat/heartbeat.keystore + file: ./config/heartbeat/beats.keystore metricbeat.yml: file: ./config/metricbeat/metricbeat.yml metricbeat.keystore: - file: ./config/metricbeat/metricbeat.keystore + file: ./config/metricbeat/beats.keystore packetbeat.yml: file: ./config/packetbeat/packetbeat.yml packetbeat.keystore: - file: ./config/packetbeat/packetbeat.keystore + file: ./config/packetbeat/beats.keystore + journalbeat.yml: + file: ./config/journalbeat/journalbeat.yml + journalbeat.keystore: + file: ./config/journalbeat/beats.keystore apm-server.yml: file: ./config/apm-server/apm-server.yml apm-server.keystore: - file: ./config/apm-server/apm-server.keystore + file: ./config/apm-server/beats.keystore \ No newline at end of file diff --git a/scripts/set-pwd.sh b/scripts/set-pwd.sh new file mode 100755 index 0000000..d2ed038 --- /dev/null +++ b/scripts/set-pwd.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +es_url="https://localhost:9200" +ELASTIC_PASSWORD="IUgSCwpr693bKX3xqpC8Og==" + +curl -u "elastic:${ELASTIC_PASSWORD}" -k -s -H 'Content-Type:application/json' \ + -XPUT $es_url/_security/user/kibana/_password \ + -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" diff --git a/scripts/setup-beat.sh b/scripts/setup-beat.sh index 57d6458..1537c7f 100755 --- a/scripts/setup-beat.sh +++ b/scripts/setup-beat.sh @@ -9,16 +9,44 @@ until curl -s "http://kibana:5601/login" | grep "Loading Kibana" > /dev/null; do sleep 1 done -chmod go-w /usr/share/$beat/$beat.yml +cacert=/usr/share/${beat}/certs/ca/ca.crt +# Wait for ca file to exist before we continue. If the ca file doesn't exist +# then something went wrong. +while [ ! -f $cacert ] +do + sleep 2 +done +#ls -la $cacert + +es_url=https://elasticsearch:9200 +# Wait for Elasticsearch to start up before doing anything. +while [[ "$(curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -o /dev/null -w '%{http_code}' $es_url)" != "200" ]]; do + sleep 5 +done + +# Set the password for the beats_system user. +# REF: https://www.elastic.co/guide/en/x-pack/6.0/setting-up-authentication.html#set-built-in-user-passwords +until curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -H 'Content-Type:application/json' \ + -XPUT $es_url/_security/user/beats_system/_password \ + -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" +do + sleep 2 + echo "Waiting for elasticsearch..." +done +chmod u=rw,go=r,o=r /usr/share/$beat/$beat.yml +#chown root:${beat} /usr/share/${beat}/${beat}.yml +chown root:1000 /usr/share/${beat}/${beat}.yml +#chown root:root /usr/share/${beat}/${beat}.yml +ls -la /usr/share/$beat/$beat.yml +#chown root:root /usr/share/${beat}/${beat}.yml echo "Creating keystore..." # create beat keystore -${beat} --strict.perms=false keystore create --force -chown 1000 /usr/share/$beat/$beat.keystore -chmod go-w /usr/share/$beat/$beat.yml +$beat keystore create --force +chown root:root /usr/share/${beat}/data/beats.keystore -echo "adding ES_PASSWORD to keystore..." +echo "adding ELASTIC_PASSWORD to keystore..." echo "$ELASTIC_PASSWORD" | ${beat} --strict.perms=false keystore add ELASTIC_PASSWORD --stdin ${beat} --strict.perms=false keystore list @@ -28,5 +56,5 @@ echo "Setting up dashboards..." ${beat} --strict.perms=false setup -v echo "Copy keystore to ./config dir" -cp /usr/share/$beat/$beat.keystore /config/$beat/$beat.keystore -chown 1000:1000 /config/$beat/$beat.keystore +cp /usr/share/$beat/data/beats.keystore /config/$beat/beats.keystore +chown 1000:1000 /config/$beat/beats.keystore diff --git a/scripts/setup-elasticsearch.sh b/scripts/setup-elasticsearch.sh index 13830e8..a88222a 100755 --- a/scripts/setup-elasticsearch.sh +++ b/scripts/setup-elasticsearch.sh @@ -5,60 +5,59 @@ if [ -f /config/elasticsearch/elasticsearch.keystore ]; then exit 0 fi -# Determine if x-pack is enabled -echo "Determining if x-pack is installed..." -if [[ -d /usr/share/elasticsearch/bin/x-pack ]]; then - if [[ -n "$ELASTIC_PASSWORD" ]]; then - echo "=== CREATE Keystore ===" - echo "Elastic password is: $ELASTIC_PASSWORD" - if [ -f /config/elasticsearch/elasticsearch.keystore ]; then - echo "Remove old elasticsearch.keystore" - rm /config/elasticsearch/elasticsearch.keystore - fi - [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (/usr/share/elasticsearch/bin/elasticsearch-keystore create) - echo "Setting bootstrap.password..." - (echo "$ELASTIC_PASSWORD" | /usr/share/elasticsearch/bin/elasticsearch-keystore add -x 'bootstrap.password') - mv /usr/share/elasticsearch/config/elasticsearch.keystore /config/elasticsearch/elasticsearch.keystore +if [[ -n "$ELASTIC_PASSWORD" ]]; then - # Create SSL Certs - echo "=== CREATE SSL CERTS ===" + echo "=== CREATE Keystore ===" + echo "Elastic password is: $ELASTIC_PASSWORD" + if [ -f /config/elasticsearch/elasticsearch.keystore ]; then + echo "Remove old elasticsearch.keystore" + rm /config/elasticsearch/elasticsearch.keystore + fi + [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (/usr/share/elasticsearch/bin/elasticsearch-keystore create) + echo "Setting bootstrap.password..." + (echo "$ELASTIC_PASSWORD" | /usr/share/elasticsearch/bin/elasticsearch-keystore add -x 'bootstrap.password') - # check if old docker-cluster-ca.zip exists, if it does remove and create a new one. - if [ -f /config/ssl/docker-cluster-ca.zip ]; then - echo "Remove old ca zip..." - rm /config/ssl/docker-cluster-ca.zip - fi - echo "Creating docker-cluster-ca.zip..." - /usr/share/elasticsearch/bin/elasticsearch-certutil ca --pem --silent --out /config/ssl/docker-cluster-ca.zip + + mv /usr/share/elasticsearch/config/elasticsearch.keystore /config/elasticsearch/elasticsearch.keystore - # check if ca directory exists, if does, remove then unzip new files - if [ -d /config/ssl/ca ]; then - echo "CA directory exists, removing..." - rm -rf /config/ssl/ca - fi - echo "Unzip ca files..." - unzip /config/ssl/docker-cluster-ca.zip -d /config/ssl + # Create SSL Certs + echo "=== CREATE SSL CERTS ===" - # check if certs zip exist. If it does remove and create a new one. - if [ -f /config/ssl/docker-cluster.zip ]; then - echo "Remove old docker-cluster.zip zip..." - rm /config/ssl/docker-cluster.zip - fi - echo "Create cluster certs zipfile..." - /usr/share/elasticsearch/bin/elasticsearch-certutil cert --silent --pem --in /config/ssl/instances.yml --out /config/ssl/docker-cluster.zip --ca-cert /config/ssl/ca/ca.crt --ca-key /config/ssl/ca/ca.key + # check if old docker-cluster-ca.zip exists, if it does remove and create a new one. + if [ -f /config/ssl/docker-cluster-ca.zip ]; then + echo "Remove old ca zip..." + rm /config/ssl/docker-cluster-ca.zip + fi + echo "Creating docker-cluster-ca.zip..." + /usr/share/elasticsearch/bin/elasticsearch-certutil ca --pem --silent --out /config/ssl/docker-cluster-ca.zip - if [ -d /config/ssl/docker-cluster ]; then - rm -rf /config/ssl/docker-cluster - fi - echo "Unzipping cluster certs zipfile..." - unzip /config/ssl/docker-cluster.zip -d /config/ssl/docker-cluster + # check if ca directory exists, if does, remove then unzip new files + if [ -d /config/ssl/ca ]; then + echo "CA directory exists, removing..." + rm -rf /config/ssl/ca + fi + echo "Unzip ca files..." + unzip /config/ssl/docker-cluster-ca.zip -d /config/ssl + + # check if certs zip exist. If it does remove and create a new one. + if [ -f /config/ssl/docker-cluster.zip ]; then + echo "Remove old docker-cluster.zip zip..." + rm /config/ssl/docker-cluster.zip + fi + echo "Create cluster certs zipfile..." + /usr/share/elasticsearch/bin/elasticsearch-certutil cert --silent --pem --in /config/ssl/instances.yml --out /config/ssl/docker-cluster.zip --ca-cert /config/ssl/ca/ca.crt --ca-key /config/ssl/ca/ca.key - echo "Move logstash certs to logstash config dir..." - mv /config/ssl/docker-cluster/logstash/* /config/logstash/ - echo "Move kibana certs to kibana config dir..." - mv /config/ssl/docker-cluster/kibana/* /config/kibana/ - echo "Move elasticsearch certs to elasticsearch config dir..." - mv /config/ssl/docker-cluster/elasticsearch/* /config/elasticsearch/ + if [ -d /config/ssl/docker-cluster ]; then + rm -rf /config/ssl/docker-cluster fi + echo "Unzipping cluster certs zipfile..." + unzip /config/ssl/docker-cluster.zip -d /config/ssl/docker-cluster + + echo "Move logstash certs to logstash config dir..." + mv /config/ssl/docker-cluster/logstash/* /config/logstash/ + echo "Move kibana certs to kibana config dir..." + mv /config/ssl/docker-cluster/kibana/* /config/kibana/ + echo "Move elasticsearch certs to elasticsearch config dir..." + mv /config/ssl/docker-cluster/elasticsearch/* /config/elasticsearch/ fi diff --git a/scripts/setup-kibana.sh b/scripts/setup-kibana.sh index e7b1851..074ca5b 100755 --- a/scripts/setup-kibana.sh +++ b/scripts/setup-kibana.sh @@ -17,12 +17,17 @@ es_url=https://elasticsearch:9200 while [[ "$(curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -o /dev/null -w '%{http_code}' $es_url)" != "200" ]]; do sleep 5 + echo "Wait for elasticsearch..." done # Set the password for the kibana user. -# REF: https://www.elastic.co/guide/en/x-pack/6.0/setting-up-authentication.html#set-built-in-user-passwords +# REF: https://www.elastic.co/guide/en/elastic-stack-overview/current/get-started-kibana-user.html +# -XPUT $es_url/_xpack/security/user/kibana/_password \ + +echo "Set password ${ELASTIC_PASSWORD} for the kibana user." + until curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -H 'Content-Type:application/json' \ - -XPUT $es_url/_xpack/security/user/kibana/_password \ + -XPUT $es_url/_security/user/kibana/_password \ -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" do sleep 2 @@ -37,6 +42,7 @@ if [ -f /config/kibana/kibana.keystore ]; then fi /usr/share/kibana/bin/kibana-keystore create echo "Setting elasticsearch.password: $ELASTIC_PASSWORD" -echo "$ELASTIC_PASSWORD" | /usr/share/kibana/bin/kibana-keystore add 'elasticsearch.password' -x +(echo "kibana" | /usr/share/kibana/bin/kibana-keystore add 'elasticsearch.username' -x) +(echo "$ELASTIC_PASSWORD" | /usr/share/kibana/bin/kibana-keystore add 'elasticsearch.password' -x) mv /usr/share/kibana/data/kibana.keystore /config/kibana/kibana.keystore diff --git a/scripts/setup-logstash.sh b/scripts/setup-logstash.sh index 84a7896..7525809 100755 --- a/scripts/setup-logstash.sh +++ b/scripts/setup-logstash.sh @@ -20,7 +20,7 @@ done # Set the password for the logstash user. # REF: https://www.elastic.co/guide/en/x-pack/6.0/setting-up-authentication.html#set-built-in-user-passwords until curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -H 'Content-Type:application/json' \ - -XPUT $es_url/_xpack/security/user/logstash_system/_password \ + -XPUT $es_url/_security/user/logstash_system/_password \ -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" do sleep 2 @@ -35,5 +35,7 @@ if [ -f /config/logstash/logstash.keystore ]; then fi echo "y" | /usr/share/logstash/bin/logstash-keystore create echo "Setting ELASTIC_PASSWORD..." -echo "$ELASTIC_PASSWORD" | /usr/share/logstash/bin/logstash-keystore add 'ELASTIC_PASSWORD' -x +#(echo "logstash_system" | /usr/share/logstash/bin/logstash-keystore add 'elasticsearch.username' -x) +(echo "$ELASTIC_PASSWORD" | /usr/share/logstash/bin/logstash-keystore add 'ELASTIC_PASSWORD' -x) + mv /usr/share/logstash/config/logstash.keystore /config/logstash/logstash.keystore diff --git a/scripts/setup.sh b/scripts/setup.sh index daccbf3..28907e4 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -11,12 +11,31 @@ fi PW=$(openssl rand -base64 16;) ELASTIC_PASSWORD="${ELASTIC_PASSWORD:-$PW}" export ELASTIC_PASSWORD -docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_elasticsearch -# setup kibana and logstash (and system passwords) -docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_kibana setup_logstash -# setup beats and apm server -docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_auditbeat setup_filebeat setup_heartbeat setup_metricbeat setup_packetbeat setup_apm_server +# setup elasticsearch (and system passwords) +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_elasticsearch + +# setup kibana (and system passwords) +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_kibana + +# setup logstash +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_logstash + +# setup beats +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_auditbeat + +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_filebeat + +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_heartbeat + +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_metricbeat + +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_packetbeat + +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_journalbeat + +# setup apm server +docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_apm_server printf "Setup completed successfully. To start the stack please run:\n\t docker-compose up -d\n" printf "\nIf you wish to remove the setup containers please run:\n\tdocker-compose -f docker-compose.yml -f docker-compose.setup.yml down --remove-orphans\n" diff --git a/setup.yml b/setup.yml old mode 100644 new mode 100755 index 2419a4f..ab1d682 --- a/setup.yml +++ b/setup.yml @@ -1,7 +1,7 @@ version: "3.6" services: setup: - image: docker/compose:1.21.2 + image: docker/compose:1.24.0 working_dir: "${PWD}" cap_add: ['SYS_ADMIN'] environment: