From 0f8dcd6155b6d4956329ad225549278a23f5e799 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 11 Jan 2025 20:17:22 +0700 Subject: [PATCH 1/6] Introduce patches with external kafka --- patches/README.md | 8 ++ patches/_lib.sh | 16 +++ patches/external-kafka.sh | 227 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+) create mode 100644 patches/README.md create mode 100755 patches/_lib.sh create mode 100755 patches/external-kafka.sh diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000000..1dd5153a2a --- /dev/null +++ b/patches/README.md @@ -0,0 +1,8 @@ +# Self-Hosted Sentry Patches + +Other than the default self-hosted Sentry installation, sometimes users can leverage their existing infrastructure to help them +with limited resources. "Patches", or you might call this like a "plugin system", is a bunch of bash scripts that can be used +to modify the existing configuration to achieve the desired goal. + +Beware that this is very experimental and might not work as expected. +Use it at your own risk. diff --git a/patches/_lib.sh b/patches/_lib.sh new file mode 100755 index 0000000000..42555742c0 --- /dev/null +++ b/patches/_lib.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail +test "${DEBUG:-}" && set -x + +function patch_file() { + target="$1" + content="$2" + if [[ -f "$target" ]]; then + echo "🙈 Patching $target ..." + patch -p1 <"$content" + else + echo "🙊 Skipping $target ..." + fi +} + diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh new file mode 100755 index 0000000000..b5e192881f --- /dev/null +++ b/patches/external-kafka.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash + +# Kafka plays a very significant role on Sentry's infrastructure, from ingesting +# to processing events until they end up on ClickHouse or filesystem for permanent +# storage. Since Kafka may require a significant amount of resources on the server +# it may make sense to split it from the main Sentry installation. This can be +# particularly appealing if you already have a managed Kafka cluster set up. +# +# Sentry (the company) itself uses a Kafka cluster on production with a very +# tailored setup, especially for authentication. Some Kafka configuration options +# (such as `SASL_SSL` security protocol) might not be available for some services, +# but since everything is open source, you are encouraged to contribute to +# implement those missing things. +# +# If you are using authentication, make sure that the user is able to create +# new topics. As of now, there is no support for prefixed topic name. +# +# PLEASE NOTE: This patch will only modify the existing configuration files. +# You will have to run `./install.sh` again to apply the changes. + +source patches/_lib.sh + +# If `sentry/sentry.conf.py` exists, we'll modify it. +# Otherwise, we'll use `sentry/sentry.conf.example.py`. +SENTRY_CONFIG_PY="sentry/sentry.conf.py" +if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then + SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" +fi + +ENV_FILE=".env.custom" +if [[ -f "$ENV_FILE" ]]; then + ENV_FILE=".env" +fi + +RELAY_CONFIG_YML="relay/config.yml" +if [[ ! -f "$RELAY_CONFIG_YML" ]]; then + RELAY_CONFIG_YML="relay/config.example.yml" +fi + +patch -p1 $SENTRY_CONFIG_PY <<"EOF" +@@ -136,9 +136,17 @@ + SENTRY_CACHE = "sentry.cache.redis.RedisCache" + + DEFAULT_KAFKA_OPTIONS = { +- "bootstrap.servers": "kafka:9092", ++ "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), + "message.max.bytes": 50000000, + "socket.timeout.ms": 1000, ++ "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ++ # If you don't use any of these options below, you can remove them or set them to `None`. ++ "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. ++ "sasl.username": env("KAFKA_SASL_USERNAME", None), ++ "sasl.password": env("KAFKA_SASL_PASSWORD", None), ++ "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. + } + + SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" +EOF + +# Add additional Kafka options to the ENV_FILE +cat <>"$ENV_FILE" +################################################################################ +## Additional External Kafka options +################################################################################ +KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 +# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL +KAFKA_SECURITY_PROTOCOL=PLAINTEXT +# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. +# KAFKA_SASL_MECHANISM=PLAIN +# KAFKA_SASL_USERNAME=username +# KAFKA_SASL_PASSWORD=password +# Put your certificates on the \`certificates/kafka\` directory. +# The certificates will be mounted as read-only volumes. +# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem +# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem +# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key +EOF + +#TODO: Modify relay config file + +COMPOSE_OVERRIDE_CONTENT=$(cat <<-EOF +x-snuba-defaults: &snuba_defaults + environment: + DEFAULT_BROKERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + KAFKA_SSL_CA_PATH: \${KAFKA_SSL_CA_LOCATION:-} + KAFKA_SSL_CERT_PATH: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} + KAFKA_SSL_KEY_PATH: \${KAFKA_SSL_KEY_LOCATION:-} + KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} + KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} + KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} + volumes: + - ./certificates/kafka:/kafka-certificates:ro +x-sentry-defaults: &sentry_defaults + environment: + KAFKA_BOOTSTRAP_SERVERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + KAFKA_SSL_CA_LOCATION: \${KAFKA_SSL_CA_LOCATION:-} + KAFKA_SSL_CERTIFICATE_LOCATION: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} + KAFKA_SSL_KEY_LOCATION: \${KAFKA_SSL_KEY_LOCATION:-} + KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} + KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} + KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} + volumes: + - ./certificates/kafka:/kafka-certificates:ro + +services: + kafka: !reset null + vroom: + environment: + SENTRY_KAFKA_BROKERS_PROFILING: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SENTRY_KAFKA_BROKERS_OCCURRENCES: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SENTRY_KAFKA_BROKERS_SPANS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SENTRY_KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + SENTRY_KAFKA_SSL_CA_PATH: \${KAFKA_SSL_CA_LOCATION:-} + SENTRY_KAFKA_SSL_CERT_PATH: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} + SENTRY_KAFKA_SSL_KEY_PATH: \${KAFKA_SSL_KEY_LOCATION:-} + SENTRY_KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} + SENTRY_KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} + SENTRY_KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} + volumes: + - ./certificates/kafka:/kafka-certificates:ro + snuba-api: + <<: *snuba_defaults + snuba-errors-consumer: + <<: *snuba_defaults + snuba-outcomes-consumer: + <<: *snuba_defaults + snuba-outcomes-billing-consumer: + <<: *snuba_defaults + snuba-group-attributes-consumer: + <<: *snuba_defaults + snuba-replacer: + <<: *snuba_defaults + snuba-subscription-consumer-events: + <<: *snuba_defaults + snuba-transactions-consumer: + <<: *snuba_defaults + snuba-replays-consumer: + <<: *snuba_defaults + snuba-issue-occurrence-consumer: + <<: *snuba_defaults + snuba-metrics-consumer: + <<: *snuba_defaults + snuba-subscription-consumer-transactions: + <<: *snuba_defaults + snuba-subscription-consumer-metrics: + <<: *snuba_defaults + snuba-generic-metrics-distributions-consumer: + <<: *snuba_defaults + snuba-generic-metrics-sets-consumer: + <<: *snuba_defaults + snuba-generic-metrics-counters-consumer: + <<: *snuba_defaults + snuba-generic-metrics-gauges-consumer: + <<: *snuba_defaults + snuba-profiling-profiles-consumer: + <<: *snuba_defaults + snuba-profiling-functions-consumer: + <<: *snuba_defaults + snuba-spans-consumer: + <<: *snuba_defaults + web: + <<: *sentry_defaults + cron: + <<: *sentry_defaults + worker: + <<: *sentry_defaults + events-consumer: + <<: *sentry_defaults + attachments-consumer: + <<: *sentry_defaults + post-process-forwarder-errors: + <<: *sentry_defaults + subscription-consumer-events: + <<: *sentry_defaults + transactions-consumer: + <<: *sentry_defaults + metrics-consumer: + <<: *sentry_defaults + generic-metrics-consumer: + <<: *sentry_defaults + billing-metrics-consumer: + <<: *sentry_defaults + ingest-replay-recordings: + <<: *sentry_defaults + ingest-occurrences: + <<: *sentry_defaults + ingest-profiles: + <<: *sentry_defaults + ingest-monitors: + <<: *sentry_defaults + ingest-feedback-events: + <<: *sentry_defaults + monitors-clock-tick: + <<: *sentry_defaults + monitors-clock-tasks: + <<: *sentry_defaults + post-process-forwarder-transactions: + <<: *sentry_defaults + post-process-forwarder-issue-platform: + <<: *sentry_defaults + subscription-consumer-transactions: + <<: *sentry_defaults + subscription-consumer-metrics: + <<: *sentry_defaults + subscription-consumer-generic-metrics: + <<: *sentry_defaults +EOF +) +if [[ -f "docker-compose.override.yml" ]]; then + echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" + echo "$COMPOSE_OVERRIDE_CONTENT" +else + echo "🚨 docker-compose.override.yml does not exist. Creating it now." + echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml +fi + +echo "" +echo "" +echo "--------------------------------------------------------------" +echo "- Finished patching external-kafka.sh" +echo "- Modify the Kafka credentials on your $ENV_FILE file" +echo "- Then, run ./install.sh" +echo "--------------------------------------------------------------" From 62d5a4f905735f327a24c91043c49107638a8450 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 11 Jan 2025 20:21:55 +0700 Subject: [PATCH 2/6] Fix pre-commit hooks --- patches/external-kafka.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index b5e192881f..08e797bab5 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -24,23 +24,23 @@ source patches/_lib.sh # Otherwise, we'll use `sentry/sentry.conf.example.py`. SENTRY_CONFIG_PY="sentry/sentry.conf.py" if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then - SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" + SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" fi ENV_FILE=".env.custom" -if [[ -f "$ENV_FILE" ]]; then - ENV_FILE=".env" +if [[ ! -f "$ENV_FILE" ]]; then + ENV_FILE=".env" fi RELAY_CONFIG_YML="relay/config.yml" if [[ ! -f "$RELAY_CONFIG_YML" ]]; then - RELAY_CONFIG_YML="relay/config.example.yml" + RELAY_CONFIG_YML="relay/config.example.yml" fi patch -p1 $SENTRY_CONFIG_PY <<"EOF" @@ -136,9 +136,17 @@ SENTRY_CACHE = "sentry.cache.redis.RedisCache" - + DEFAULT_KAFKA_OPTIONS = { - "bootstrap.servers": "kafka:9092", + "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), @@ -55,12 +55,13 @@ patch -p1 $SENTRY_CONFIG_PY <<"EOF" + "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. + "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. } - + SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" EOF # Add additional Kafka options to the ENV_FILE cat <>"$ENV_FILE" + ################################################################################ ## Additional External Kafka options ################################################################################ @@ -80,7 +81,8 @@ EOF #TODO: Modify relay config file -COMPOSE_OVERRIDE_CONTENT=$(cat <<-EOF +COMPOSE_OVERRIDE_CONTENT=$( + cat <<-EOF x-snuba-defaults: &snuba_defaults environment: DEFAULT_BROKERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} @@ -211,11 +213,11 @@ services: EOF ) if [[ -f "docker-compose.override.yml" ]]; then - echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" - echo "$COMPOSE_OVERRIDE_CONTENT" + echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" + echo "$COMPOSE_OVERRIDE_CONTENT" else - echo "🚨 docker-compose.override.yml does not exist. Creating it now." - echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml + echo "🚨 docker-compose.override.yml does not exist. Creating it now." + echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml fi echo "" From f9946894901aca54dbcc114b8e250540b4b75b8b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 11 Jan 2025 20:31:23 +0700 Subject: [PATCH 3/6] Patch relay config file --- patches/_lib.sh | 1 - patches/external-kafka.sh | 31 ++++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/patches/_lib.sh b/patches/_lib.sh index 42555742c0..46e55d3e25 100755 --- a/patches/_lib.sh +++ b/patches/_lib.sh @@ -13,4 +13,3 @@ function patch_file() { echo "🙊 Skipping $target ..." fi } - diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index 08e797bab5..3c4802f6ca 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -79,7 +79,24 @@ KAFKA_SECURITY_PROTOCOL=PLAINTEXT # KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key EOF -#TODO: Modify relay config file +patch -p1 $RELAY_CONFIG_YML <<"EOF" +@@ -7,8 +7,15 @@ + processing: + enabled: true + kafka_config: +- - {name: "bootstrap.servers", value: "kafka:9092"} ++ - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} + - {name: "message.max.bytes", value: 50000000} # 50MB ++ - {name: "security.protocol", value: "PLAINTEXT"} ++ - {name: "sasl.mechanism", value: "PLAIN"} ++ - {name: "sasl.username", value: "username"} ++ - {name: "sasl.password", value: "password"} ++ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} ++ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} ++ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} + redis: redis://redis:6379 + geoip_path: "/geoip/GeoLite2-City.mmdb" +EOF COMPOSE_OVERRIDE_CONTENT=$( cat <<-EOF @@ -124,6 +141,9 @@ services: SENTRY_KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} volumes: - ./certificates/kafka:/kafka-certificates:ro + relay: + volumes: + - ./certificates/kafka:/kafka-certificates:ro snuba-api: <<: *snuba_defaults snuba-errors-consumer: @@ -222,8 +242,9 @@ fi echo "" echo "" -echo "--------------------------------------------------------------" -echo "- Finished patching external-kafka.sh" -echo "- Modify the Kafka credentials on your $ENV_FILE file" +echo "------------------------------------------------------------------------" +echo "- Finished patching external-kafka.sh. Some things you'll need to do:" +echo "- Modify the Kafka credentials on your $ENV_FILE file." +echo "- Modify the Kafka credentials on your $RELAY_CONFIG_YML file." echo "- Then, run ./install.sh" -echo "--------------------------------------------------------------" +echo "------------------------------------------------------------------------" From 8e5aa7d0f5bea074ace92e1beb85a251acefd03b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 12 Jan 2025 07:56:18 +0700 Subject: [PATCH 4/6] Documentation for patches stuff --- patches/README.md | 24 +++++++++++++++++++++--- patches/external-kafka.sh | 30 ++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/patches/README.md b/patches/README.md index 1dd5153a2a..0319a3729d 100644 --- a/patches/README.md +++ b/patches/README.md @@ -1,8 +1,26 @@ # Self-Hosted Sentry Patches Other than the default self-hosted Sentry installation, sometimes users can leverage their existing infrastructure to help them -with limited resources. "Patches", or you might call this like a "plugin system", is a bunch of bash scripts that can be used +with limited resources. "Patches", or you might call this like a "plugin system", is a collection of bash scripts that can be used to modify the existing configuration to achieve the desired goal. -Beware that this is very experimental and might not work as expected. -Use it at your own risk. +> [!WARNING] +> Beware that this is very experimental and might not work as expected. +> +> **Use it at your own risk!** + +## How to use patches + +The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. +They should be run from the root directory. For example, the `external-kafka.sh` patch should be run as: +```bash +./patches/external-kafka.sh +``` + +Some patches might require additional steps to be taken, like providing credentials or additional TLS certificates. + +## Official support + +Sentry employees are not obliged to provide dedicated support for patches, but they can help by providing information to move us forward. We encourage the community to contribute for any bug fixes or improvements. + +See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information. diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index 3c4802f6ca..00f4883868 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -22,6 +22,7 @@ source patches/_lib.sh # If `sentry/sentry.conf.py` exists, we'll modify it. # Otherwise, we'll use `sentry/sentry.conf.example.py`. +# This kind of conditional logic will be used with other files. SENTRY_CONFIG_PY="sentry/sentry.conf.py" if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" @@ -37,6 +38,8 @@ if [[ ! -f "$RELAY_CONFIG_YML" ]]; then RELAY_CONFIG_YML="relay/config.example.yml" fi +## XXX(aldy505): Create the diff by running `diff -u sentry/sentry.conf.py sentry/sentry.conf.example.py`. +## But you'll need to have your own `sentry/sentry.conf.py` file with the changes already set. patch -p1 $SENTRY_CONFIG_PY <<"EOF" @@ -136,9 +136,17 @@ SENTRY_CACHE = "sentry.cache.redis.RedisCache" @@ -60,7 +63,11 @@ patch -p1 $SENTRY_CONFIG_PY <<"EOF" EOF # Add additional Kafka options to the ENV_FILE -cat <>"$ENV_FILE" +# Only patch this when "KAFKA_BOOTSTRAP_SERVERS" is not set. +if [[ grep -q "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}" ]]; then + echo "🚨 Skipping patching of ${ENV_FILE}" +else + cat <>"$ENV_FILE" ################################################################################ ## Additional External Kafka options @@ -78,6 +85,7 @@ KAFKA_SECURITY_PROTOCOL=PLAINTEXT # KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem # KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key EOF +fi patch -p1 $RELAY_CONFIG_YML <<"EOF" @@ -7,8 +7,15 @@ @@ -88,12 +96,12 @@ patch -p1 $RELAY_CONFIG_YML <<"EOF" + - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} - {name: "message.max.bytes", value: 50000000} # 50MB + - {name: "security.protocol", value: "PLAINTEXT"} -+ - {name: "sasl.mechanism", value: "PLAIN"} -+ - {name: "sasl.username", value: "username"} -+ - {name: "sasl.password", value: "password"} -+ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} -+ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} -+ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} ++ - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used. ++ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used. redis: redis://redis:6379 geoip_path: "/geoip/GeoLite2-City.mmdb" EOF @@ -244,7 +252,9 @@ echo "" echo "" echo "------------------------------------------------------------------------" echo "- Finished patching external-kafka.sh. Some things you'll need to do:" -echo "- Modify the Kafka credentials on your $ENV_FILE file." -echo "- Modify the Kafka credentials on your $RELAY_CONFIG_YML file." -echo "- Then, run ./install.sh" +echo "- 1. Modify the Kafka credentials on your $ENV_FILE file." +echo "- 2. Modify the Kafka credentials on your $RELAY_CONFIG_YML file." +echo "- 3. Run ./install.sh" +echo "-" +echo "- NOTE: Remove or comment the corresponding line if you don't use it." echo "------------------------------------------------------------------------" From 4135ee6139fd6d6915a95bdcaa5b875bd80ce885 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 12 Jan 2025 08:08:15 +0700 Subject: [PATCH 5/6] Provide more helpful information for Docker Compose Override file --- patches/README.md | 2 +- patches/external-kafka.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/patches/README.md b/patches/README.md index 0319a3729d..dfdf48f912 100644 --- a/patches/README.md +++ b/patches/README.md @@ -11,7 +11,7 @@ to modify the existing configuration to achieve the desired goal. ## How to use patches -The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. +The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. They should be run from the root directory. For example, the `external-kafka.sh` patch should be run as: ```bash ./patches/external-kafka.sh diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index 00f4883868..bbf17a3a43 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -241,7 +241,9 @@ services: EOF ) if [[ -f "docker-compose.override.yml" ]]; then - echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" + echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually with the following content." + echo "Changes: Reset the 'kafka' service block, add additional Kafka-related environment variables to 'vroom', 'relay', 'sentry'-related and 'snuba'-related services." + echo "" echo "$COMPOSE_OVERRIDE_CONTENT" else echo "🚨 docker-compose.override.yml does not exist. Creating it now." @@ -257,4 +259,6 @@ echo "- 2. Modify the Kafka credentials on your $RELAY_CONFIG_YML file." echo "- 3. Run ./install.sh" echo "-" echo "- NOTE: Remove or comment the corresponding line if you don't use it." +echo "- It is also safe to prune/delete 'sentry-kafka'" +echo "- and 'sentry-kafka-log' Docker volumes." echo "------------------------------------------------------------------------" From 9a204dda9214094a4a11e6082451d97ee5a7d467 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 12 Jan 2025 08:23:34 +0700 Subject: [PATCH 6/6] Fix grep command --- patches/external-kafka.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index bbf17a3a43..a024dcee28 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -64,8 +64,9 @@ EOF # Add additional Kafka options to the ENV_FILE # Only patch this when "KAFKA_BOOTSTRAP_SERVERS" is not set. -if [[ grep -q "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}" ]]; then - echo "🚨 Skipping patching of ${ENV_FILE}" +kafka_config_exists_on_env=$(grep -c "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}") +if [[ "$kafka_config_exists_on_env" -ne 0 ]]; then + echo "🚨 Skipping patching of ${ENV_FILE}. You may already have Kafka-related environment variables set." else cat <>"$ENV_FILE"