feat: set log_level confluent-kafka-go

topfreegames · Jun 16, 2023 · f20867f · f20867f
1 parent a1e9028
commit f20867f
Show file tree

Hide file tree

Showing 17 changed files with 274 additions and 83 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -22,15 +22,15 @@ FROM golang:1.10-alpine AS dependencies
 
 Label MAINTAINER="TFG Co <[email protected]>"
 
-ENV LIBRDKAFKA_VERSION 0.11.5
+ENV LIBRDKAFKA_VERSION 1.4.2
 ENV CPLUS_INCLUDE_PATH /usr/local/include
 ENV LIBRARY_PATH /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 WORKDIR /go/src/github.com/topfreegames/pusher
 
 RUN apk add --no-cache make git g++ bash python wget pkgconfig && \
-    wget -O /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz && \
+    wget -O /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/confluentinc/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz && \
     tar -xzf /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz -C /root && \
     cd /root/librdkafka-${LIBRDKAFKA_VERSION} && \
     ./configure && make && make install && make clean && ./configure --clean && \

diff --git a/Dockerfile.ci-test b/Dockerfile.ci-test
@@ -20,18 +20,21 @@
 
 FROM golang:1.10
 
-ENV LIBRDKAFKA_VERSION 0.11.5
+ENV LIBRDKAFKA_VERSION 1.4.2
 ENV CPLUS_INCLUDE_PATH /usr/local/include
 ENV LIBRARY_PATH /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
 ENV GOLANGCI_LINT_VERSION 1.15.0
 ENV GINKGO_VERSION 1.4.0
 
-RUN apt-get install make git g++ bash python wget pkg-config
+RUN sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list \
+    && sed -i 's/security.debian.org/archive.debian.org/g' /etc/apt/sources.list \
+    && sed -i '$ d' /etc/apt/sources.list
+RUN apt install -y make git g++ bash python wget pkg-config
 RUN apt update
 RUN apt install -y postgresql
 RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
-RUN wget --no-check-certificate -O /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz && \
+RUN wget --no-check-certificate -O /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/confluentinc/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz && \
     tar -xzf /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz -C /root && \
     cd /root/librdkafka-${LIBRDKAFKA_VERSION} && \
     ./configure && make && make install && make clean && ./configure --clean

diff --git a/Dockerfile.local b/Dockerfile.local
@@ -20,16 +20,19 @@
 
 FROM golang:1.10
 
-ENV LIBRDKAFKA_VERSION 0.11.5
+ENV LIBRDKAFKA_VERSION 1.4.2
 ENV CPLUS_INCLUDE_PATH /usr/local/include
 ENV LIBRARY_PATH /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
-ENV KAFKA_DEV_VERSION 3.3.2
+ENV KAFKA_DEV_VERSION 3.4.1
 ENV KAFKA_SCRIPTS_VERSION 2.13-${KAFKA_DEV_VERSION}
 ENV GOLANGCI_LINT_VERSION 1.15.0
 ENV GINKGO_VERSION 1.4.0
 
-RUN apt-get install \
+RUN sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list \
+    && sed -i 's/security.debian.org/archive.debian.org/g' /etc/apt/sources.list \
+    && sed -i '$ d' /etc/apt/sources.list
+RUN apt install -y \
     make \
     git \
     g++ \
@@ -38,7 +41,7 @@ RUN apt-get install \
     wget \
     pkg-config
 RUN export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
-RUN wget --no-check-certificate -O /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz \
+RUN wget --no-check-certificate -O /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/confluentinc/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz \
     && tar -xzf /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz -C /root \
     && cd /root/librdkafka-${LIBRDKAFKA_VERSION} \
     && ./configure && make && make install && make clean && ./configure --clean

diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Gopkg.toml b/Gopkg.toml
@@ -1,6 +1,6 @@
 [[constraint]]
   name = "github.com/confluentinc/confluent-kafka-go"
-  version = "0.11.0"
+  version = "1.4.2"
 
 [[constraint]]
   branch = "master"

diff --git a/Makefile b/Makefile
@@ -32,20 +32,13 @@ setup-ci-deps:
 
 setup-ci: dep setup-ci-deps
 
-wait-for-pg:
-	@until docker exec pusher_postgres_1 pg_isready; do echo 'Waiting for Postgres...' && sleep 1; done
-	@sleep 2
-
-deps: start-deps wait-for-pg
-
 start-deps:
 	@echo "Starting dependencies..."
-	@docker-compose --project-name pusher up -d
-	@while [ "`echo "health" | nc 127.0.0.1 40002`" != "health: up" ]; do echo "Waiting for StatsD to come up..." && sleep 1; done
+	@docker compose --project-name pusher up --wait --renew-anon-volumes --build
 	@echo "Dependencies started successfully."
 
 stop-deps:
-	@docker-compose --project-name pusher down
+	@docker compose --project-name pusher down --remove-orphans --volumes
 
 rtfd:
 	@rm -rf docs/_build
@@ -62,7 +55,7 @@ apns:
 	@go run main.go apns --certificate=./tls/_fixtures/certificate-valid.pem
 
 local-deps:
-	@docker-compose --project-name pusher up -d
+	@docker compose --project-name pusher up --wait --renew-anon-volumes --build
 
 setup:
 	# Ensuring librdkafka is installed in Mac OS
@@ -101,7 +94,7 @@ test-coverage-html cover:
 test-coverage-write-html:
 	@go tool cover -html=_build/test-coverage-all.out -o _build/test-coverage.html
 
-test-services: stop-deps deps test-db-drop test-db-create
+test-services: stop-deps start-deps test-db-drop test-db-create
 	@echo "Required test services are up."
 
 test-db-drop:
@@ -130,14 +123,14 @@ run-integration-test:
 	@echo "=               Running integration tests...             ="
 	@echo "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
 	@echo
-	@ginkgo -v -r -tags=integration --randomizeAllSpecs --randomizeSuites --focus="\[Integration\].*" .
+	@ginkgo -v -r --randomizeSuites --focus="\[Integration\].*" .
 	@echo
 	@echo "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
 	@echo "=               Integration tests finished.              ="
 	@echo "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
 	@echo
 
-test-integration integration func: deps test-db-drop test-db-create run-integration-test
+test-integration: start-deps test-db-drop test-db-create run-integration-test
 
 lint:
 	@golangci-lint run
@@ -147,26 +140,34 @@ build-image-dev:
 
 lint-container-dev: build-image-dev
 	@docker run \
+		--rm \
 		--volume "${PWD}":/go/src/github.com/topfreegames/pusher \
 		pusher:local golangci-lint run
 
 build-container-dev: build-image-dev
 	@docker run \
+		--rm \
 		--volume "${PWD}":/go/src/github.com/topfreegames/pusher \
 		pusher:local bash -c 'dep ensure && make build'
 
 unit-test-container-dev: build-image-dev
 	@docker run \
+		--rm \
 		--volume "${PWD}":/go/src/github.com/topfreegames/pusher \
 		pusher:local bash -c 'dep ensure && make unit'
 
 start-deps-container-dev:
 	@echo "Starting dependencies..."
-	@docker-compose -f docker-compose-container-dev.yml --project-name pusher up -d
-	@while [ "`echo "health" | nc 127.0.0.1 40002`" != "health: up" ]; do echo "Waiting for StatsD to come up..." && sleep 1; done
-	@$(MAKE) wait-for-pg
+	@docker compose -f docker-compose-container-dev.yml --project-name pusher up --wait --renew-anon-volumes --build
 	@echo "Dependencies started successfully."
 
 integration-test-container-dev: build-image-dev start-deps-container-dev test-db-drop test-db-create
-	@docker run -t -i --network pusher_default -e CONFIG_FILE="../config/docker_test.yaml" pusher:local make run-integration-test
+	@docker run \
+		-t \
+		-i \
+		--rm \
+		--network pusher_default \
+		-e CONFIG_FILE="../config/docker_test.yaml" \
+		--volume "${PWD}":/go/src/github.com/topfreegames/pusher \
+		pusher:local make run-integration-test
 	@$(MAKE) stop-deps
diff --git a/README.md b/README.md
@@ -256,7 +256,7 @@ make integration
 ```
 
 If you are running integration tests locally with the most recent librdkafka version (such as installed by brew) some of them will fail due to incompatible librdkafka version. 
-Tests should work for librdkafka v0.11.5.
+Tests should work for librdkafka v1.4.2.
 
 ### Benchmark
 

diff --git a/config/default.yaml b/config/default.yaml
@@ -61,15 +61,15 @@ invalidToken:
     connectionTimeout: 100
 feedbackListeners:
   queue:
-      topics:
-        - "^push-[^-_]+-(apns|gcm)-feedbacks"
-      brokers: "localhost:9941"
-      group: testGroup
-      sessionTimeout: 6000
-      fetch.min.bytes: 1
-      fetch.wait.max.ms: 100
-      offsetResetStrategy: latest
-      handleAllMessagesBeforeExiting: true
+    topics:
+      - "^push-[^-_]+-(apns|gcm)-feedbacks"
+    brokers: "localhost:9941"
+    group: testGroup
+    sessionTimeout: 6000
+    fetch.min.bytes: 1
+    fetch.wait.max.ms: 100
+    offsetResetStrategy: latest
+    handleAllMessagesBeforeExiting: true
   broker:
     invalidTokenChan:
       size: 999

diff --git a/config/docker_test.yaml b/config/docker_test.yaml
@@ -63,15 +63,15 @@ invalidToken:
     connectionTimeout: 100
 feedbackListeners:
   queue:
-      topics:
-        - "^push-[^-_]+-(apns|gcm)-feedbacks"
-      brokers: "kafka:9092"
-      group: testGroup
-      sessionTimeout: 6000
-      fetch.min.bytes: 1
-      fetch.wait.max.ms: 100
-      offsetResetStrategy: latest
-      handleAllMessagesBeforeExiting: true
+    topics:
+      - "^push-[^-_]+-(apns|gcm)-feedbacks"
+    brokers: "kafka:9092"
+    group: testGroup
+    sessionTimeout: 6000
+    fetch.min.bytes: 1
+    fetch.wait.max.ms: 100
+    offsetResetStrategy: latest
+    handleAllMessagesBeforeExiting: true
   broker:
     invalidTokenChan:
       size: 999
@@ -90,4 +90,3 @@ feedbackListeners:
       maxRetries: 3
       database: push
       connectionTimeout: 100
-
diff --git a/config/test.yaml b/config/test.yaml
@@ -63,15 +63,15 @@ invalidToken:
     connectionTimeout: 100
 feedbackListeners:
   queue:
-      topics:
-        - "^push-[^-_]+-(apns|gcm)-feedbacks"
-      brokers: "localhost:9941"
-      group: testGroup
-      sessionTimeout: 6000
-      fetch.min.bytes: 1
-      fetch.wait.max.ms: 100
-      offsetResetStrategy: latest
-      handleAllMessagesBeforeExiting: true
+    topics:
+      - "^push-[^-_]+-(apns|gcm)-feedbacks"
+    brokers: "localhost:9941"
+    group: testGroup
+    sessionTimeout: 6000
+    fetch.min.bytes: 1
+    fetch.wait.max.ms: 100
+    offsetResetStrategy: latest
+    handleAllMessagesBeforeExiting: true
   broker:
     invalidTokenChan:
       size: 999
@@ -90,4 +90,3 @@ feedbackListeners:
       maxRetries: 3
       database: push
       connectionTimeout: 100
-
diff --git a/debian-install-librdkafka.sh b/debian-install-librdkafka.sh
@@ -6,7 +6,7 @@ apt-get install -y gcc g++ python
 
 rm -rf ./librdkafka
 
-git clone --depth 1 --branch "v0.11.3" https://github.com/edenhill/librdkafka.git
+git clone --depth 1 --branch "v1.4.2" https://github.com/confluentinc/librdkafka.git
 (
     cd librdkafka
     ./configure