fix: Adding OTEL_EXPORTER_OTLP_ENDPOINT (#189)

ctrlplanedev · Nov 9, 2024 · 6637883 · 6637883
1 parent df01033
commit 6637883
Show file tree

Hide file tree

Showing 12 changed files with 744 additions and 733 deletions.
diff --git a/apps/webservice/Dockerfile b/apps/webservice/Dockerfile
@@ -20,6 +20,7 @@ COPY turbo.json turbo.json
 RUN pnpm add -g turbo
 
 COPY package.json package.json
+COPY .npmrc .npmrc
 COPY pnpm-*.yaml .
 
 COPY tooling/tailwind/package.json ./tooling/tailwind/package.json

diff --git a/apps/webservice/next.config.js b/apps/webservice/next.config.js
@@ -26,17 +26,7 @@ const config = {
 
   experimental: {
     instrumentationHook: true,
-    optimizePackageImports: ["bullmq", "googleapis"],
-    /** @see https://github.com/open-telemetry/opentelemetry-js/issues/4297 */
-    serverComponentsExternalPackages: [
-      "@opentelemetry/sdk-node",
-      "@opentelemetry/auto-instrumentations-node",
-      "@appsignal/opentelemetry-instrumentation-bullmq",
-      "@opentelemetry/exporter-trace-otlp-http",
-      "@opentelemetry/resources",
-      "@opentelemetry/sdk-node",
-      "@opentelemetry/semantic-conventions",
-    ],
+    optimizePackageImports: ["googleapis"],
   },
 
   async rewrites() {

diff --git a/apps/webservice/package.json b/apps/webservice/package.json
@@ -15,7 +15,6 @@
     "openapi": "ts-node tooling/openapi/merge.ts"
   },
   "dependencies": {
-    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.1",
     "@ctrlplane/api": "workspace:*",
     "@ctrlplane/auth": "workspace:*",
     "@ctrlplane/db": "workspace:*",
@@ -33,9 +32,8 @@
     "@octokit/rest": "catalog:",
     "@octokit/webhooks": "^13.3.0",
     "@octokit/webhooks-types": "^7.5.1",
-    "@opentelemetry/auto-instrumentations-node": "^0.50.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.53.0",
-    "@opentelemetry/sdk-node": "^0.53.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.52.1",
+    "@opentelemetry/sdk-node": "^0.54.2",
     "@t3-oss/env-nextjs": "catalog:",
     "@tabler/icons-react": "^3.17.0",
     "@tailwindcss/typography": "^0.5.13",

diff --git a/apps/webservice/src/instrumentation.node.ts → apps/webservice/src/instrumentation-node.ts b/apps/webservice/src/instrumentation.node.ts → apps/webservice/src/instrumentation-node.ts
@@ -44,10 +44,6 @@ const sdk = new NodeSDK({
     }),
   ],
   sampler: new AlwaysOnSampler(),
-  // sampler:
-  //   env.NODE_ENV === "development"
-  //     ? new AlwaysOnSampler()
-  //     : new TraceIdRatioBasedSampler(env.OTEL_SAMPLER_RATIO),
 });
 
 try {

diff --git a/apps/webservice/src/instrumentation.ts b/apps/webservice/src/instrumentation.ts
@@ -1,5 +1,5 @@
 export async function register() {
   // eslint-disable-next-line no-restricted-properties
   if (process.env.NEXT_RUNTIME === "nodejs")
-    await import("./instrumentation.node");
+    await import("./instrumentation-node");
 }
diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml
@@ -33,6 +33,19 @@ services:
     volumes:
       - redisinsight-data:/data
 
+  # otel-collector:
+  #   image: otel/opentelemetry-collector-contrib:latest
+  #   container_name: ctrlplane-otel-collector
+  #   command: ["--config=/etc/otel-collector-config.yaml"]
+  #   volumes:
+  #     - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+  #   ports:
+  #     - "4317:4317" # OTLP gRPC
+  #     - "4318:4318" # OTLP HTTP
+  #     - "8125:8125" # StatsD
+  #     - "8888:8888" # Telemetry
+  #     - "9109:9109" # Prometheus endpoint
+
 volumes:
   db-data:
   redis-data:

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,4 +1,4 @@
-version: "3.8"
+version: "3.9"
 
 services:
   postgres:
@@ -26,52 +26,79 @@ services:
 
   migrations:
     platform: linux/amd64
-    image: ctrlplane/migrations:c656deb
+    image: ctrlplane/migrations:75685e0
     container_name: ctrlplane-migrations
     restart: "no"
     environment:
-      DATABASE_URL: postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane
+      POSTGRES_URL: "postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane"
     depends_on:
       - postgres
 
   webservice:
-    platform: linux/amd64
-    image: ctrlplane/webservice:c656deb
+    # platform: linux/amd64
+    # image: ctrlplane/webservice:75685e0
+    build:
+      context: .
+      dockerfile: ./apps/webservice/Dockerfile
+
     container_name: ctrlplane-webservice
     environment:
       BASE_URL: "http://127.0.0.1" # <set your instance url>
-      AUTH_SECRET: "<auth-secret>"
-      # AUTH_OIDC_ISSUER: "<oidc-issuer>"
-      # AUTH_OIDC_CLIENT_ID: "<oidc-client-id>"
-      # AUTH_OIDC_CLIENT_SECRET: "<oidc-client-secret>"
+      AUTH_SECRET: ${AUTH_SECRET}
+      # AUTH_OIDC_ISSUER: ${AUTH_OIDC_ISSUER}
+      # AUTH_OIDC_CLIENT_ID: ${AUTH_OIDC_CLIENT_ID}
+      # AUTH_OIDC_CLIENT_SECRET: ${AUTH_OIDC_CLIENT_SECRET}
+
+      AUTH_GOOGLE_CLIENT_ID: ${AUTH_GOOGLE_CLIENT_ID}
+      AUTH_GOOGLE_CLIENT_SECRET: ${AUTH_GOOGLE_CLIENT_SECRET}
 
       NEXTAUTH_URL: ${BASE_URL}
-      DATABASE_URL: postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
+      REDIS_URL: redis://redis:6379
+      POSTGRES_URL: postgresql://ctrlplane:ctrlplane@postgres:5432/ctrlplane
+      VARIABLES_AES_256_KEY: d2506d733ee210078461c08ee8e1605be75ed73b0941d4b513ab7b159c6fbcd9
     depends_on:
       - postgres
       - migrations
     ports:
-      - "8080:8080"
+      - "3000:3000"
 
   jobs:
     platform: linux/amd64
-    image: ctrlplane/jobs:c656deb
+    image: ctrlplane/jobs:75685e0
     container_name: ctrlplane-jobs
     environment:
-      POSTGRES_URL: postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane
+      POSTGRES_URL: "postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane"
+      REDIS_URL: "redis://redis:6379"
+      VARIABLES_AES_256_KEY: d2506d733ee210078461c08ee8e1605be75ed73b0941d4b513ab7b159c6fbcd9
     depends_on:
       - postgres
 
   event-worker:
     platform: linux/amd64
-    image: ctrlplane/event-worker:c656deb
+    image: ctrlplane/event-worker:75685e0
     container_name: ctrlplane-event-worker
     environment:
-      POSTGRES_URL: postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane
+      POSTGRES_URL: "postgres://ctrlplane:ctrlplane@postgres:5432/ctrlplane"
+      REDIS_URL: "redis://redis:6379"
+      VARIABLES_AES_256_KEY: d2506d733ee210078461c08ee8e1605be75ed73b0941d4b513ab7b159c6fbcd9
     depends_on:
       - postgres
       - redis
 
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: ctrlplane-otel-collector
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP
+      - "8125:8125" # StatsD
+      - "8888:8888" # Telemetry
+      - "9109:9109" # Prometheus endpoint
+
 volumes:
   postgres_data:
   redis_data:
diff --git a/otel-collector-config.yaml b/otel-collector-config.yaml
@@ -0,0 +1,80 @@
+connectors:
+  datadog/connector: {}
+exporters:
+  datadog/exporter:
+    api:
+      key: ${DD_API_KEY}
+      site: ${DD_SITE}
+  debug: {}
+  debug/detailed:
+    verbosity: detailed
+  prometheus:
+    endpoint: 0.0.0.0:9109
+extensions:
+  health_check: {}
+processors:
+  batch: {}
+  memory_limiter:
+    check_interval: 5s
+    limit_percentage: 80
+    spike_limit_percentage: 25
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+        include_metadata: true
+      http:
+        endpoint: 0.0.0.0:4318
+        include_metadata: true
+  statsd:
+    endpoint: 0.0.0.0:8125
+service:
+  extensions:
+    - health_check
+  pipelines:
+    logs/datadog:
+      exporters:
+        - datadog/exporter
+      processors:
+        - batch
+        - memory_limiter
+      receivers:
+        - otlp
+    metrics/datadog:
+      exporters:
+        - datadog/exporter
+      processors:
+        - batch
+        - memory_limiter
+      receivers:
+        - otlp
+        - datadog/connector
+    traces:
+      exporters:
+        - debug
+      processors:
+        - batch
+        - memory_limiter
+      receivers:
+        - otlp
+    traces/datadog:
+      exporters:
+        - datadog/connector
+        - datadog/exporter
+      processors:
+        - batch
+        - memory_limiter
+      receivers:
+        - otlp
+    logs:
+      exporters:
+        - debug
+      processors:
+        - batch
+        - memory_limiter
+      receivers:
+        - otlp
+  telemetry:
+    metrics:
+      address: 0.0.0.0:8888
diff --git a/packages/logger/package.json b/packages/logger/package.json
@@ -20,6 +20,7 @@
   },
   "dependencies": {
     "@colors/colors": "^1.6.0",
+    "@opentelemetry/winston-transport": "^0.7.0",
     "winston": "^3.14.2",
     "zod": "catalog:"
   },

diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts
@@ -1,4 +1,5 @@
 import colors from "@colors/colors/safe.js";
+import { OpenTelemetryTransportV3 } from "@opentelemetry/winston-transport";
 import * as winston from "winston";
 
 const { LOG_LEVEL, NODE_ENV } = process.env;
@@ -26,7 +27,10 @@ function createLogger(level: string) {
   return winston.createLogger({
     level,
     format: winston.format.combine(...format),
-    transports: [new winston.transports.Console()],
+    transports: [
+      new winston.transports.Console(),
+      new OpenTelemetryTransportV3(),
+    ],
   });
 }