From f4920669d9c12dfda8b274231c555316c4ad646c Mon Sep 17 00:00:00 2001 From: Alexander Samusev Date: Sat, 6 Aug 2022 15:02:50 +0200 Subject: [PATCH] feat: Add Loki to collect container logs --- README.md | 22 ++++++++++++++- caddy/Caddyfile | 7 +++-- docker-compose.exporters.yml | 14 +++++++++- docker-compose.yml | 34 +++++++++++++++++++++++ loki/loki-config.yaml | 47 +++++++++++++++++++++++++++++++ loki/promtail-config.yaml | 54 ++++++++++++++++++++++++++++++++++++ 6 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 loki/loki-config.yaml create mode 100644 loki/promtail-config.yaml diff --git a/README.md b/README.md index a984fcc8..44a877c6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # dockprom -A monitoring solution for Docker hosts and containers with [Prometheus](https://prometheus.io/), [Grafana](http://grafana.org/), [cAdvisor](https://github.com/google/cadvisor), +A monitoring solution for Docker hosts and containers with [Prometheus](https://prometheus.io/), [Grafana](http://grafana.org/), [cAdvisor](https://github.com/google/cadvisor), [Grafana Loki](https://grafana.com/oss/loki/) [NodeExporter](https://github.com/prometheus/node_exporter) and alerting with [AlertManager](https://github.com/prometheus/alertmanager). ***If you're looking for the Docker Swarm version please go to [stefanprodan/swarmprom](https://github.com/stefanprodan/swarmprom)*** @@ -383,3 +383,23 @@ To run the grafana container as `user: 104` change your `docker-compose.yml` lik labels: org.label-schema.group: "monitoring" ``` + +## Reading logs with Loki + +To make Loki work with other containers add `logging` section to each container in your `docker-compose.yml` file: + +```yml + logging: + options: + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" +``` + +To add Loki in Grafana: + +- Add data source in Grafana (Configuration -> Data sources -> Add Data Source): + ``` + Name: Loki + URL: http://loki:3100 + ``` +- Go to `Explore` and choose `Loki` as a Data Source +- Use this query for gettings logs: `{container_name="your_container"}` diff --git a/caddy/Caddyfile b/caddy/Caddyfile index c8995396..204e426b 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -16,10 +16,13 @@ basicauth /* { {$ADMIN_USER} {$ADMIN_PASSWORD_HASH} } - + reverse_proxy pushgateway:9091 } :3000 { reverse_proxy grafana:3000 -} \ No newline at end of file +} +:3100 { + reverse_proxy loki:3100 +} diff --git a/docker-compose.exporters.yml b/docker-compose.exporters.yml index cb806e91..f2eb11c9 100644 --- a/docker-compose.exporters.yml +++ b/docker-compose.exporters.yml @@ -1,5 +1,9 @@ version: '2.1' +x-logging: &default-logging + options: + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" + services: nodeexporter: @@ -35,5 +39,13 @@ services: network_mode: host labels: org.label-schema.group: "monitoring" - + logging: *default-logging + promtail: + image: grafana/promtail:2.6.0 + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/log:/var/log:ro + - ./loki:/etc/promtail + command: -config.file=/etc/promtail/promtail-config.yml + logging: *default-logging diff --git a/docker-compose.yml b/docker-compose.yml index 022f4bb7..e991dc54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,9 @@ version: '2.1' +x-logging: &default-logging + options: + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" + networks: monitor-net: driver: bridge @@ -30,6 +34,7 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging alertmanager: image: prom/alertmanager:v0.24.0 @@ -46,6 +51,7 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging nodeexporter: image: prom/node-exporter:v1.3.1 @@ -66,6 +72,7 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging cadvisor: image: gcr.io/cadvisor/cadvisor:v0.44.0 @@ -86,6 +93,7 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging grafana: image: grafana/grafana:9.0.5 @@ -105,6 +113,7 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging pushgateway: image: prom/pushgateway:v1.4.3 @@ -116,6 +125,29 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging + + loki: + image: grafana/loki:2.6.0 + expose: + - 3100 + volumes: + - ./loki:/etc/loki + command: -config.file=/etc/loki/loki-config.yaml + networks: + - monitor-net + logging: *default-logging + + promtail: + image: grafana/promtail:2.6.0 + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/log:/var/log:ro + - ./loki:/etc/promtail + command: -config.file=/etc/promtail/promtail-config.yml + networks: + - monitor-net + logging: *default-logging caddy: image: caddy:2.3.0 @@ -125,6 +157,7 @@ services: - "9090:9090" - "9093:9093" - "9091:9091" + - "3100:3100" volumes: - ./caddy:/etc/caddy environment: @@ -136,3 +169,4 @@ services: - monitor-net labels: org.label-schema.group: "monitoring" + logging: *default-logging diff --git a/loki/loki-config.yaml b/loki/loki-config.yaml new file mode 100644 index 00000000..91e16a8b --- /dev/null +++ b/loki/loki-config.yaml @@ -0,0 +1,47 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +limits_config: + ingestion_rate_mb: 1024 + ingestion_burst_size_mb: 1024 + +ruler: + alertmanager_url: http://localhost:9093 + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false diff --git a/loki/promtail-config.yaml b/loki/promtail-config.yaml new file mode 100644 index 00000000..c67c97c2 --- /dev/null +++ b/loki/promtail-config.yaml @@ -0,0 +1,54 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + # replace "loki" with server address for external hosts + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: +- job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log + +- job_name: containers + static_configs: + - targets: + - localhost + labels: + job: containerlogs + __path__: /var/lib/docker/containers/*/*log + + pipeline_stages: + - json: + expressions: + output: log + stream: stream + timestamp: time + attrs: + - json: + expressions: + tag: + source: attrs + - regex: + expression: (?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])) + source: tag + - timestamp: + format: RFC3339Nano + source: time + - labels: + tag: + stream: + image_name: + container_name: + image_id: + container_id: + - output: + source: output