feat: Add Loki to collect container logs

README.md

@@ -1,6 +1,6 @@
 # dockprom
 
-A monitoring solution for Docker hosts and containers with [Prometheus](https://prometheus.io/), [Grafana](http://grafana.org/), [cAdvisor](https://github.com/google/cadvisor),
+A monitoring solution for Docker hosts and containers with [Prometheus](https://prometheus.io/), [Grafana](http://grafana.org/), [cAdvisor](https://github.com/google/cadvisor), [Grafana Loki](https://grafana.com/oss/loki/)
 [NodeExporter](https://github.com/prometheus/node_exporter) and alerting with [AlertManager](https://github.com/prometheus/alertmanager).
 
 ***If you're looking for the Docker Swarm version please go to [stefanprodan/swarmprom](https://github.com/stefanprodan/swarmprom)***
@@ -383,3 +383,23 @@ To run the grafana container as `user: 104` change your `docker-compose.yml` lik
     labels:
       org.label-schema.group: "monitoring"
 ```
+
+## Reading logs with Loki
+
+To make Loki work with other containers add `logging` section to each container in your `docker-compose.yml` file:
+
+```yml
+    logging:
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+```
+
+To add Loki in Grafana:
+
+- Add data source in Grafana (Configuration -> Data sources -> Add Data Source):
+  ```
+  Name: Loki
+  URL: http://loki:3100
+  ```
+- Go to `Explore` and choose `Loki` as a Data Source
+- Use this query for gettings logs: `{container_name="your_container"}`

caddy/Caddyfile

@@ -16,10 +16,13 @@
     basicauth /* {
         {$ADMIN_USER} {$ADMIN_PASSWORD_HASH}
     }
-    
+
     reverse_proxy pushgateway:9091
 }
 
 :3000 {
     reverse_proxy grafana:3000
-}
+}
+:3100 {
+    reverse_proxy loki:3100
+}

docker-compose.exporters.yml

@@ -1,5 +1,9 @@
 version: '2.1'
 
+x-logging: &default-logging
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+
 services:
 
   nodeexporter:
@@ -35,5 +39,13 @@ services:
     network_mode: host
     labels:
       org.label-schema.group: "monitoring"
-
+    logging: *default-logging
 
+  promtail:
+    image: grafana/promtail:2.6.0
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - ./loki:/etc/promtail
+    command: -config.file=/etc/promtail/promtail-config.yml
+    logging: *default-logging

docker-compose.yml

@@ -1,5 +1,9 @@
 version: '2.1'
 
+x-logging: &default-logging
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+
 networks:
   monitor-net:
     driver: bridge
@@ -30,6 +34,7 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging
 
   alertmanager:
     image: prom/alertmanager:v0.24.0
@@ -46,6 +51,7 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging
 
   nodeexporter:
     image: prom/node-exporter:v1.3.1
@@ -66,6 +72,7 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging
 
   cadvisor:
     image: gcr.io/cadvisor/cadvisor:v0.44.0
@@ -86,6 +93,7 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging
 
   grafana:
     image: grafana/grafana:9.0.5
@@ -105,6 +113,7 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging
 
   pushgateway:
     image: prom/pushgateway:v1.4.3
@@ -116,6 +125,29 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging
+
+  loki:
+    image: grafana/loki:2.6.0
+    expose:
+      - 3100
+    volumes:
+      - ./loki:/etc/loki
+    command: -config.file=/etc/loki/loki-config.yaml
+    networks:
+      - monitor-net
+    logging: *default-logging
+
+  promtail:
+    image: grafana/promtail:2.6.0
+    volumes:
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro
+      - ./loki:/etc/promtail
+    command: -config.file=/etc/promtail/promtail-config.yml
+    networks:
+      - monitor-net
+    logging: *default-logging
 
   caddy:
     image: caddy:2.3.0
@@ -125,6 +157,7 @@ services:
       - "9090:9090"
       - "9093:9093"
       - "9091:9091"
+      - "3100:3100"
     volumes:
       - ./caddy:/etc/caddy
     environment:
@@ -136,3 +169,4 @@ services:
       - monitor-net
     labels:
       org.label-schema.group: "monitoring"
+    logging: *default-logging

loki/loki-config.yaml

@@ -0,0 +1,47 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  ingestion_rate_mb: 1024
+  ingestion_burst_size_mb: 1024
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

loki/promtail-config.yaml

@@ -0,0 +1,54 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  # replace "loki" with server address for external hosts
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
+
+- job_name: containers
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: containerlogs
+      __path__: /var/lib/docker/containers/*/*log
+
+  pipeline_stages:
+  - json:
+      expressions:
+        output: log
+        stream: stream
+        timestamp: time
+        attrs:
+  - json:
+      expressions:
+        tag:
+      source: attrs
+  - regex:
+      expression: (?P<image_name>(?:[^|]*[^|])).(?P<container_name>(?:[^|]*[^|])).(?P<image_id>(?:[^|]*[^|])).(?P<container_id>(?:[^|]*[^|]))
+      source: tag
+  - timestamp:
+      format: RFC3339Nano
+      source: time
+  - labels:
+      tag:
+      stream:
+      image_name:
+      container_name:
+      image_id:
+      container_id:
+  - output:
+      source: output