Merge pull request #265 from ahysing/concurrent-thanos-compact

Added multithreading support to thanos compact

CHANGELOG.md

@@ -34,6 +34,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ### Added
 
+- [#265](https://github.com/thanos-io/kube-thanos/pull/265) Added support for multithreaded thanos compact.
 - [#237](https://github.com/thanos-io/kube-thanos/pull/237) Add new bucket replicate component.
 - [#245](https://github.com/thanos-io/kube-thanos/pull/245) Support scraping config reloader sidecar for ruler.
 - [#251](https://github.com/thanos-io/kube-thanos/pull/251) Add support for extraEnv (custom environment variables) to all components.

examples/all/manifests/thanos-compact-shard0-statefulSet.yaml

@@ -58,6 +58,8 @@ spec:
         - --retention.resolution-5m=0d
         - --retention.resolution-1h=0d
         - --delete-delay=48h
+        - --compact.concurrency=1
+        - --downsample.concurrency=1
         - --downsampling.disable
         - |-
           --tracing.config="config":

examples/all/manifests/thanos-compact-shard1-statefulSet.yaml

@@ -58,6 +58,8 @@ spec:
         - --retention.resolution-5m=0d
         - --retention.resolution-1h=0d
         - --delete-delay=48h
+        - --compact.concurrency=1
+        - --downsample.concurrency=1
         - --downsampling.disable
         - |-
           --tracing.config="config":

examples/all/manifests/thanos-compact-shard2-statefulSet.yaml

@@ -58,6 +58,8 @@ spec:
         - --retention.resolution-5m=0d
         - --retention.resolution-1h=0d
         - --delete-delay=48h
+        - --compact.concurrency=1
+        - --downsample.concurrency=1
         - --downsampling.disable
         - |-
           --tracing.config="config":

examples/all/manifests/thanos-compact-statefulSet.yaml

@@ -55,6 +55,8 @@ spec:
         - --retention.resolution-5m=0d
         - --retention.resolution-1h=0d
         - --delete-delay=48h
+        - --compact.concurrency=1
+        - --downsample.concurrency=1
         - --downsampling.disable
         - --deduplication.replica-label=prometheus_replica
         - --deduplication.replica-label=rule_replica

jsonnet/kube-thanos/kube-thanos-compact-default-params.libsonnet

@@ -17,9 +17,11 @@
   retentionResolutionRaw: '0d',
   retentionResolution5m: '0d',
   retentionResolution1h: '0d',
+  compactConcurrency: 1,
+  deduplicationReplicaLabels: [],
   deleteDelay: '48h',
   disableDownsampling: false,
-  deduplicationReplicaLabels: [],
+  downsampleConcurrency: 1,
   ports: {
     http: 10902,
   },

jsonnet/kube-thanos/kube-thanos-compact.libsonnet

@@ -6,6 +6,8 @@ function(params) {
   // Combine the defaults and the passed params to make the component's config.
   config:: defaults + params,
   // Safety checks for combined config of defaults and params
+  assert std.isNumber(tc.config.compactConcurrency),
+  assert std.isNumber(tc.config.downsampleConcurrency),
   assert std.isNumber(tc.config.replicas) && (tc.config.replicas == 0 || tc.config.replicas == 1) : 'thanos compact replicas can only be 0 or 1',
   assert std.isObject(tc.config.resources),
   assert std.isObject(tc.config.volumeClaimTemplate),
@@ -65,6 +67,8 @@ function(params) {
         '--retention.resolution-5m=' + tc.config.retentionResolution5m,
         '--retention.resolution-1h=' + tc.config.retentionResolution1h,
         '--delete-delay=' + tc.config.deleteDelay,
+        '--compact.concurrency=' + tc.config.compactConcurrency,
+        '--downsample.concurrency=' + tc.config.downsampleConcurrency,
       ] + (
         if tc.config.disableDownsampling then ['--downsampling.disable'] else []
       ) + (