kubewharf · SOF3 · Sep 19, 2023 · Aug 11, 2023 · Aug 11, 2023 · Aug 15, 2023
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -136,7 +136,7 @@ jobs:
           TRACE_ID=$(grep "Location: /trace/" curl-output.http | cut -d/ -f3 | tr -d '\r')
 
           mkdir -p output/api/traces
-          for mode in ff{0,1,2,3}{0,1}00000{0,1}; do
+          for mode in ff{0,1,2,3}{0,1,5,6}000000; do
             mode_trace=${mode}${TRACE_ID:10}
             curl -o output/api/traces/$mode_trace http://localhost:16686/api/traces/$mode_trace
           done
@@ -172,12 +172,12 @@ jobs:
               <html>
                 <head>
                   <title>Redirecting to ${basename}</title>
-                  <meta http-equiv="refresh" content="0;URL='/kelemetry/trace/ff20000000$(cat $trace_dir/trace_id)'" />
+                  <meta http-equiv="refresh" content="0;URL='/kelemetry/trace/ff26000000$(cat $trace_dir/trace_id)'" />
                 </head>
               </html>
           EOF
-            mkdir -p output/trace/ff20000000$(cat $trace_dir/trace_id)
-            cp spa.html output/trace/ff20000000$(cat $trace_dir/trace_id)/index.html
+            mkdir -p output/trace/ff26000000$(cat $trace_dir/trace_id)
+            cp spa.html output/trace/ff26000000$(cat $trace_dir/trace_id)/index.html
             cp $trace_dir/api/traces/* output/api/traces/
           done
           mv output/dot-usage/* output/
@@ -261,7 +261,7 @@ jobs:
                 promises.push((async () => {
                   const traceName = await fs.readFile(`pages/${trace}/trace_name`, {encoding: "utf8"})
                   const partialTraceId = await fs.readFile(`pages/${trace}/trace_id`, {encoding: "utf8"})
-                  const traceId = "ff20000000" + partialTraceId
+                  const traceId = "ff26000000" + partialTraceId
 
                   console.log(`Loading trace ${traceName}`)
 

diff --git a/Makefile b/Makefile
@@ -34,6 +34,8 @@ else
 	LOG_FILE_ARG ?=
 endif
 
+LINKER_WORKER_COUNT ?= 1
+
 CONTROLLERS ?= audit-consumer,audit-producer,audit-webhook,event-informer,annotation-linker,owner-linker,resource-object-tag,resource-event-tag,diff-decorator,diff-controller,diff-api,pprof,jaeger-storage-plugin,jaeger-redirect-server,kelemetrix
 ifeq ($(CONTROLLERS),)
 	ENABLE_ARGS ?=
@@ -69,7 +71,7 @@ endif
 .PHONY: run dump-rotate test usage dot kind stack pre-commit
 run: output/kelemetry $(DUMP_ROTATE_DEP)
 	GIN_MODE=debug \
-		./output/kelemetry \
+		$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) \
 		--mq=local \
 		--audit-consumer-partition=$(PARTITIONS) \
 		--http-address=0.0.0.0 \
@@ -85,6 +87,7 @@ run: output/kelemetry $(DUMP_ROTATE_DEP)
 		--log-file=$(LOG_FILE) \
 		--aggregator-pseudo-span-global-tags=runId=$(RUN_ID) \
 		--aggregator-event-span-global-tags=run=$(RUN_ID) \
+		--linker-worker-count=$(LINKER_WORKER_COUNT) \
 		--pprof-addr=:6030 \
 		--diff-cache=$(ETCD_OR_LOCAL) \
 		--diff-cache-etcd-endpoints=127.0.0.1:2379 \
@@ -95,6 +98,7 @@ run: output/kelemetry $(DUMP_ROTATE_DEP)
 		--span-cache-etcd-endpoints=127.0.0.1:2379 \
 		--tracer-otel-endpoint=$(OTEL_EXPORTER_OTLP_ENDPOINT) \
 		--tracer-otel-insecure \
+		--object-cache-size=16777216 \
 		--jaeger-cluster-names=$(CLUSTER_NAME) \
 		--jaeger-storage-plugin-address=0.0.0.0:17271 \
 		--jaeger-backend=jaeger-storage \
@@ -117,15 +121,15 @@ test:
 	go test -v -race -coverpkg=./pkg/... -coverprofile=coverage.out $(INTEGRATION_ARG) $(BUILD_ARGS) ./pkg/...
 
 usage: output/kelemetry
-	./output/kelemetry --usage=USAGE.txt
+	$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) --usage=USAGE.txt
 
 dot: output/kelemetry
-	./output/kelemetry --dot=depgraph.dot
+	$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) --dot=depgraph.dot
 	dot -Tpng depgraph.dot >depgraph.png
 	dot -Tsvg depgraph.dot >depgraph.svg
 
 output/kelemetry: go.mod go.sum $(shell find -type f -name "*.go")
-	go build -v $(RACE_ARG) -ldflags=$(LDFLAGS) -o $@ $(BUILD_ARGS) .
+	go build -v $(RACE_ARG) -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) -o $@ $(BUILD_ARGS) .
 
 kind:
 	kind delete cluster --name tracetest

diff --git a/charts/kelemetry/templates/_helpers.yaml b/charts/kelemetry/templates/_helpers.yaml
@@ -111,6 +111,7 @@ span-cache-etcd-prefix: {{ .Values.aggregator.spanCache.etcd.prefix | toJson }}
 {{- end }}
 
 {{/* LINKERS */}}
+linker-worker-count: {{ .Values.linkers.workerCount }}
 annotation-linker-enable: {{ .Values.linkers.annotation }}
 owner-linker-enable: {{ .Values.linkers.ownerReference }}
 

diff --git a/charts/kelemetry/values.yaml b/charts/kelemetry/values.yaml
@@ -323,6 +323,9 @@ aggregator:
 
 # Linkers associated objects together.
 linkers:
+  # Maximum number of concurrent link jobs.
+  # Each link job runs each of the linkers for a single object in series.
+  workerCount: 8
   # Enable the owner linker, which links objects based on native owner references.
   ownerReference: true
   # Enable the annotation linker, which links objects based on the `kelemetry.kubewharf.io/parent-link` annotation.

diff --git a/dev.docker-compose.yaml b/dev.docker-compose.yaml
@@ -54,6 +54,7 @@ services:
       - 127.0.0.1:17272:17271
     volumes:
       - badger:/mnt/badger
+    restart: always
 
   # Web frontend for raw trace database view.
   jaeger-query-raw:

diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md
@@ -20,8 +20,12 @@ The "Service" field selects one of the display modes:
   Additional information is available in event tags.
 - `timeline`: All events are displayed as children of the root object.
 
-By default, the whole trace is displayed, including parent and sibling objects of the searched object.
-Enabling the `exclusive` option limits the output to the subtree under the object matched in the search.
+By default, only the trace for a single object is displayed.
+More traces are available by configuration:
+
+- `full tree`: view the full tree from the deepest ancestor
+- `ancestors`: include transitive owners
+- `children`: include child objects
 
 ### Cluster
 

diff --git a/hack/tfconfig.yaml b/hack/tfconfig.yaml
@@ -13,22 +13,21 @@ configs:
       - kind: Batch
         batchName: initial
       - kind: ExtractNestingVisitor
-        matchesNestLevel:
-          oneOf: []
-          negate: true
+        matchesPseudoType:
+          oneOf: ["object"]
       - kind: Batch
         batchName: final
   - id: "20000000"
     name: "tracing"
     steps:
       - kind: Batch
         batchName: initial
-      - kind: ExtractNestingVisitor
-        matchesNestLevel:
-          oneOf: ["object"]
-          negate: true
       - kind: Batch
         batchName: collapse
+      - kind: ExtractNestingVisitor
+        matchesPseudoType:
+          oneOf: ["linkClass"]
+        matchesName:
       - kind: CompactDurationVisitor
       - kind: Batch
         batchName: final
@@ -37,28 +36,48 @@ configs:
     steps:
       - kind: Batch
         batchName: initial
-      - kind: ExtractNestingVisitor
-        matchesNestLevel:
-          oneOf: ["object"]
-          negate: true
       - kind: Batch
         batchName: collapse
       - kind: GroupByTraceSourceVisitor
         shouldBeGrouped:
           oneOf: ["event"]
-          negate: true
+          then: false
       - kind: CompactDurationVisitor
       - kind: Batch
         batchName: final
 
 modifiers:
+  # Multiple active link-selector modifiers are additive (union)
   "01000000":
-    displayName: exclusive
-    modifierName: exclusive
+    # the entire tree under the deepest (up to 3 levels) ancestor
+    displayName: full tree
+    modifierName: link-selector
+    args:
+      modifierClass: owner-ref
+      includeSiblings: false
+      upwardDistance: 3
+      downwardDistance: 3
+  "02000000":
+    # include all ancestors (up to 3) but not siblings of ancestors
+    displayName: ancestors
+    modifierName: link-selector
+    args:
+      modifierClass: owner-ref
+      includeSiblings: true
+      upwardDistance: 3
+  "04000000":
+    # the entire subtree under this object
+    displayName: owned objects
+    modifierName: link-selector
+    args:
+      modifierClass: owner-ref
+      ifAll:
+        - linkClass: children
+          fromChild: false
+      downwardDistance: 3
 
 # Uncomment to enable extension trace from apiserver
 #   "00000001":
-#     # We want to run extension modifiers after exclusive modifier to avoid fetching unused traces
 #     displayName: apiserver trace
 #     modifierName: extension
 #     args:
@@ -122,9 +141,6 @@ batches:
   - name: collapse
     steps:
       - kind: CollapseNestingVisitor
-        shouldCollapse:
-          oneOf: []
-          negate: true
         tagMappings:
           "audit":
             - fromSpanTag: "userAgent"