Improve resiliency in the DefaultManyReconciler processing loop (#1234)

Netflix · Mar 21, 2022 · 6d56e2a · 6d56e2a
1 parent b281e23
commit 6d56e2a
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 19 deletions.
diff --git a/...a/com/netflix/titus/common/framework/simplereconciler/internal/DefaultManyReconciler.java b/...a/com/netflix/titus/common/framework/simplereconciler/internal/DefaultManyReconciler.java
@@ -216,31 +216,32 @@ public Flux<List<SimpleReconcilerEvent<DATA>>> changes(String clientId) {
 
     private void doSchedule(long delayMs) {
         reconcilerWorker.schedule(() -> {
-            // Self-terminate
-            if (stateRef.get() == ReconcilerState.Closed) {
-                reconcilerWorker.dispose();
-                return;
-            }
+            long startTimeMs = clock.wallTime();
+            long startTimeNs = clock.nanoTime();
 
-            if (stateRef.get() == ReconcilerState.Closing && tryClose()) {
-                reconcilerWorker.dispose();
-                return;
-            }
+            try {
+                // Self-terminate
+                if (stateRef.get() == ReconcilerState.Closed) {
+                    reconcilerWorker.dispose();
+                    return;
+                }
 
-            connectEventListeners();
+                if (stateRef.get() == ReconcilerState.Closing && tryClose()) {
+                    reconcilerWorker.dispose();
+                    return;
+                }
 
-            long startTimeMs = clock.wallTime();
-            long startTimeNs = clock.nanoTime();
-            boolean fullCycle = (startTimeMs - lastLongCycleTimestamp) >= longCycleMs;
+                connectEventListeners();
 
-            if (fullCycle) {
-                lastLongCycleTimestamp = startTimeMs;
-            }
+                boolean fullCycle = (startTimeMs - lastLongCycleTimestamp) >= longCycleMs;
+
+                if (fullCycle) {
+                    lastLongCycleTimestamp = startTimeMs;
+                }
 
-            try {
                 doProcess(fullCycle);
                 doSchedule(quickCycleMs);
-            } catch (Exception e) {
+            } catch (Throwable e) {
                 metrics.evaluated(clock.nanoTime() - startTimeNs, e);
                 logger.warn("Unexpected error in the reconciliation loop", e);
                 doSchedule(longCycleMs);

diff --git a/...m/netflix/titus/common/framework/simplereconciler/internal/ReconcilerExecutorMetrics.java b/...m/netflix/titus/common/framework/simplereconciler/internal/ReconcilerExecutorMetrics.java
@@ -72,7 +72,7 @@ void evaluated(long executionTimeNs) {
         lastEvaluationTimestamp = clock.wallTime();
     }
 
-    void evaluated(long executionTimeNs, Exception error) {
+    void evaluated(long executionTimeNs, Throwable error) {
         registry.timer(evaluationId.withTag("error", error.getClass().getSimpleName())).record(executionTimeNs, TimeUnit.NANOSECONDS);
     }