From 79ad92c51b41c52a03ffe72d517d499900c51c61 Mon Sep 17 00:00:00 2001
From: Ilia Babanov <ilia.babanov@databricks.com>
Date: Thu, 12 Dec 2024 12:59:51 +0100
Subject: [PATCH] Do not load too many events (#1484)

## Changes
While preloading updates in order to generate datasets and their
schemas,
it's possible to be stuck for a while in the loading (e.g. in the case
of continuous pipelines with a lot of events).


## Tests
<!-- How is this tested? -->
---
 .../src/bundle/BundlePipelinesManager.ts                 | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts
index 5f9999784..688d19518 100644
--- a/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts
+++ b/packages/databricks-vscode/src/bundle/BundlePipelinesManager.ts
@@ -67,6 +67,8 @@ type SourceLocation = {
     notebook_cell_number?: number;
 };
 
+const MAX_EVENTS_TO_LOAD = 1000;
+
 export class BundlePipelinesManager {
     private disposables: Disposable[] = [];
     private readonly triggeredState: Map<string, PipelineState> = new Map();
@@ -303,11 +305,16 @@ export class BundlePipelinesManager {
                 pipelineId,
                 runs
             );
+            let loadedEventsCount = 0;
             for await (const event of listing) {
                 const runState = runs.get(event.origin?.update_id ?? "");
                 if (runState?.events) {
                     runState.events.push(event);
                 }
+                loadedEventsCount++;
+                if (loadedEventsCount >= MAX_EVENTS_TO_LOAD) {
+                    break;
+                }
             }
             const extractedData = extractPipelineDatasets(
                 new Set(runs.values())
@@ -358,8 +365,6 @@ export class BundlePipelinesManager {
         if (oldestUpdateTime) {
             const timestamp = new Date(oldestUpdateTime).toISOString();
             listEventsOptions.filter = `timestamp >= '${timestamp}'`;
-        } else {
-            listEventsOptions.max_results = 100;
         }
         return client.pipelines.listPipelineEvents(listEventsOptions);
     }