Refactor of error handling

evaluation/EDA/run_infer.py

@@ -158,7 +158,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'success': test_result,
             'final_message': final_message,

evaluation/agent_bench/run_infer.py

@@ -283,7 +283,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'agent_answer': agent_answer,
             'final_answer': final_ans,

evaluation/aider_bench/run_infer.py

@@ -261,7 +261,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result,
     )
     return output

evaluation/biocoder/run_infer.py

@@ -311,7 +311,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result,
     )
     return output

evaluation/bird/run_infer.py

@@ -440,7 +440,7 @@ def execute_sql(db_path, sql):
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result,
     )
     return output

evaluation/browsing_delegation/run_infer.py

@@ -121,7 +121,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'query': instance.instruction,
             'action': last_delegate_action,

evaluation/gaia/run_infer.py

@@ -213,7 +213,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result,
     )
     return output

evaluation/gorilla/run_infer.py

@@ -121,7 +121,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'text': model_answer_raw,
             'correct': correct,

evaluation/gpqa/run_infer.py

@@ -302,7 +302,7 @@ def process_instance(
         metadata=metadata,
         history=state.history.compatibility_for_eval_history_pairs(),
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'result': test_result,
             'found_answers': found_answers,

evaluation/humanevalfix/run_infer.py

@@ -264,7 +264,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result,
     )
     return output

evaluation/integration_tests/run_infer.py

@@ -144,7 +144,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result.model_dump(),
     )
     return output

evaluation/logic_reasoning/run_infer.py

@@ -256,7 +256,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result=test_result,
     )
     return output

evaluation/miniwob/run_infer.py

@@ -173,7 +173,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'reward': reward,
         },

evaluation/mint/run_infer.py

@@ -212,7 +212,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'success': task_state.success if task_state else False,
         },

evaluation/swe_bench/run_infer.py

@@ -409,11 +409,11 @@ def process_instance(
 
         # if fatal error, throw EvalError to trigger re-run
         if (
-            state.last_error
-            and 'fatal error during agent execution' in state.last_error
+            state.get_last_error()
+            and 'fatal error during agent execution' in state.get_last_error()
             and 'stuck in a loop' not in state.last_error
         ):
-            raise EvalException('Fatal error detected: ' + state.last_error)
+            raise EvalException('Fatal error detected: ' + state.get_last_error())
 
         # ======= THIS IS SWE-Bench specific =======
         # Get git patch
@@ -450,7 +450,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
     )
     return output
 

evaluation/toolqa/run_infer.py

@@ -149,7 +149,7 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
     )
     return output
 

evaluation/webarena/run_infer.py

@@ -187,7 +187,7 @@ def process_instance(
         metadata=metadata,
         history=histories,
         metrics=metrics,
-        error=state.last_error if state and state.last_error else None,
+        error=state.get_last_error() if state and state.get_last_error() else None,
         test_result={
             'reward': reward,
         },

frontend/__tests__/components/chat/chat-interface.test.tsx

@@ -128,14 +128,14 @@ describe.skip("ChatInterface", () => {
         timestamp: new Date().toISOString(),
       },
       {
-        error: "Woops!",
+        error: true,
+        id: "",
         message: "Something went wrong",
       },
     ];
     renderChatInterface(messages);
 
     const error = screen.getByTestId("error-message");
-    expect(within(error).getByText("Woops!")).toBeInTheDocument();
     expect(within(error).getByText("Something went wrong")).toBeInTheDocument();
   });
 

frontend/src/components/AgentStatusBar.tsx

@@ -1,6 +1,7 @@
 import React, { useEffect } from "react";
 import { useTranslation } from "react-i18next";
 import { useSelector } from "react-redux";
+import toast from "react-hot-toast";
 import { I18nKey } from "#/i18n/declaration";
 import { RootState } from "#/store";
 import AgentState from "#/types/AgentState";
@@ -16,7 +17,7 @@ enum IndicatorColor {
 }
 
 function AgentStatusBar() {
-  const { t } = useTranslation();
+  const { t, i18n } = useTranslation();
   const { curAgentState } = useSelector((state: RootState) => state.agent);
   const { curStatusMessage } = useSelector((state: RootState) => state.status);
 
@@ -94,15 +95,27 @@ function AgentStatusBar() {
   const [statusMessage, setStatusMessage] = React.useState<string>("");
 
   React.useEffect(() => {
-    if (curAgentState === AgentState.LOADING) {
-      const trimmedCustomMessage = curStatusMessage.status.trim();
-      if (trimmedCustomMessage) {
-        setStatusMessage(t(trimmedCustomMessage));
-        return;
+    let message = curStatusMessage.message || "";
+    if (curStatusMessage?.id) {
+      const id = curStatusMessage.id.trim();
+      if (i18n.exists(id)) {
+        message = t(curStatusMessage.id.trim()) || message;
       }
     }
+    if (curStatusMessage?.type === "error") {
+      toast.error(message);
+      return;
+    }
+    if (curAgentState === AgentState.LOADING && message.trim()) {
+      setStatusMessage(message);
+    } else {
+      setStatusMessage(AgentStatusMap[curAgentState].message);
+    }
+  }, [curStatusMessage.id]);
+
+  React.useEffect(() => {
     setStatusMessage(AgentStatusMap[curAgentState].message);
-  }, [curAgentState, curStatusMessage.status]);
+  }, [curAgentState]);
 
   return (
     <div className="flex flex-col items-center">

frontend/src/components/chat-interface.tsx

@@ -73,7 +73,7 @@ export function ChatInterface() {
           isErrorMessage(message) ? (
             <ErrorMessage
               key={index}
-              error={message.error}
+              id={message.id}
               message={message.message}
             />
           ) : (

frontend/src/components/chat/message.d.ts

@@ -6,6 +6,7 @@ type Message = {
 };
 
 type ErrorMessage = {
-  error: string;
+  error: boolean;
+  id?: string;
   message: string;
 };

frontend/src/components/error-message.tsx

@@ -1,14 +1,41 @@
+import { useState, useEffect } from "react";
+import { useTranslation } from "react-i18next";
+
 interface ErrorMessageProps {
-  error: string;
+  id?: string;
   message: string;
 }
 
-export function ErrorMessage({ error, message }: ErrorMessageProps) {
+export function ErrorMessage({ id, message }: ErrorMessageProps) {
+  const { t, i18n } = useTranslation();
+  const [showDetails, setShowDetails] = useState(true);
+  const [headline, setHeadline] = useState("");
+  const [details, setDetails] = useState(message);
+
+  useEffect(() => {
+    if (id && i18n.exists(id)) {
+      setHeadline(t(id));
+      setDetails(message);
+      setShowDetails(false);
+    }
+  }, [id, message, i18n.language]);
+
   return (
     <div className="flex gap-2 items-center justify-start border-l-2 border-danger pl-2 my-2 py-2">
       <div className="text-sm leading-4 flex flex-col gap-2">
-        <p className="text-danger font-bold">{error}</p>
-        <p className="text-neutral-300">{message}</p>
+        {headline && <p className="text-danger font-bold">{headline}</p>}
+        {headline && (
+          <button
+            type="button"
+            onClick={() => setShowDetails(!showDetails)}
+            className="cursor-pointer text-left"
+          >
+            {showDetails
+              ? t("ERROR_MESSAGE$HIDE_DETAILS")
+              : t("ERROR_MESSAGE$SHOW_DETAILS")}
+          </button>
+        )}
+        {showDetails && <p className="text-neutral-300">{details}</p>}
       </div>
     </div>
   );

frontend/src/i18n/translation.json

@@ -1441,6 +1441,12 @@
     "fr": "Privé",
     "tr": "Özel"
   },
+  "ERROR_MESSAGE$SHOW_DETAILS": {
+    "en": "Show details"
+  },
+  "ERROR_MESSAGE$HIDE_DETAILS": {
+    "en": "Hide details"
+  },
   "STATUS$STARTING_RUNTIME": {
     "en": "Starting Runtime...",
     "zh-CN": "启动运行时...",
@@ -1510,5 +1516,17 @@
     "ar": "في انتظار جاهزية العميل...",
     "fr": "En attente que le client soit prêt...",
     "tr": "İstemcinin hazır olması bekleniyor..."
+  },
+  "STATUS$ERROR_LLM_AUTHENTICATION": {
+    "en": "Error authenticating with the LLM provider. Please check your API key"
+  },
+  "STATUS$ERROR_RUNTIME_DISCONNECTED": {
+    "en": "There was an error while connecting to the runtime. Please refresh the page."
+  },
+  "AGENT_ERROR$BAD_ACTION": {
+    "en": "Agent tried to execute a malformed action."
+  },
+  "AGENT_ERROR$ACTION_TIMEOUT": {
+    "en": "Action timed out."
   }
 }

frontend/src/routes/_oh.app.tsx

@@ -99,7 +99,7 @@
     const data = await retrieveLatestGitHubCommit(ghToken, repo);
     if (isGitHubErrorReponse(data)) {
       // TODO: Handle error
       console.error("Failed to retrieve latest commit", data);
     } else {
       [lastCommit] = data;
     }
@@ -185,21 +185,6 @@
     if (q) addIntialQueryToChat(q, files);
   }, [settings]);
 
-  const handleError = (message: string) => {
-    const [error, ...rest] = message.split(":");
-    const details = rest.join(":");
-    if (!details) {
-      dispatch(
-        addErrorMessage({
-          error: "An error has occured",
-          message: error,
-        }),
-      );
-    } else {
-      dispatch(addErrorMessage({ error, message: details }));
-    }
-  };
-
   const handleMessage = React.useCallback(
     (message: MessageEvent<WebSocket.Data>) => {
       // set token received from the server
@@ -225,7 +210,12 @@
         return;
       }
       if (isErrorObservation(parsed)) {
-        handleError(parsed.message);
+        dispatch(
+          addErrorMessage({
+            id: parsed.extras?.error_id,
+            message: parsed.message,
+          }),
+        );
         return;
       }