diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index 30bbd51e..824fd021 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -184,6 +184,8 @@ "- `query` _(the input question)_\n", "- `expected_answer` _(the ideal or ground truth answer)_\n", "- `expected_uri` _(the webpage url or more generally the uri that contains the answer to `query`)_.\n", + "- `user_metadata` _(optional user metadata passed to datastore route with user query. Can be one of [str, dict])_\n", + "- `parameters` _(optional session parameters to include with the user query. Can be one of [str, dict])_\n", "\n", "In addition to the required columns the RougeL metric can also use the following optional column:\n", "\n", @@ -191,12 +193,12 @@ "\n", "An example for the queryset can be seen in this table:\n", "\n", - "| conversation_id | turn_index | query | expected_answer | expected_uri |\n", - "| --- | --- | --- | --- | --- |\n", - "| 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france |\n", - "| 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris |\n", - "| 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common |\n", - "| 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus |\n", + "| conversation_id | turn_index | query | expected_answer | expected_uri | user_metadata | parameters\n", + "| --- | --- | --- | --- | --- | --- | --- |\n", + "| 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france | None | None |\n", + "| 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris | {\"some_end_user_key\": \"some_value\"} | {\"param1\": 1, \"param2\": \"some_string\"} |\n", + "| 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common | None | None |\n", + "| 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus | None | None |\n", "\n", "---\n", "\n", @@ -224,14 +226,14 @@ "source": [ "# @markdown `run this cell to load data manually`\n", "INPUT_SCHEMA_REQUIRED_COLUMNS = [\n", - " \"conversation_id\", \"turn_index\", \"query\", \"expected_answer\", \"expected_uri\", \"user_metadata\"\n", + " \"conversation_id\", \"turn_index\", \"query\", \"expected_answer\", \"expected_uri\", \"user_metadata\", \"parameters\"\n", "]\n", "\n", "sample_df = pd.DataFrame(columns=INPUT_SCHEMA_REQUIRED_COLUMNS)\n", "\n", - "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None]\n", - "sample_df.loc[1] = [\"1\", 1 ,\"Which is the cheapest plan?\", \"Basic plan\", \"www.google.com\", None]\n", - "sample_df.loc[2] = [\"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\", None]\n", + "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None, None]\n", + "sample_df.loc[1] = [\"1\", 1 ,\"Who is the CEO?\", \"The CEO is Matt Reinjes.\", \"www.yeti.com\", {\"some_end_user_key\": \"some_value\"}, {\"param1\": 1, \"param2\": \"some_string\"}]\n", + "sample_df.loc[2] = [\"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\", None, None]\n", "queryset = sample_df" ] }, diff --git a/src/dfcx_scrapi/tools/datastore_scraper.py b/src/dfcx_scrapi/tools/datastore_scraper.py index 6f3cd721..332bb3ab 100644 --- a/src/dfcx_scrapi/tools/datastore_scraper.py +++ b/src/dfcx_scrapi/tools/datastore_scraper.py @@ -36,6 +36,7 @@ "expected_answer", "expected_uri", "user_metadata", + "parameters" ] def load_spreadsheet( @@ -191,16 +192,25 @@ def scrape_detect_intent( query: str, session_id: Union[str, None] = None, user_metadata: Union[str, None] = None, + parameters: Union[str, None] = None ) -> AgentResponse: if session_id is None: session_id = self.sessions.build_session_id(self.agent_id) if user_metadata: try: - user_metadata = json.loads(user_metadata) + if isinstance(user_metadata, str): + user_metadata = json.loads(user_metadata) except ValueError as err: raise UserWarning("Invalid user metadata") from err + if parameters: + try: + if isinstance(parameters, str): + parameters = json.loads(parameters) + except ValueError as err: + raise UserWarning("Invalid parameters") from err + response = self.sessions.detect_intent( agent_id=self.agent_id, session_id=session_id, @@ -208,6 +218,7 @@ def scrape_detect_intent( language_code=self.language_code, end_user_metadata=user_metadata, populate_data_store_connection_signals=True, + parameters=parameters ) ar = AgentResponse() @@ -224,7 +235,10 @@ def run( def scrape(row): result = self.scrape_detect_intent( - row["query"], row["session_id"], row["user_metadata"] + query=row["query"], + session_id=row["session_id"], + user_metadata=row["user_metadata"], + parameters=row["parameters"] ) progress_bar.update()