Skip to content

Commit

Permalink
feat: add support for flow invoke; clean up creds passing in evals
Browse files Browse the repository at this point in the history
  • Loading branch information
@kmaphoenix
kmaphoenix committed Oct 7, 2024
1 parent 6476985 commit f9f8157
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 23 deletions.
56 changes: 42 additions & 14 deletions src/dfcx_scrapi/core/sessions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from dfcx_scrapi.core import scrapi_base
from dfcx_scrapi.core import tools
from dfcx_scrapi.core import playbooks

from dfcx_scrapi.core import flows

# logging config
logging.basicConfig(
Expand All @@ -48,7 +48,8 @@ def __init__(
agent_id: str = None,
session_id: str = None,
tools_map: Dict[str, str] = None,
playbooks_map: Dict[str, str] = None
playbooks_map: Dict[str, str] = None,
flows_map: Dict[str, str] = None
):
super().__init__(
creds_path=creds_path, creds_dict=creds_dict,
Expand All @@ -59,6 +60,7 @@ def __init__(
self.agent_id = agent_id
self.tools_map = tools_map
self.playbooks_map = playbooks_map
self.flows_map = flows_map

@property
def session_id(self):
Expand Down Expand Up @@ -122,18 +124,28 @@ def get_tool_params(self, params: maps.MapComposite):
def get_playbook_name(self, playbook_id: str):
agent_id = self.parse_agent_id(playbook_id)
if not self.playbooks_map:
playbook_client = playbooks.Playbooks(agent_id)
playbook_client = playbooks.Playbooks(
agent_id=agent_id, creds=self.creds
)
self.playbooks_map = playbook_client.get_playbooks_map(agent_id)

return self.playbooks_map[playbook_id]

def get_tool_name(self, tool_use: types.example.ToolUse) -> str:
agent_id = self.parse_agent_id(tool_use.tool)
if not self.tools_map:
tool_client = tools.Tools()
tool_client = tools.Tools(creds=self.creds)
self.tools_map = tool_client.get_tools_map(agent_id)
return self.tools_map[tool_use.tool]

def get_flow_name(self, flow_id: str):
agent_id = self.parse_agent_id(flow_id)
if not self.flows_map:
flow_client = flows.Flows(creds=self.creds)
self.flows_map = flow_client.get_flows_map(agent_id)

return self.flows_map[flow_id]

def collect_tool_responses(
self, res: types.session.QueryResult
) -> List[Dict[str, str]]:
Expand Down Expand Up @@ -168,20 +180,36 @@ def collect_playbook_responses(
)
}
)
else:
# If no playbook invocation was found
# return the current Playbook
if len(res.generative_info.current_playbooks) > 0:
playbook_responses.append(
{
"playbook_name": self.get_playbook_name(
res.generative_info.current_playbooks[-1]
# If no playbook invocation was found try to return the current
# Playbook
elif len(res.generative_info.current_playbooks) > 0:
playbook_responses.append(
{
"playbook_name": self.get_playbook_name(
res.generative_info.current_playbooks[-1]
)
}
)
}
)

return playbook_responses

def collect_flow_responses(
self, res: types.session.QueryResult
) -> List[Dict[str, str]]:
"""Gather all the flow repsonses into a list of dicts."""
flow_responses = []
for action in res.generative_info.action_tracing_info.actions:
if action.flow_invocation:
flow_responses.append(
{
"flow_name": self.get_flow_name(
action.flow_invocation.flow
)
}
)

return flow_responses

def build_session_id(
self, agent_id: str = None, overwrite: bool = True
) -> str:
Expand Down
75 changes: 66 additions & 9 deletions src/dfcx_scrapi/tools/evaluations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,13 @@ def __init__(

print("Initializing Vertex AI...")
self.init_vertex(self.agent_id)
self.s = Sessions(agent_id=self.agent_id, tools_map=tools_map)
self.p = Playbooks(agent_id=self.agent_id, playbooks_map=playbooks_map)
self.t = Tools(agent_id=self.agent_id, tools_map=tools_map)
self.s = Sessions(
agent_id=self.agent_id, tools_map=tools_map, creds=self.creds)
self.p = Playbooks(
agent_id=self.agent_id,
playbooks_map=playbooks_map, creds=self.creds)
self.t = Tools(
agent_id=self.agent_id, tools_map=tools_map, creds=self.creds)
self.ar = AgentResponse()

self.generation_model = self.model_setup(generation_model)
Expand All @@ -94,7 +98,12 @@ def __init__(
def clean_outputs(df: pd.DataFrame) -> pd.DataFrame:
"""Clean final output dataframe."""
# drop cols used for response mapping
df = df.drop(columns=["utterance_pair", "tool_pair", "playbook_pair"])
df = df.drop(columns=[
"utterance_pair",
"tool_pair",
"playbook_pair",
"flow_pair"
])
value_map = {}
for col, dtype in zip(df.columns, df.dtypes):
if dtype in ["string", "object"]:
Expand All @@ -108,10 +117,10 @@ def clean_outputs(df: pd.DataFrame) -> pd.DataFrame:

@staticmethod
def process_playbook_invocations(
responses: List[str],
index: int,
row: pd.Series,
df: pd.DataFrame) -> pd.DataFrame:
responses: List[str],
index: int,
row: pd.Series,
df: pd.DataFrame) -> pd.DataFrame:
if row["playbook_pair"] in [None, "", "NaN", "nan"]:
playbook_index_list = [index]
else:
Expand All @@ -123,6 +132,23 @@ def process_playbook_invocations(

return df

@staticmethod
def process_flow_invocations(
responses: List[str],
index: int,
row: pd.Series,
df: pd.DataFrame) -> pd.DataFrame:
if row["flow_pair"] in [None, "", "NaN", "nan"]:
flow_index_list = [index]
else:
flow_index_list = literal_eval(row["flow_pair"])

for idx in flow_index_list:
flow = responses.pop(0)
df.loc[int(idx), "res_flow_name"] = flow["flow_name"]

return df

@staticmethod
def process_tool_invocations(
tool_responses: List[str],
Expand Down Expand Up @@ -201,13 +227,20 @@ def run_detect_intent_queries(self, df: pd.DataFrame) -> pd.DataFrame:
utterance_idx = int(row["utterance_pair"])
df.loc[utterance_idx, ["agent_response"]] = [text_res]

# Handle Play Invocations
# Handle Playbook Invocations
playbook_responses = self.s.collect_playbook_responses(res)
if len(playbook_responses) > 0:
df = self.process_playbook_invocations(
playbook_responses, index, row, df
)

# Handle Flow Invocations
flow_responses = self.s.collect_flow_responses(res)
if len(flow_responses) > 0:
df = self.process_flow_invocations(
flow_responses, index, row, df
)

# Handle Tool Invocations
tool_responses = self.s.collect_tool_responses(res)
if len(tool_responses) > 0:
Expand Down Expand Up @@ -390,6 +423,29 @@ def pair_playbook_calls(self, df: pd.DataFrame) -> pd.DataFrame:
df.loc[pair[0], "playbook_pair"] = str(pair[1])

return df

def pair_flow_calls(self, df: pd.DataFrame) -> pd.DataFrame:
"Identifies pairings of agent_utterance/flow_invocation by eval_id."
df["flow_pair"] = pd.Series(dtype="string")
grouped = df.groupby("eval_id")

for _, group in grouped:
user = group[
group["action_type"] == "User Utterance"
].index.tolist()
flow_list = group[
group["action_type"] == "Flow Invocation"
].index.tolist()

pairs = self.get_matching_list_idx(
user, flow_list
)

# Create pairs of user/flow_list row indices
for pair in pairs:
df.loc[pair[0], "flow_pair"] = str(pair[1])

return df

def validate_input_columns(self, df: pd.DataFrame) -> pd.DataFrame:
"""Validate input columns"""
Expand Down Expand Up @@ -510,6 +566,7 @@ def validate_and_prep_inputs(self, df: pd.DataFrame) -> pd.DataFrame:
df = self.pair_utterances(df)
df = self.pair_tool_calls(df)
df = self.pair_playbook_calls(df)
df = self.pair_flow_calls(df)

# fill remaining NA with empty string
for col in df.columns:
Expand Down

0 comments on commit f9f8157

Please sign in to comment.