From 83ff883de4e80e3b561de67db28bf1c00abaeb9d Mon Sep 17 00:00:00 2001 From: Lucas <31957045+SketchingDev@users.noreply.github.com> Date: Sat, 2 Dec 2023 21:56:45 +0000 Subject: [PATCH] Fix deployment config for AI command (#137) Fixes issue #132 --- .../README.md | 3 +- .../commands/ai/configSectionLoaded.spec.ts | 117 ++++++++++++++++++ .../RetryOnUnorderedMessageFailure.spec.ts | 4 +- .../scripted}/ui.spec.ts | 2 +- .../configSectionLoaded.spec.ts | 4 +- .../configSectionValidated.spec.ts | 2 +- .../scenarios/scenarioSectionLoaded.spec.ts | 2 +- .../scenarioSectionValidated.spec.ts | 2 +- .../yamlTestScript/testScript.spec.ts | 2 +- .../yamlTestScript/yamlFileValidated.spec.ts | 2 +- .../package.json | 2 +- .../commands/aiTest/createAiTestCommand.ts | 34 ++--- .../aiTest/prompt/shouldEndConversation.ts | 2 +- .../scriptedTest/createScriptedTestCommand.ts | 4 +- 14 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/configSectionLoaded.spec.ts rename packages/genesys-web-messaging-tester-cli/__tests__/{ => commands/scripted}/RetryOnUnorderedMessageFailure.spec.ts (98%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/ui.spec.ts (97%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/yamlTestScript/configSectionLoaded.spec.ts (95%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/yamlTestScript/configSectionValidated.spec.ts (98%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts (98%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts (97%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/yamlTestScript/testScript.spec.ts (96%) rename packages/genesys-web-messaging-tester-cli/__tests__/{testScenarioCommand => commands/scripted}/yamlTestScript/yamlFileValidated.spec.ts (96%) diff --git a/packages/genesys-web-messaging-tester-cli/README.md b/packages/genesys-web-messaging-tester-cli/README.md index 5c5d405..c9c83a4 100644 --- a/packages/genesys-web-messaging-tester-cli/README.md +++ b/packages/genesys-web-messaging-tester-cli/README.md @@ -112,7 +112,8 @@ Start by setting up an API key for ChatGPT: Write a scenario file containing all the scenarios you wish to run along with the [ID and region of your Web Messenger Deployment](https://help.mypurecloud.com/articles/deploy-messenger/). -The scenarios are written as ChatGPT Prompts, these can take some fine-tuning to get right ([see examples here](https://genesys-messenger-tester.makingchatbots.com/writing-tests/ai/example-prompts.html)). +The scenarios are written as ChatGPT Prompts, these can take some fine-tuning to get +right ([see examples here](https://genesys-messenger-tester.makingchatbots.com/writing-tests/ai/example-prompts.html)). The `terminatingPhrases` section defines the phrases you instruct ChatGPT to say to pass or fail a test. > [examples/cli-ai-tests/example.yml](https://github.com/ovotech/genesys-web-messaging-tester/tree/main/examples/cli-ai-tests/example.yml) diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/configSectionLoaded.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/configSectionLoaded.spec.ts new file mode 100644 index 0000000..a0f94db --- /dev/null +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/configSectionLoaded.spec.ts @@ -0,0 +1,117 @@ +import { readFileSync } from 'fs'; +import { Command } from 'commander'; +import { AiTestCommandDependencies } from '../../../src/commands/aiTest/createAiTestCommand'; +import { createCli } from '../../../src/createCli'; +import { OpenAI } from 'openai'; + +describe('Session Config', () => { + let fsReadFileSync: jest.MockedFunction; + + let webMessengerSessionFactory: jest.Mocked< + AiTestCommandDependencies['webMessengerSessionFactory'] + >; + let conversationFactory: jest.Mocked; + let mockOpenApiChatCompletions: jest.Mocked>; + + let cli: Command; + + beforeEach(() => { + fsReadFileSync = jest.fn(); + + const webMessengerSession = { on: jest.fn(), close: jest.fn() }; + webMessengerSessionFactory = jest.fn().mockReturnValue(webMessengerSession); + + const conversation = { waitForConversationToStart: jest.fn(), sendText: jest.fn() }; + conversationFactory = jest.fn().mockReturnValue(conversation); + + const cliCommand = new Command().exitOverride(() => { + throw new Error('CLI Command errored'); + }); + + const scenarioTestCommand = new Command().exitOverride(() => { + throw new Error('Scenario Test Command errored'); + }); + + cli = createCli(cliCommand, undefined, { + command: scenarioTestCommand, + fsReadFileSync, + fsAccessSync: jest.fn(), + webMessengerSessionFactory, + openAiChatCompletionFactory: () => mockOpenApiChatCompletions, + conversationFactory, + processEnv: { OPENAI_API_KEY: 'test' }, + }); + }); + + test('session config in Test Script loaded', async () => { + fsReadFileSync.mockReturnValue(` +config: + deploymentId: test-deployment-id-1 + region: test-region-1 + origin: test-origin-1 +scenarios: + Test: + setup: + prompt: Test prompt + terminatingPhrases: + pass: ["PASS"] + fail: ["FAIL"] +`); + const completion: OpenAI.Chat.ChatCompletion = { + choices: [{ message: { role: 'system', content: 'PASS' }, finish_reason: 'stop', index: 0 }], + created: 0, + id: '', + model: '', + object: '', + }; + + mockOpenApiChatCompletions = { create: jest.fn().mockResolvedValue(completion) }; + + await cli.parseAsync([...['node', '/path/to/cli'], 'ai', ...['/test/path']]); + + expect(webMessengerSessionFactory).toHaveBeenCalledWith({ + deploymentId: 'test-deployment-id-1', + region: 'test-region-1', + origin: 'test-origin-1', + }); + }); + + test('session config not necessary if session config args provided', async () => { + fsReadFileSync.mockReturnValue(` + scenarios: + Test: + setup: + prompt: Test prompt + terminatingPhrases: + pass: ["PASS"] + fail: ["FAIL"] + `); + + const completion: OpenAI.Chat.ChatCompletion = { + choices: [{ message: { role: 'system', content: 'PASS' }, finish_reason: 'stop', index: 0 }], + created: 0, + id: '', + model: '', + object: '', + }; + + mockOpenApiChatCompletions = { + create: jest.fn().mockResolvedValue(completion), + }; + + await cli.parseAsync([ + ...['node', '/path/to/cli'], + 'ai', + ...['--deployment-id', 'test-deployment-id-2'], + ...['--region', 'test-region-2'], + ...['--origin', 'test-origin-2'], + ...['/test/path'], + ]); + + expect(webMessengerSessionFactory).toHaveBeenCalledWith({ + deploymentId: 'test-deployment-id-2', + region: 'test-region-2', + origin: 'test-origin-2', + }); + }); +}); diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/RetryOnUnorderedMessageFailure.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/RetryOnUnorderedMessageFailure.spec.ts similarity index 98% rename from packages/genesys-web-messaging-tester-cli/__tests__/RetryOnUnorderedMessageFailure.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/RetryOnUnorderedMessageFailure.spec.ts index dc60171..93f4317 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/RetryOnUnorderedMessageFailure.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/RetryOnUnorderedMessageFailure.spec.ts @@ -8,8 +8,8 @@ import { import stripAnsi from 'strip-ansi'; import getPort from 'get-port'; import WebSocket from 'ws'; -import { waitForMs } from './fixtures/wait'; -import { createCli } from '../lib/createCli'; +import { waitForMs } from '../../fixtures/wait'; +import { createCli } from '../../../src/createCli'; jest.setTimeout(50000); describe('Retry unordered message', () => { diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/ui.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/ui.spec.ts similarity index 97% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/ui.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/ui.spec.ts index c154575..b870d4f 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/ui.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/ui.spec.ts @@ -2,7 +2,7 @@ import { accessSync, readFileSync } from 'fs'; import { Command } from 'commander'; import { Conversation, WebMessengerSession } from '@ovotech/genesys-web-messaging-tester'; import stripAnsi from 'strip-ansi'; -import { createCli } from '../../src/createCli'; +import { createCli } from '../../../src/createCli'; describe('Test script YAML loaded', () => { let capturedOutput: { diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/configSectionLoaded.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/configSectionLoaded.spec.ts similarity index 95% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/configSectionLoaded.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/configSectionLoaded.spec.ts index d1acfd0..3482c2d 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/configSectionLoaded.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/configSectionLoaded.spec.ts @@ -1,7 +1,7 @@ -import { ScriptedTestCommandDependencies } from '../../../src/commands/scriptedTest/createScriptedTestCommand'; +import { ScriptedTestCommandDependencies } from '../../../../src/commands/scriptedTest/createScriptedTestCommand'; import { readFileSync } from 'fs'; import { Command } from 'commander'; -import { createCli } from '../../../src/createCli'; +import { createCli } from '../../../../src/createCli'; import { ReorderedMessageDelayer } from '@ovotech/genesys-web-messaging-tester'; describe('Session Config', () => { diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/configSectionValidated.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/configSectionValidated.spec.ts similarity index 98% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/configSectionValidated.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/configSectionValidated.spec.ts index a3c3e5d..94488ed 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/configSectionValidated.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/configSectionValidated.spec.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import { Command } from 'commander'; import stripAnsi from 'strip-ansi'; -import { createCli } from '../../../src/createCli'; +import { createCli } from '../../../../src/createCli'; describe('Session Config Validated', () => { let capturedOutput: { diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts similarity index 98% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts index 2172886..aea3e0c 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/scenarios/scenarioSectionLoaded.spec.ts @@ -3,7 +3,7 @@ import { Command } from 'commander'; import { when } from 'jest-when'; import { Conversation, WebMessengerSession } from '@ovotech/genesys-web-messaging-tester'; import stripAnsi from 'strip-ansi'; -import { createCli } from '../../../../src/createCli'; +import { createCli } from '../../../../../src/createCli'; describe('Test script YAML loaded', () => { const validScenarioFilePath = '/test/path/config.json'; diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts similarity index 97% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts index 211d957..dcc5da1 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/scenarios/scenarioSectionValidated.spec.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import { Command } from 'commander'; import stripAnsi from 'strip-ansi'; -import { createCli } from '../../../../src/createCli'; +import { createCli } from '../../../../../src/createCli'; describe('Scenario Section Validated', () => { let capturedOutput: { diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/testScript.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/testScript.spec.ts similarity index 96% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/testScript.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/testScript.spec.ts index 670005a..c2f2ba4 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/testScript.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/testScript.spec.ts @@ -1,6 +1,6 @@ import { accessSync, readFileSync } from 'fs'; import { Command } from 'commander'; -import { createCli } from '../../../src/createCli'; +import { createCli } from '../../../../src/createCli'; describe('Test-script read from disk', () => { let fsReadFileSync: jest.MockedFunction; diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/yamlFileValidated.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/yamlFileValidated.spec.ts similarity index 96% rename from packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/yamlFileValidated.spec.ts rename to packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/yamlFileValidated.spec.ts index d46f0d7..be9435c 100644 --- a/packages/genesys-web-messaging-tester-cli/__tests__/testScenarioCommand/yamlTestScript/yamlFileValidated.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/__tests__/commands/scripted/yamlTestScript/yamlFileValidated.spec.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import { Command } from 'commander'; import stripAnsi from 'strip-ansi'; -import { createCli } from '../../../src/createCli'; +import { createCli } from '../../../../src/createCli'; describe('YAML file Validated', () => { let capturedOutput: { diff --git a/packages/genesys-web-messaging-tester-cli/package.json b/packages/genesys-web-messaging-tester-cli/package.json index e7ba530..7c0b4fc 100644 --- a/packages/genesys-web-messaging-tester-cli/package.json +++ b/packages/genesys-web-messaging-tester-cli/package.json @@ -1,6 +1,6 @@ { "name": "@ovotech/genesys-web-messaging-tester-cli", - "version": "2.0.0", + "version": "2.0.1", "main": "lib/index.js", "types": "lib/index.d.ts", "license": "Apache-2.0", diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts index 24b5c97..73e596b 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts @@ -8,7 +8,7 @@ import { WebMessengerGuestSession, WebMessengerSession, } from '@ovotech/genesys-web-messaging-tester'; -import OpenAI, { ClientOptions } from 'openai'; +import { ClientOptions, OpenAI } from 'openai'; import { validateOpenAiEnvVariables } from './validateOpenAIEnvVariables'; import { Ui } from './ui'; import { validateSessionConfig } from './validateSessionConfig'; @@ -32,9 +32,10 @@ const temperature = 0.6; export interface AiTestCommandDependencies { command?: Command; ui?: Ui; - openAiApiFactory?: (config: ClientOptions) => OpenAI; + openAiChatCompletionFactory?: (config: ClientOptions) => Pick; webMessengerSessionFactory?: (sessionConfig: SessionConfig) => WebMessengerSession; conversationFactory?: (session: WebMessengerSession) => Conversation; + processEnv?: NodeJS.ProcessEnv; fsReadFileSync?: typeof readFileSync; fsAccessSync?: typeof accessSync; chatGptModel?: OpenAI.CompletionCreateParams['model']; @@ -43,9 +44,10 @@ export interface AiTestCommandDependencies { export function createAiTestCommand({ command = new Command(), ui = new Ui(), - openAiApiFactory = (config) => new OpenAI(config), + openAiChatCompletionFactory = (config) => new OpenAI(config).chat.completions, webMessengerSessionFactory = (config) => new WebMessengerGuestSession(config, { IsTest: 'true' }), conversationFactory = (session) => new Conversation(session), + processEnv = process.env, fsReadFileSync = readFileSync, fsAccessSync = accessSync, chatGptModel = 'gpt-3.5-turbo', @@ -75,23 +77,11 @@ export function createAiTestCommand({ options: { deploymentId?: string; region?: string; origin?: string }, ) => { const outputConfig = command.configureOutput(); - if (!outputConfig) { - throw new Error('Output must be defined'); + if (!outputConfig?.writeOut || !outputConfig?.writeErr) { + throw new Error('No writeOut and/or writeErr'); } - if (!outputConfig.writeOut || !outputConfig.writeErr) { - throw new Error('No writeOut'); - } - - const sessionValidationResult = validateSessionConfig(options); - if (!sessionValidationResult.validSessionConfig) { - outputConfig.writeErr( - ui.validatingOpenAiEnvValidationFailed(sessionValidationResult.error), - ); - throw new CommandExpectedlyFailedError(); - } - - const openAiEnvValidationResult = validateOpenAiEnvVariables(process.env); + const openAiEnvValidationResult = validateOpenAiEnvVariables(processEnv); if (!openAiEnvValidationResult.openAikey) { outputConfig.writeErr( ui.validatingOpenAiEnvValidationFailed(openAiEnvValidationResult.error), @@ -141,9 +131,11 @@ export function createAiTestCommand({ const scenario = Object.entries(validPromptScript?.scenarios)[0][1]; - const session = webMessengerSessionFactory(sessionValidationResult.validSessionConfig); + const session = webMessengerSessionFactory( + sessionConfigValidationResults.validSessionConfig, + ); - const openai = openAiApiFactory({ + const openaiChatCompletion = openAiChatCompletionFactory({ apiKey: openAiEnvValidationResult.openAikey, maxRetries: 5, }); @@ -166,7 +158,7 @@ export function createAiTestCommand({ hasEnded: false, }; do { - const { choices } = await openai.chat.completions.create({ + const { choices } = await openaiChatCompletion.create({ model: chatGptModel, n: 1, // Number of choices temperature, diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts index d123716..ad55e5e 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts @@ -1,4 +1,4 @@ -import OpenAI from 'openai'; +import { OpenAI } from 'openai'; import { containsTerminatingPhrases } from './containsTerminatingPhrases'; interface Reason { diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts b/packages/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts index 71322d8..43a4a7f 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts @@ -125,8 +125,8 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`, }, ) => { const outputConfig = command.configureOutput(); - if (!outputConfig.writeOut || !outputConfig.writeErr) { - throw new Error('No writeOut'); + if (!outputConfig?.writeOut || !outputConfig?.writeErr) { + throw new Error('No writeOut and/or writeErr'); } let associateId: { enabled: false } | { enabled: true; client: MessageIdToConvoIdClient };