Fix deployment config for AI command (#137)

Fixes issue #132
ovotech · Dec 2, 2023 · 83ff883 · 83ff883
1 parent c6d190e
commit 83ff883
Show file tree

Hide file tree

Showing 14 changed files with 146 additions and 36 deletions.
diff --git a/packages/genesys-web-messaging-tester-cli/README.md b/packages/genesys-web-messaging-tester-cli/README.md
@@ -112,7 +112,8 @@ Start by setting up an API key for ChatGPT:
 Write a scenario file containing all the scenarios you wish to run along with
 the [ID and region of your Web Messenger Deployment](https://help.mypurecloud.com/articles/deploy-messenger/).
 
-The scenarios are written as ChatGPT Prompts, these can take some fine-tuning to get right ([see examples here](https://genesys-messenger-tester.makingchatbots.com/writing-tests/ai/example-prompts.html)).
+The scenarios are written as ChatGPT Prompts, these can take some fine-tuning to get
+right ([see examples here](https://genesys-messenger-tester.makingchatbots.com/writing-tests/ai/example-prompts.html)).
 The `terminatingPhrases` section defines the phrases you instruct ChatGPT to say to pass or fail a test.
 
 > [examples/cli-ai-tests/example.yml](https://github.com/ovotech/genesys-web-messaging-tester/tree/main/examples/cli-ai-tests/example.yml)

diff --git a/packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/configSectionLoaded.spec.ts b/packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/configSectionLoaded.spec.ts
@@ -0,0 +1,117 @@
+import { readFileSync } from 'fs';
+import { Command } from 'commander';
+import { AiTestCommandDependencies } from '../../../src/commands/aiTest/createAiTestCommand';
+import { createCli } from '../../../src/createCli';
+import { OpenAI } from 'openai';
+
+describe('Session Config', () => {
+  let fsReadFileSync: jest.MockedFunction<typeof readFileSync>;
+
+  let webMessengerSessionFactory: jest.Mocked<
+    AiTestCommandDependencies['webMessengerSessionFactory']
+  >;
+  let conversationFactory: jest.Mocked<AiTestCommandDependencies['conversationFactory']>;
+  let mockOpenApiChatCompletions: jest.Mocked<Pick<OpenAI.Chat.Completions, 'create'>>;
+
+  let cli: Command;
+
+  beforeEach(() => {
+    fsReadFileSync = jest.fn();
+
+    const webMessengerSession = { on: jest.fn(), close: jest.fn() };
+    webMessengerSessionFactory = jest.fn().mockReturnValue(webMessengerSession);
+
+    const conversation = { waitForConversationToStart: jest.fn(), sendText: jest.fn() };
+    conversationFactory = jest.fn().mockReturnValue(conversation);
+
+    const cliCommand = new Command().exitOverride(() => {
+      throw new Error('CLI Command errored');
+    });
+
+    const scenarioTestCommand = new Command().exitOverride(() => {
+      throw new Error('Scenario Test Command errored');
+    });
+
+    cli = createCli(cliCommand, undefined, {
+      command: scenarioTestCommand,
+      fsReadFileSync,
+      fsAccessSync: jest.fn(),
+      webMessengerSessionFactory,
+      openAiChatCompletionFactory: () => mockOpenApiChatCompletions,
+      conversationFactory,
+      processEnv: { OPENAI_API_KEY: 'test' },
+    });
+  });
+
+  test('session config in Test Script loaded', async () => {
+    fsReadFileSync.mockReturnValue(`
+config:
+  deploymentId: test-deployment-id-1
+  region: test-region-1
+  origin: test-origin-1
+scenarios:
+  Test:
+    setup:
+      prompt: Test prompt
+      terminatingPhrases:
+        pass: ["PASS"]
+        fail: ["FAIL"]
+`);
+    const completion: OpenAI.Chat.ChatCompletion = {
+      choices: [{ message: { role: 'system', content: 'PASS' }, finish_reason: 'stop', index: 0 }],
+      created: 0,
+      id: '',
+      model: '',
+      object: '',
+    };
+
+    mockOpenApiChatCompletions = { create: jest.fn().mockResolvedValue(completion) };
+
+    await cli.parseAsync([...['node', '/path/to/cli'], 'ai', ...['/test/path']]);
+
+    expect(webMessengerSessionFactory).toHaveBeenCalledWith({
+      deploymentId: 'test-deployment-id-1',
+      region: 'test-region-1',
+      origin: 'test-origin-1',
+    });
+  });
+
+  test('session config not necessary if session config args provided', async () => {
+    fsReadFileSync.mockReturnValue(`
+  scenarios:
+    Test:
+      setup:
+        prompt: Test prompt
+        terminatingPhrases:
+          pass: ["PASS"]
+          fail: ["FAIL"]
+  `);
+
+    const completion: OpenAI.Chat.ChatCompletion = {
+      choices: [{ message: { role: 'system', content: 'PASS' }, finish_reason: 'stop', index: 0 }],
+      created: 0,
+      id: '',
+      model: '',
+      object: '',
+    };
+
+    mockOpenApiChatCompletions = {
+      create: jest.fn().mockResolvedValue(completion),
+    };
+
+    await cli.parseAsync([
+      ...['node', '/path/to/cli'],
+      'ai',
+      ...['--deployment-id', 'test-deployment-id-2'],
+      ...['--region', 'test-region-2'],
+      ...['--origin', 'test-origin-2'],
+      ...['/test/path'],
+    ]);
+
+    expect(webMessengerSessionFactory).toHaveBeenCalledWith({
+      deploymentId: 'test-deployment-id-2',
+      region: 'test-region-2',
+      origin: 'test-origin-2',
+    });
+  });
+});
diff --git a/...__/RetryOnUnorderedMessageFailure.spec.ts → ...ed/RetryOnUnorderedMessageFailure.spec.ts b/...__/RetryOnUnorderedMessageFailure.spec.ts → ...ed/RetryOnUnorderedMessageFailure.spec.ts
@@ -8,8 +8,8 @@ import {
 import stripAnsi from 'strip-ansi';
 import getPort from 'get-port';
 import WebSocket from 'ws';
-import { waitForMs } from './fixtures/wait';
-import { createCli } from '../lib/createCli';
+import { waitForMs } from '../../fixtures/wait';
+import { createCli } from '../../../src/createCli';
 
 jest.setTimeout(50000);
 describe('Retry unordered message', () => {

diff --git a/.../__tests__/testScenarioCommand/ui.spec.ts → ...li/__tests__/commands/scripted/ui.spec.ts b/.../__tests__/testScenarioCommand/ui.spec.ts → ...li/__tests__/commands/scripted/ui.spec.ts
@@ -2,7 +2,7 @@ import { accessSync, readFileSync } from 'fs';
 import { Command } from 'commander';
 import { Conversation, WebMessengerSession } from '@ovotech/genesys-web-messaging-tester';
 import stripAnsi from 'strip-ansi';
-import { createCli } from '../../src/createCli';
+import { createCli } from '../../../src/createCli';
 
 describe('Test script YAML loaded', () => {
   let capturedOutput: {

diff --git a/...amlTestScript/configSectionLoaded.spec.ts → ...amlTestScript/configSectionLoaded.spec.ts b/...amlTestScript/configSectionLoaded.spec.ts → ...amlTestScript/configSectionLoaded.spec.ts
@@ -1,7 +1,7 @@
-import { ScriptedTestCommandDependencies } from '../../../src/commands/scriptedTest/createScriptedTestCommand';
+import { ScriptedTestCommandDependencies } from '../../../../src/commands/scriptedTest/createScriptedTestCommand';
 import { readFileSync } from 'fs';
 import { Command } from 'commander';
-import { createCli } from '../../../src/createCli';
+import { createCli } from '../../../../src/createCli';
 import { ReorderedMessageDelayer } from '@ovotech/genesys-web-messaging-tester';
 
 describe('Session Config', () => {

diff --git a/...TestScript/configSectionValidated.spec.ts → ...TestScript/configSectionValidated.spec.ts b/...TestScript/configSectionValidated.spec.ts → ...TestScript/configSectionValidated.spec.ts
@@ -1,7 +1,7 @@
 import { readFileSync } from 'fs';
 import { Command } from 'commander';
 import stripAnsi from 'strip-ansi';
-import { createCli } from '../../../src/createCli';
+import { createCli } from '../../../../src/createCli';
 
 describe('Session Config Validated', () => {
   let capturedOutput: {

diff --git a/...t/scenarios/scenarioSectionLoaded.spec.ts → ...t/scenarios/scenarioSectionLoaded.spec.ts b/...t/scenarios/scenarioSectionLoaded.spec.ts → ...t/scenarios/scenarioSectionLoaded.spec.ts
@@ -3,7 +3,7 @@ import { Command } from 'commander';
 import { when } from 'jest-when';
 import { Conversation, WebMessengerSession } from '@ovotech/genesys-web-messaging-tester';
 import stripAnsi from 'strip-ansi';
-import { createCli } from '../../../../src/createCli';
+import { createCli } from '../../../../../src/createCli';
 
 describe('Test script YAML loaded', () => {
   const validScenarioFilePath = '/test/path/config.json';

diff --git a/...cenarios/scenarioSectionValidated.spec.ts → ...cenarios/scenarioSectionValidated.spec.ts b/...cenarios/scenarioSectionValidated.spec.ts → ...cenarios/scenarioSectionValidated.spec.ts
@@ -1,7 +1,7 @@
 import { readFileSync } from 'fs';
 import { Command } from 'commander';
 import stripAnsi from 'strip-ansi';
-import { createCli } from '../../../../src/createCli';
+import { createCli } from '../../../../../src/createCli';
 
 describe('Scenario Section Validated', () => {
   let capturedOutput: {

diff --git a/...Command/yamlTestScript/testScript.spec.ts → ...cripted/yamlTestScript/testScript.spec.ts b/...Command/yamlTestScript/testScript.spec.ts → ...cripted/yamlTestScript/testScript.spec.ts
@@ -1,6 +1,6 @@
 import { accessSync, readFileSync } from 'fs';
 import { Command } from 'commander';
-import { createCli } from '../../../src/createCli';
+import { createCli } from '../../../../src/createCli';
 
 describe('Test-script read from disk', () => {
   let fsReadFileSync: jest.MockedFunction<typeof readFileSync>;

diff --git a/.../yamlTestScript/yamlFileValidated.spec.ts → .../yamlTestScript/yamlFileValidated.spec.ts b/.../yamlTestScript/yamlFileValidated.spec.ts → .../yamlTestScript/yamlFileValidated.spec.ts
@@ -1,7 +1,7 @@
 import { readFileSync } from 'fs';
 import { Command } from 'commander';
 import stripAnsi from 'strip-ansi';
-import { createCli } from '../../../src/createCli';
+import { createCli } from '../../../../src/createCli';
 
 describe('YAML file Validated', () => {
   let capturedOutput: {

diff --git a/packages/genesys-web-messaging-tester-cli/package.json b/packages/genesys-web-messaging-tester-cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@ovotech/genesys-web-messaging-tester-cli",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "license": "Apache-2.0",

diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts
@@ -8,7 +8,7 @@ import {
   WebMessengerGuestSession,
   WebMessengerSession,
 } from '@ovotech/genesys-web-messaging-tester';
-import OpenAI, { ClientOptions } from 'openai';
+import { ClientOptions, OpenAI } from 'openai';
 import { validateOpenAiEnvVariables } from './validateOpenAIEnvVariables';
 import { Ui } from './ui';
 import { validateSessionConfig } from './validateSessionConfig';
@@ -32,9 +32,10 @@ const temperature = 0.6;
 export interface AiTestCommandDependencies {
   command?: Command;
   ui?: Ui;
-  openAiApiFactory?: (config: ClientOptions) => OpenAI;
+  openAiChatCompletionFactory?: (config: ClientOptions) => Pick<OpenAI.Chat.Completions, 'create'>;
   webMessengerSessionFactory?: (sessionConfig: SessionConfig) => WebMessengerSession;
   conversationFactory?: (session: WebMessengerSession) => Conversation;
+  processEnv?: NodeJS.ProcessEnv;
   fsReadFileSync?: typeof readFileSync;
   fsAccessSync?: typeof accessSync;
   chatGptModel?: OpenAI.CompletionCreateParams['model'];
@@ -43,9 +44,10 @@ export interface AiTestCommandDependencies {
 export function createAiTestCommand({
   command = new Command(),
   ui = new Ui(),
-  openAiApiFactory = (config) => new OpenAI(config),
+  openAiChatCompletionFactory = (config) => new OpenAI(config).chat.completions,
   webMessengerSessionFactory = (config) => new WebMessengerGuestSession(config, { IsTest: 'true' }),
   conversationFactory = (session) => new Conversation(session),
+  processEnv = process.env,
   fsReadFileSync = readFileSync,
   fsAccessSync = accessSync,
   chatGptModel = 'gpt-3.5-turbo',
@@ -75,23 +77,11 @@ export function createAiTestCommand({
         options: { deploymentId?: string; region?: string; origin?: string },
       ) => {
         const outputConfig = command.configureOutput();
-        if (!outputConfig) {
-          throw new Error('Output must be defined');
+        if (!outputConfig?.writeOut || !outputConfig?.writeErr) {
+          throw new Error('No writeOut and/or writeErr');
         }
 
-        if (!outputConfig.writeOut || !outputConfig.writeErr) {
-          throw new Error('No writeOut');
-        }
-
-        const sessionValidationResult = validateSessionConfig(options);
-        if (!sessionValidationResult.validSessionConfig) {
-          outputConfig.writeErr(
-            ui.validatingOpenAiEnvValidationFailed(sessionValidationResult.error),
-          );
-          throw new CommandExpectedlyFailedError();
-        }
-
-        const openAiEnvValidationResult = validateOpenAiEnvVariables(process.env);
+        const openAiEnvValidationResult = validateOpenAiEnvVariables(processEnv);
         if (!openAiEnvValidationResult.openAikey) {
           outputConfig.writeErr(
             ui.validatingOpenAiEnvValidationFailed(openAiEnvValidationResult.error),
@@ -141,9 +131,11 @@ export function createAiTestCommand({
 
         const scenario = Object.entries(validPromptScript?.scenarios)[0][1];
 
-        const session = webMessengerSessionFactory(sessionValidationResult.validSessionConfig);
+        const session = webMessengerSessionFactory(
+          sessionConfigValidationResults.validSessionConfig,
+        );
 
-        const openai = openAiApiFactory({
+        const openaiChatCompletion = openAiChatCompletionFactory({
           apiKey: openAiEnvValidationResult.openAikey,
           maxRetries: 5,
         });
@@ -166,7 +158,7 @@ export function createAiTestCommand({
           hasEnded: false,
         };
         do {
-          const { choices } = await openai.chat.completions.create({
+          const { choices } = await openaiChatCompletion.create({
             model: chatGptModel,
             n: 1, // Number of choices
             temperature,

diff --git a/...ages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts b/...ages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts
@@ -1,4 +1,4 @@
-import OpenAI from 'openai';
+import { OpenAI } from 'openai';
 import { containsTerminatingPhrases } from './containsTerminatingPhrases';
 
 interface Reason {

diff --git a/...s/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts b/...s/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts
@@ -125,8 +125,8 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
         },
       ) => {
         const outputConfig = command.configureOutput();
-        if (!outputConfig.writeOut || !outputConfig.writeErr) {
-          throw new Error('No writeOut');
+        if (!outputConfig?.writeOut || !outputConfig?.writeErr) {
+          throw new Error('No writeOut and/or writeErr');
         }
 
         let associateId: { enabled: false } | { enabled: true; client: MessageIdToConvoIdClient };