From 0b5e849d77cc67828a8e998040cc967fd19a4056 Mon Sep 17 00:00:00 2001 From: maamalama Date: Mon, 29 Jul 2024 11:33:35 -0700 Subject: [PATCH 1/6] feedback init --- valhalla/jawn/src/lib/stores/ScoreStore.ts | 95 +++++++++++++++++-- .../src/managers/request/RequestManager.ts | 12 +++ .../jawn/src/managers/score/ScoreManager.ts | 30 +++++- 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/valhalla/jawn/src/lib/stores/ScoreStore.ts b/valhalla/jawn/src/lib/stores/ScoreStore.ts index 7228a1bdaa..a02eeb0cb8 100644 --- a/valhalla/jawn/src/lib/stores/ScoreStore.ts +++ b/valhalla/jawn/src/lib/stores/ScoreStore.ts @@ -10,17 +10,14 @@ export type Score = { score_attribute_key: string; score_attribute_type: string; score_attribute_value: number; -} +}; export class ScoreStore extends BaseStore { constructor(organizationId: string) { super(organizationId); } - public async putScoresIntoSupabase( - requestId: string, - scores: Score[] - ) { + public async putScoresIntoSupabase(requestId: string, scores: Score[]) { try { const scoreKeys = scores.map((score) => score.score_attribute_key); const scoreTypes = scores.map((score) => score.score_attribute_type); @@ -51,7 +48,7 @@ export class ScoreStore extends BaseStore { SELECT id, score_key FROM upserted_attributes; `; - + const { data: upsertedAttributes, error: upsertError } = await dbExecute( upsertQuery, [scoreKeys, scoreTypes, organizationIds] @@ -169,6 +166,92 @@ export class ScoreStore extends BaseStore { return ok(rowContents.data); } + public async putScoreIntoClickhouse(newVersion: { + id: string; + version: number; + provider: string; + score: Score; + }): Promise> { + let rowContents = resultMap( + await clickhouseDb.dbQuery( + ` + SELECT * + FROM request_response_versioned + WHERE request_id = {val_0: UUID} + AND version = {val_1: UInt64} + AND organization_id = {val_2: String} + AND provider = {val_3: String} + `, + [ + newVersion.id, + newVersion.version - 1, + this.organizationId, + newVersion.provider, + ] + ), + (x) => x[0] + ); + + if (rowContents.error) { + return rowContents; + } + if (!rowContents.data) { + rowContents = resultMap( + await clickhouseDb.dbQuery( + ` + SELECT * + FROM request_response_versioned + WHERE request_id = {val_0: UUID} + AND organization_id = {val_1: String} + AND provider = {val_2: String} + ORDER BY version DESC + LIMIT 1 + `, + [newVersion.id, this.organizationId, newVersion.provider] + ), + (x) => x[0] + ); + } + + if (rowContents.error || !rowContents.data) { + return err("Could not find previous version of request"); + } + + const res = await clickhouseDb.dbInsertClickhouse( + "request_response_versioned", + [ + // Delete the previous version + { + sign: -1, + version: rowContents.data.version, + request_id: newVersion.id, + organization_id: this.organizationId, + provider: newVersion.provider, + model: rowContents.data.model, + request_created_at: rowContents.data.request_created_at, + }, + // Insert the new version + { + ...rowContents.data, + sign: 1, + version: newVersion.version, + scores: { + ...rowContents.data.scores, + ...{ + [newVersion.score.score_attribute_key]: + newVersion.score.score_attribute_value, + }, + }, + }, + ] + ); + if (res.error) { + return err(res.error); + } + + return ok(rowContents.data); + } + public async bumpRequestVersion(requestId: string) { return await dbExecute<{ id: string; diff --git a/valhalla/jawn/src/managers/request/RequestManager.ts b/valhalla/jawn/src/managers/request/RequestManager.ts index ac2eefab95..9bbd0880d2 100644 --- a/valhalla/jawn/src/managers/request/RequestManager.ts +++ b/valhalla/jawn/src/managers/request/RequestManager.ts @@ -16,6 +16,7 @@ import { } from "../../lib/stores/request/request"; import { costOfPrompt } from "../../packages/cost"; import { BaseManager } from "../BaseManager"; +import { ScoreManager } from "../score/ScoreManager"; export class RequestManager extends BaseManager { private versionedRequestStore: VersionedRequestStore; @@ -147,6 +148,17 @@ export class RequestManager extends BaseManager { return err(feedbackResult.error.message); } + const scoreManager = new ScoreManager(this.authParams); + const scoreResult = await scoreManager.addSignleScoreToClickhouse( + requestId, + { + score_attribute_key: "helicone-feedback", + score_attribute_type: "number", + score_attribute_value: feedback ? 1 : 0, + } + ); + console.log("Score result", scoreResult); + return ok(null); } diff --git a/valhalla/jawn/src/managers/score/ScoreManager.ts b/valhalla/jawn/src/managers/score/ScoreManager.ts index a4ec881d1c..d8540ec3cd 100644 --- a/valhalla/jawn/src/managers/score/ScoreManager.ts +++ b/valhalla/jawn/src/managers/score/ScoreManager.ts @@ -68,12 +68,40 @@ export class ScoreManager extends BaseManager { } } + public async addSignleScoreToClickhouse( + requestId: string, + mappedScores: Score + ): Promise> { + const request = await this.scoreStore.bumpRequestVersion(requestId); + + console.log("version bumped", request); + + if (request.error || !request.data) { + return err(request.error); + } + + if (request.data.length === 0) { + return err(`Request not found: ${requestId}`); + } + + const requestInClickhouse = await this.scoreStore.putScoreIntoClickhouse({ + ...request.data[0], + score: mappedScores, + }); + + if (requestInClickhouse.error || !requestInClickhouse.data) { + return requestInClickhouse; + } + return { data: "Scores added to Clickhouse successfully", error: null }; + } + private mapScores(scores: Scores): Score[] { return Object.entries(scores).map(([key, value]) => { return { score_attribute_key: key, score_attribute_type: typeof value === "boolean" ? "boolean" : "number", - score_attribute_value: typeof value === "boolean" ? (value ? 1 : 0) : value, + score_attribute_value: + typeof value === "boolean" ? (value ? 1 : 0) : value, }; }); } From 637780d9b60810c1651d99b42c0717ca449d484e Mon Sep 17 00:00:00 2001 From: maamalama Date: Mon, 29 Jul 2024 14:00:31 -0700 Subject: [PATCH 2/6] fixes --- valhalla/jawn/src/managers/request/RequestManager.ts | 9 +++++++-- valhalla/jawn/src/managers/score/ScoreManager.ts | 2 -- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/valhalla/jawn/src/managers/request/RequestManager.ts b/valhalla/jawn/src/managers/request/RequestManager.ts index 9bbd0880d2..f0a92235a0 100644 --- a/valhalla/jawn/src/managers/request/RequestManager.ts +++ b/valhalla/jawn/src/managers/request/RequestManager.ts @@ -149,7 +149,7 @@ export class RequestManager extends BaseManager { } const scoreManager = new ScoreManager(this.authParams); - const scoreResult = await scoreManager.addSignleScoreToClickhouse( + const feedbackScoreResult = await scoreManager.addSignleScoreToClickhouse( requestId, { score_attribute_key: "helicone-feedback", @@ -157,7 +157,12 @@ export class RequestManager extends BaseManager { score_attribute_value: feedback ? 1 : 0, } ); - console.log("Score result", scoreResult); + + if (feedbackScoreResult.error) { + console.error("Error upserting feedback:", feedbackScoreResult.error); + return err(feedbackScoreResult.error); + } + return ok(null); } diff --git a/valhalla/jawn/src/managers/score/ScoreManager.ts b/valhalla/jawn/src/managers/score/ScoreManager.ts index d8540ec3cd..f36b9f10cf 100644 --- a/valhalla/jawn/src/managers/score/ScoreManager.ts +++ b/valhalla/jawn/src/managers/score/ScoreManager.ts @@ -74,8 +74,6 @@ export class ScoreManager extends BaseManager { ): Promise> { const request = await this.scoreStore.bumpRequestVersion(requestId); - console.log("version bumped", request); - if (request.error || !request.data) { return err(request.error); } From 4c62a73e8bc42023aabbeb2be2e0337eb31007ae Mon Sep 17 00:00:00 2001 From: maamalama Date: Mon, 29 Jul 2024 14:08:26 -0700 Subject: [PATCH 3/6] fix --- valhalla/jawn/src/managers/request/RequestManager.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/valhalla/jawn/src/managers/request/RequestManager.ts b/valhalla/jawn/src/managers/request/RequestManager.ts index f0a92235a0..b66202e459 100644 --- a/valhalla/jawn/src/managers/request/RequestManager.ts +++ b/valhalla/jawn/src/managers/request/RequestManager.ts @@ -152,7 +152,7 @@ export class RequestManager extends BaseManager { const feedbackScoreResult = await scoreManager.addSignleScoreToClickhouse( requestId, { - score_attribute_key: "helicone-feedback", + score_attribute_key: "helicone-score-feedback", score_attribute_type: "number", score_attribute_value: feedback ? 1 : 0, } @@ -162,7 +162,6 @@ export class RequestManager extends BaseManager { console.error("Error upserting feedback:", feedbackScoreResult.error); return err(feedbackScoreResult.error); } - return ok(null); } From 5c8c2c1aba6ca9f540f3c2c713ce547407f414f4 Mon Sep 17 00:00:00 2001 From: maamalama Date: Mon, 29 Jul 2024 14:12:45 -0700 Subject: [PATCH 4/6] fe fix --- web/components/templates/playground/playgroundPage.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/web/components/templates/playground/playgroundPage.tsx b/web/components/templates/playground/playgroundPage.tsx index 5c19475e6a..afa088f222 100644 --- a/web/components/templates/playground/playgroundPage.tsx +++ b/web/components/templates/playground/playgroundPage.tsx @@ -602,7 +602,6 @@ export default PlaygroundPage; interface PlaygroundPageProps { showNewButton?: boolean; request?: string; - showNewButton: boolean; } export type PlaygroundModel = { From a0ea0fb2b9dbbc53562768e271bb4d76a9459397 Mon Sep 17 00:00:00 2001 From: maamalama Date: Mon, 29 Jul 2024 17:06:32 -0700 Subject: [PATCH 5/6] pr fixes --- valhalla/jawn/src/lib/stores/ScoreStore.ts | 11 ++-- .../src/managers/request/RequestManager.ts | 14 +++--- .../jawn/src/managers/score/ScoreManager.ts | 50 ++++++++----------- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/valhalla/jawn/src/lib/stores/ScoreStore.ts b/valhalla/jawn/src/lib/stores/ScoreStore.ts index a02eeb0cb8..737f1d9fb8 100644 --- a/valhalla/jawn/src/lib/stores/ScoreStore.ts +++ b/valhalla/jawn/src/lib/stores/ScoreStore.ts @@ -152,10 +152,13 @@ export class ScoreStore extends BaseStore { ...rowContents.data, sign: 1, version: newVersion.version, - scores: newVersion.scores.reduce((acc, score) => { - acc[score.score_attribute_key] = score.score_attribute_value; - return acc; - }, {} as Record), + scores: { + ...rowContents.data.scores, + ...newVersion.scores.reduce((acc, score) => { + acc[score.score_attribute_key] = score.score_attribute_value; + return acc; + }, {} as Record), + }, }, ] ); diff --git a/valhalla/jawn/src/managers/request/RequestManager.ts b/valhalla/jawn/src/managers/request/RequestManager.ts index b66202e459..eb77c660eb 100644 --- a/valhalla/jawn/src/managers/request/RequestManager.ts +++ b/valhalla/jawn/src/managers/request/RequestManager.ts @@ -149,13 +149,15 @@ export class RequestManager extends BaseManager { } const scoreManager = new ScoreManager(this.authParams); - const feedbackScoreResult = await scoreManager.addSignleScoreToClickhouse( + const feedbackScoreResult = await scoreManager.addScoresToClickhouse( requestId, - { - score_attribute_key: "helicone-score-feedback", - score_attribute_type: "number", - score_attribute_value: feedback ? 1 : 0, - } + [ + { + score_attribute_key: "helicone-score-feedback", + score_attribute_type: "number", + score_attribute_value: feedback ? 1 : 0, + }, + ] ); if (feedbackScoreResult.error) { diff --git a/valhalla/jawn/src/managers/score/ScoreManager.ts b/valhalla/jawn/src/managers/score/ScoreManager.ts index f36b9f10cf..bc10a11443 100644 --- a/valhalla/jawn/src/managers/score/ScoreManager.ts +++ b/valhalla/jawn/src/managers/score/ScoreManager.ts @@ -41,6 +41,26 @@ export class ScoreManager extends BaseManager { return err(supabaseRequest.error); } + const requestInClickhouse = await this.addScoresToClickhouse( + requestId, + mappedScores + ); + + if (requestInClickhouse.error || !requestInClickhouse.data) { + return requestInClickhouse; + } + + return { data: "Scores added to Clickhouse successfully", error: null }; + } catch (error: any) { + return err(error.message); + } + } + + public async addScoresToClickhouse( + requestId: string, + mappedScores: Score[] + ): Promise> { + try { const request = await this.scoreStore.bumpRequestVersion(requestId); if (request.error || !request.data) { @@ -61,36 +81,10 @@ export class ScoreManager extends BaseManager { if (requestInClickhouse.error || !requestInClickhouse.data) { return requestInClickhouse; } - return { data: "Scores added to Clickhouse successfully", error: null }; - } catch (error: any) { - return err(error.message); - } - } - - public async addSignleScoreToClickhouse( - requestId: string, - mappedScores: Score - ): Promise> { - const request = await this.scoreStore.bumpRequestVersion(requestId); - - if (request.error || !request.data) { - return err(request.error); - } - - if (request.data.length === 0) { - return err(`Request not found: ${requestId}`); - } - - const requestInClickhouse = await this.scoreStore.putScoreIntoClickhouse({ - ...request.data[0], - score: mappedScores, - }); - - if (requestInClickhouse.error || !requestInClickhouse.data) { - return requestInClickhouse; + } catch { + return err("Error adding scores to Clickhouse"); } - return { data: "Scores added to Clickhouse successfully", error: null }; } private mapScores(scores: Scores): Score[] { From 5348c16e4ebbef9d202e8ae51fc71dea7b709645 Mon Sep 17 00:00:00 2001 From: maamalama Date: Mon, 29 Jul 2024 17:12:28 -0700 Subject: [PATCH 6/6] dont need this --- valhalla/jawn/src/lib/stores/ScoreStore.ts | 86 ---------------------- 1 file changed, 86 deletions(-) diff --git a/valhalla/jawn/src/lib/stores/ScoreStore.ts b/valhalla/jawn/src/lib/stores/ScoreStore.ts index 737f1d9fb8..c52853d3fe 100644 --- a/valhalla/jawn/src/lib/stores/ScoreStore.ts +++ b/valhalla/jawn/src/lib/stores/ScoreStore.ts @@ -169,92 +169,6 @@ export class ScoreStore extends BaseStore { return ok(rowContents.data); } - public async putScoreIntoClickhouse(newVersion: { - id: string; - version: number; - provider: string; - score: Score; - }): Promise> { - let rowContents = resultMap( - await clickhouseDb.dbQuery( - ` - SELECT * - FROM request_response_versioned - WHERE request_id = {val_0: UUID} - AND version = {val_1: UInt64} - AND organization_id = {val_2: String} - AND provider = {val_3: String} - `, - [ - newVersion.id, - newVersion.version - 1, - this.organizationId, - newVersion.provider, - ] - ), - (x) => x[0] - ); - - if (rowContents.error) { - return rowContents; - } - if (!rowContents.data) { - rowContents = resultMap( - await clickhouseDb.dbQuery( - ` - SELECT * - FROM request_response_versioned - WHERE request_id = {val_0: UUID} - AND organization_id = {val_1: String} - AND provider = {val_2: String} - ORDER BY version DESC - LIMIT 1 - `, - [newVersion.id, this.organizationId, newVersion.provider] - ), - (x) => x[0] - ); - } - - if (rowContents.error || !rowContents.data) { - return err("Could not find previous version of request"); - } - - const res = await clickhouseDb.dbInsertClickhouse( - "request_response_versioned", - [ - // Delete the previous version - { - sign: -1, - version: rowContents.data.version, - request_id: newVersion.id, - organization_id: this.organizationId, - provider: newVersion.provider, - model: rowContents.data.model, - request_created_at: rowContents.data.request_created_at, - }, - // Insert the new version - { - ...rowContents.data, - sign: 1, - version: newVersion.version, - scores: { - ...rowContents.data.scores, - ...{ - [newVersion.score.score_attribute_key]: - newVersion.score.score_attribute_value, - }, - }, - }, - ] - ); - if (res.error) { - return err(res.error); - } - - return ok(rowContents.data); - } - public async bumpRequestVersion(requestId: string) { return await dbExecute<{ id: string;