Skip to content

Commit

Permalink
test(instr-openai): update recorded responses; fix recording handling…
Browse files Browse the repository at this point in the history 
… for nock@14 (#573)

With nock@14 the recording format has changed slightly: the headers are
stored as an object instead of an array. The sanitization needed to be
updated for this.
  • Loading branch information
@trentm
trentm authored Jan 29, 2025
1 parent 2af8b3f commit 36c3745
Show file tree
Hide file tree
Showing 12 changed files with 244 additions and 463 deletions.
3 changes: 1 addition & 2 deletions packages/instrumentation-openai/TESTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,7 @@ api.openai.com, so you must have valid OpenAI auth set in your environment.
cp openai.env.example openai.env
vi openai.env # Add your OpenAI credentials.
set -a; source openai.env
npm run test:regenerate-recordings
(set -a; source openai.env; npm run test:regenerate-recordings)
```
## Filtering which fixture tests are run
Expand Down
32 changes: 16 additions & 16 deletions packages/instrumentation-openai/test/fixtures.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,14 +254,14 @@ test('fixtures', async suite => {
[ATTR_GEN_AI_REQUEST_MAX_TOKENS]: 200,
[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'],
[ATTR_GEN_AI_RESPONSE_ID]: isUnit
? 'chatcmpl-AfbMVACkhZbXSJoCkCzhuGjI9hxi9'
? 'chatcmpl-Aupa6oebo6v8G4l0QcprsBPniQdta'
: /.+/,
[ATTR_GEN_AI_RESPONSE_MODEL]: isExpectedResponseModel(
'gpt-4o-mini-2024-07-18',
process.env.TEST_CHAT_MODEL
),
[ATTR_GEN_AI_USAGE_INPUT_TOKENS]: isUnit ? 22 : isPositiveInteger,
[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: isUnit ? 3 : isPositiveInteger,
[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: isUnit ? 4 : isPositiveInteger,
},
scope: {
name: '@elastic/opentelemetry-instrumentation-openai',
Expand Down Expand Up @@ -309,7 +309,7 @@ test('fixtures', async suite => {
finish_reason: 'stop',
index: 0,
message: {
content: isUnit ? 'Southern Ocean.' : /.+/,
content: isUnit ? 'Atlantic Ocean.' : /.+/,
},
},
traceId: spans[0].traceId,
Expand Down Expand Up @@ -461,7 +461,7 @@ test('fixtures', async suite => {
[ATTR_GEN_AI_SYSTEM]: 'openai',
[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['stop'],
[ATTR_GEN_AI_RESPONSE_ID]: isUnit
? 'chatcmpl-AfbMVBL30VWqxHMtEWOUi1gulztS0'
? 'chatcmpl-Aupa7af1SkrkThXa5ZLNKFvzyDiPx'
: /.+/,
[ATTR_GEN_AI_RESPONSE_MODEL]: isExpectedResponseModel(
'gpt-4o-mini-2024-07-18',
Expand Down Expand Up @@ -777,7 +777,7 @@ test('fixtures', async suite => {
[ATTR_GEN_AI_SYSTEM]: 'openai',
[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_calls'],
[ATTR_GEN_AI_RESPONSE_ID]: isUnit
? 'chatcmpl-AfbMY0GeHGAEkO2CCeaPqeCp10Mq5'
? 'chatcmpl-AupaAaPk1VYY5tHTMvqzxc8NDoSEN'
: /.+/,
[ATTR_GEN_AI_RESPONSE_MODEL]: isExpectedResponseModel(
'gpt-4o-mini-2024-07-18',
Expand All @@ -787,7 +787,7 @@ test('fixtures', async suite => {
? 140
: isPositiveInteger,
[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: isUnit
? 19
? 20
: isPositiveInteger,
},
scope: {
Expand Down Expand Up @@ -862,7 +862,7 @@ test('fixtures', async suite => {
message: {
tool_calls: [
{
id: isUnit ? 'call_ibw82IbShUYxvRG7J6ojeZVe' : /.+/,
id: isUnit ? 'call_ju2Cqzfdrel1ugvEaW0HtaZ4' : /.+/,
type: 'function',
function: {
name: 'get_delivery_date',
Expand Down Expand Up @@ -909,7 +909,7 @@ test('fixtures', async suite => {
[ATTR_GEN_AI_SYSTEM]: 'openai',
[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_calls'],
[ATTR_GEN_AI_RESPONSE_ID]: isUnit
? 'chatcmpl-AfbMZdIABwae3PqzsHzvjahWPVqL6'
? 'chatcmpl-AupaBny5TtBqCkjiH9q77Czg4vOPt'
: /.+/,
[ATTR_GEN_AI_RESPONSE_MODEL]: isExpectedResponseModel(
'gpt-4o-mini-2024-07-18',
Expand All @@ -919,7 +919,7 @@ test('fixtures', async suite => {
? 140
: isPositiveInteger,
[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: isUnit
? 19
? 20
: isPositiveInteger,
},
scope: {
Expand Down Expand Up @@ -995,7 +995,7 @@ test('fixtures', async suite => {
role: 'assistant',
tool_calls: [
{
id: isUnit ? 'call_ltrRGOHzmLMWSIRAZImscFEy' : /.+/,
id: isUnit ? 'call_5CHeMESVhk3E23kwKzTFuGlZ' : /.+/,
type: 'function',
function: {
name: 'get_delivery_date',
Expand Down Expand Up @@ -1048,15 +1048,15 @@ test('fixtures', async suite => {
[ATTR_GEN_AI_SYSTEM]: 'openai',
[ATTR_GEN_AI_RESPONSE_FINISH_REASONS]: ['tool_calls'],
[ATTR_GEN_AI_RESPONSE_ID]: isUnit
? 'chatcmpl-AfbMawxsp83RQ9QuzFdwpTdBbCEQu'
? 'chatcmpl-AupaBx10BqaJquUN3Vqj27CH52Sqg'
: /.+/,
[ATTR_GEN_AI_RESPONSE_MODEL]: isExpectedResponseModel(
'gpt-4o-mini-2024-07-18',
process.env.TEST_CHAT_MODEL
),
[ATTR_GEN_AI_USAGE_INPUT_TOKENS]: isUnit ? 56 : isPositiveInteger,
[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: isUnit
? 45
? 46
: isPositiveInteger,
},
events: undefined,
Expand Down Expand Up @@ -1104,7 +1104,7 @@ test('fixtures', async suite => {
role: 'assistant',
tool_calls: [
{
id: isUnit ? 'call_c70DUNhsnSAQ0y6d8OkyHQeg' : /.+/,
id: isUnit ? 'call_pPFjIPIb7W7HkxCqGdpTIzVy' : /.+/,
type: 'function',
function: {
name: 'get_weather',
Expand All @@ -1114,7 +1114,7 @@ test('fixtures', async suite => {
},
},
{
id: isUnit ? 'call_5XlUHHFmQpDB0GUeNHNsNYYa' : /.+/,
id: isUnit ? 'call_pORZbhSG8VtXET83iaotru1X' : /.+/,
type: 'function',
function: {
name: 'get_weather',
Expand Down Expand Up @@ -1173,15 +1173,15 @@ test('fixtures', async suite => {
role: 'assistant',
tool_calls: [
{
id: isUnit ? 'call_c70DUNhsnSAQ0y6d8OkyHQeg' : /.+/,
id: isUnit ? 'call_pPFjIPIb7W7HkxCqGdpTIzVy' : /.+/,
type: 'function',
function: {
name: 'get_weather',
arguments: undefined, // This must not be captured.
},
},
{
id: isUnit ? 'call_5XlUHHFmQpDB0GUeNHNsNYYa' : /.+/,
id: isUnit ? 'call_pORZbhSG8VtXET83iaotru1X' : /.+/,
type: 'function',
function: {
name: 'get_weather',
Expand Down
74 changes: 25 additions & 49 deletions packages/instrumentation-openai/test/fixtures/nock-recordings/chat-completion.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,56 +15,32 @@
},
"status": 200,
"response": [
"1f8b0800000000000003000000ffff",
"8c924d4bc4301086effd1521e7adb4bbd5fdb8c91e44413c08228a94349db671d34c48a6a02efbdf25dd8f7651c14b0ef3ccfbe69d49b611635c957cc5b86c04c9d6eaf8ba2aee9faed79be6a5787ebcc3f566fdd57437efb7cbe6432df92428b078074947d585c4d66a2085668fa50341105cd3f92ccbe68b6c96f6a0c5127490d596e20ce35619154f93691627f3385d1cd40d2a099eafd86bc41863dbfe0c394d091f7cc592c9b1d282f7a206be3a3531c61dea50e1c27be54918e293014a3404a68ffe881d35e00c7b9020ccc5b8cb41d57911929a4eeb437d77ba56636d1d16fec04ff54a19e59bdc81f068c2159ed0f29eee22c6defaf1bab3c4dc3a6c2de5841b30c1703adddbf161a9039c1d1821093dd25c4e7e31cb4b20a1b41f6d874b211b2807e5b04ad1950a47201a8dfc33cb6fdefbb195a9ff633f0029c11294b975502a793eefd0e620fcb8bfda4e2bee0373ffe909dabc52a606679ddabf7765f3ab4aa609a409143cda45df000000ffff",
"030097853686fd020000"
],
"rawHeaders": [
"Date",
"Tue, 17 Dec 2024 23:33:51 GMT",
"Content-Type",
"application/json",
"Transfer-Encoding",
"chunked",
"Connection",
"keep-alive",
"access-control-expose-headers",
"X-Request-ID",
"openai-organization",
"user-t27sfkqnaatdbowu2i7ij4dc",
"openai-processing-ms",
"210",
"openai-version",
"2020-10-01",
"x-ratelimit-limit-requests",
"10000",
"x-ratelimit-limit-tokens",
"200000",
"x-ratelimit-remaining-requests",
"9999",
"x-ratelimit-remaining-tokens",
"199783",
"x-ratelimit-reset-requests",
"8.64s",
"x-ratelimit-reset-tokens",
"64ms",
"x-request-id",
"req_c9d258baa91e2e3fdd1774ce95ab8521",
"strict-transport-security",
"max-age=31536000; includeSubDomains; preload",
"CF-Cache-Status",
"DYNAMIC",
"X-Content-Type-Options",
"nosniff",
"Server",
"cloudflare",
"CF-RAY",
"8f3ab86e68adf88e-SIN",
"Content-Encoding",
"gzip",
"alt-svc",
"h3=\":443\"; ma=86400"
"1f8b08000000000000038c92416fdb300c85effe1582ce71e1786e12e4d65d76dc7a1e0a43a168879b2c0a125d6c28f2df0bd969ec621dd08b0ffcf89e1e69be144a69b2faa8349c8dc0105cf93006b3633cf1eef9f0ad71d5238498befef0f468c5e84d56f0e91782bca9ee8087e05088fd8c21a211ccaedbfd97c3b63a54db6a02035b7459d607291b2e07f254d655dd94d5bedc1eaeea331360d247f5b3504aa997e99b737a8b7ff4514d5e5365c0944c8ffa786b524a4776b9a24d4a94c478d19b05027b413f457f1067bc10a8ef80c6dfadbb2276633239a91f9dbbd62fb7671df721f2295df9adde91a7746e239ac43e3f9184839ee8a550ea691a6f7c975887c8439056f837fa6c58d7b39d5e96bac0e6ca84c5b89566b7f9c0acb528865c5a6d47838133da45b9acd28c9678058ad5c8ff66f9c87b1e9b7cff19fb05006010b46d886809decfbbb445cc17f7bfb6db8aa7c03a617c26c0560863fe0d163b33baf90e74fa9b0487b623df630c91e663e842bbafd1eecde9be015d5c8a57000000ffff03008f2ed28a1a030000"
],
"rawHeaders": {
"access-control-expose-headers": "X-Request-ID",
"alt-svc": "h3=\":443\"; ma=86400",
"cf-cache-status": "DYNAMIC",
"cf-ray": "9094dd3b59308438-YVR",
"connection": "close",
"content-encoding": "gzip",
"content-type": "application/json",
"date": "Tue, 28 Jan 2025 23:46:51 GMT",
"openai-organization": "elastic-observability",
"openai-processing-ms": "246",
"openai-version": "2020-10-01",
"server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"transfer-encoding": "chunked",
"x-content-type-options": "nosniff",
"x-ratelimit-limit-requests": "10000",
"x-ratelimit-limit-tokens": "200000",
"x-ratelimit-remaining-requests": "9999",
"x-ratelimit-remaining-tokens": "199783",
"x-ratelimit-reset-requests": "8.64s",
"x-ratelimit-reset-tokens": "64ms",
"x-request-id": "req_7f3ca2d6815c342f85128e4eb97d1b68"
},
"responseIsBinary": false
}
]
Loading

0 comments on commit 36c3745

Please sign in to comment.