Skip to content

feat(vercel): add gen.ai.input.messages + gen.ai.output.messages #711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 25 commits into
base: main
Choose a base branch
from
Open

feat(vercel): add gen.ai.input.messages + gen.ai.output.messages #711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
d58835a
add instro
nina-kollman Sep 9, 2025
92e145e
lint
nina-kollman Sep 10, 2025
6340049
comm
nina-kollman Sep 10, 2025
8e007ec
added test
nina-kollman Sep 10, 2025
224f548
pretty
nina-kollman Sep 10, 2025
f98bdf5
added test
nina-kollman Sep 14, 2025
259191f
fix lint
nina-kollman Sep 14, 2025
2264bf2
add
nina-kollman Sep 14, 2025
ae63671
pretty
nina-kollman Sep 14, 2025
e9268a0
change to 1.37
nina-kollman Sep 16, 2025
50c87c1
change to att
nina-kollman Sep 16, 2025
1ec2dfa
change import
nina-kollman Sep 16, 2025
d47215d
fix build
nina-kollman Sep 16, 2025
baeafd4
incubating
nina-kollman Sep 16, 2025
1fc523a
rollup solution
nina-kollman Sep 16, 2025
9b536b3
fix test
nina-kollman Sep 16, 2025
d62fa05
Revert "rollup solution"
nina-kollman Sep 16, 2025
be4725a
correct import
nina-kollman Sep 16, 2025
6fcf577
try skip
nina-kollman Sep 16, 2025
94e732b
remove file
nina-kollman Sep 16, 2025
2d76807
delete
nina-kollman Sep 16, 2025
4318488
Revert "delete"
nina-kollman Sep 16, 2025
bd83d51
Revert "remove file"
nina-kollman Sep 16, 2025
6ea013e
remove integration
nina-kollman Sep 16, 2025
43de098
delete
nina-kollman Sep 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/ai-semantic-conventions/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
"access": "public"
},
"dependencies": {
"@opentelemetry/api": "^1.9.0"
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/semantic-conventions": "^1.37.0"
},
"homepage": "https://github.com/traceloop/openllmetry-js/tree/main/packages/ai-semantic-conventions",
"gitHead": "ef1e70d6037f7b5c061056ef2be16e3f55f02ed5"
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-anthropic/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"tslib": "^2.8.1"
},
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-bedrock/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"tslib": "^2.8.1"
},
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-chromadb/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"tslib": "^2.8.1"
},
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-cohere/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"tslib": "^2.8.1"
},
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-langchain/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"tslib": "^2.8.1"
Comment on lines +45 to 47
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

🧩 Analysis chain

Bump to @opentelemetry/semantic-conventions ^1.37.0 looks good; verify Node engines vs OTel 2.x.

Ensure the "engines": { "node": ">=14" } requirement is compatible with your OTel 2.x deps at runtime; if OTel requires a higher Node version in practice, bump the engines field accordingly.

Run the checks below to confirm workspace consistency and engine alignment:

🏁 Script executed:

#!/bin/bash
# Show all packages depending on @opentelemetry/semantic-conventions and their requested ranges
rg -n --glob "packages/**/package.json" '"@opentelemetry/semantic-conventions":' -C2

# Print this package's Node engine
jq -r '.engines' packages/instrumentation-langchain/package.json

# (Optional) Quick peek at OTel SDK packages requested here to cross-check Node support
rg -n --glob "packages/**/package.json" '"@opentelemetry/(sdk|sdk-trace-node|core|api)":' -C2

Length of output: 17312

Bump Node engine to >=18.19 to match OpenTelemetry JS SDK 2.x.

OpenTelemetry JS SDK 2.x raised its minimum Node requirement to ^18.19.0 || >=20.6.0; this repo has @opentelemetry/*@2.x in multiple packages while packages/instrumentation-langchain/package.json still declares "engines": { "node": ">=14" } — update engines to at least "^18.19.0 || >=20.6.0" (or ">=18.19.0") to avoid install/runtime breakage. (opentelemetry.io)

🤖 Prompt for AI Agents 
In packages/instrumentation-langchain/package.json around lines 45 to 47, the
package.json still declares "engines": { "node": ">=14" } while OpenTelemetry JS
SDK 2.x requires Node >=18.19 (or ^18.19.0 || >=20.6.0); update the
"engines.node" field to at least ">=18.19.0" (or use "^18.19.0 || >=20.6.0") to
match the OpenTelemetry dependency requirement and prevent install/runtime
breakage, then run an install/test to ensure compatibility across the monorepo.

},
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-llamaindex/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"lodash": "^4.17.21",
"tslib": "^2.8.1"
Expand Down
2 changes: 1 addition & 1 deletion packages/instrumentation-openai/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.203.0",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@opentelemetry/semantic-conventions": "^1.37.0",
"@traceloop/ai-semantic-conventions": "workspace:*",
"js-tiktoken": "^1.0.20",
"tslib": "^2.8.1"
Expand Down
94 changes: 94 additions & 0 deletions packages/instrumentation-openai/test/instrumentation.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,50 @@ import {
InMemorySpanExporter,
SimpleSpanProcessor,
} from "@opentelemetry/sdk-trace-node";
import {
ATTR_GEN_AI_INPUT_MESSAGES,
ATTR_GEN_AI_OUTPUT_MESSAGES,
} from "@opentelemetry/semantic-conventions/build/src/experimental_attributes";

// Minimal transformation function to test ATTR_GEN_AI_INPUT_MESSAGES and ATTR_GEN_AI_OUTPUT_MESSAGES
const transformToStandardFormat = (attributes: any) => {
// Transform prompts to ATTR_GEN_AI_INPUT_MESSAGES
const inputMessages = [];
let i = 0;
while (attributes[`${SpanAttributes.LLM_PROMPTS}.${i}.role`]) {
const role = attributes[`${SpanAttributes.LLM_PROMPTS}.${i}.role`];
const content = attributes[`${SpanAttributes.LLM_PROMPTS}.${i}.content`];
if (role && content) {
inputMessages.push({
role,
parts: [{ type: "text", content }],
});
}
i++;
}
if (inputMessages.length > 0) {
attributes[ATTR_GEN_AI_INPUT_MESSAGES] = JSON.stringify(inputMessages);
}

// Transform completions to SemanticAttributes.GEN_AI_OUTPUT_MESSAGES
const outputMessages = [];
let j = 0;
while (attributes[`${SpanAttributes.LLM_COMPLETIONS}.${j}.role`]) {
const role = attributes[`${SpanAttributes.LLM_COMPLETIONS}.${j}.role`];
const content =
attributes[`${SpanAttributes.LLM_COMPLETIONS}.${j}.content`];
if (role && content) {
outputMessages.push({
role,
parts: [{ type: "text", content }],
});
}
j++;
}
if (outputMessages.length > 0) {
attributes[ATTR_GEN_AI_OUTPUT_MESSAGES] = JSON.stringify(outputMessages);
}
};

import type * as OpenAIModule from "openai";
import { toFile } from "openai";
Expand Down Expand Up @@ -878,4 +922,54 @@ describe("Test OpenAI instrumentation", async function () {
4160,
);
});

it("should set ATTR_GEN_AI_INPUT_MESSAGES and ATTR_GEN_AI_OUTPUT_MESSAGES attributes for chat completions", async () => {
const result = await openai.chat.completions.create({
messages: [
{ role: "user", content: "Tell me a joke about OpenTelemetry" },
],
model: "gpt-3.5-turbo",
});

const spans = memoryExporter.getFinishedSpans();
const completionSpan = spans.find((span) => span.name === "openai.chat");

assert.ok(result);
assert.ok(completionSpan);

// Apply transformations to create ATTR_GEN_AI_INPUT_MESSAGES and ATTR_GEN_AI_OUTPUT_MESSAGES
transformToStandardFormat(completionSpan.attributes);

// Verify ATTR_GEN_AI_INPUT_MESSAGES attribute exists and is valid JSON
assert.ok(completionSpan.attributes[ATTR_GEN_AI_INPUT_MESSAGES]);
const inputMessages = JSON.parse(
completionSpan.attributes[ATTR_GEN_AI_INPUT_MESSAGES] as string,
);
assert.ok(Array.isArray(inputMessages));
assert.strictEqual(inputMessages.length, 1);

// Check user message structure
assert.strictEqual(inputMessages[0].role, "user");
assert.ok(Array.isArray(inputMessages[0].parts));
assert.strictEqual(inputMessages[0].parts[0].type, "text");
assert.strictEqual(
inputMessages[0].parts[0].content,
"Tell me a joke about OpenTelemetry",
);

// Verify ATTR_GEN_AI_OUTPUT_MESSAGES attribute exists and is valid JSON
assert.ok(completionSpan.attributes[ATTR_GEN_AI_OUTPUT_MESSAGES]);
const outputMessages = JSON.parse(
completionSpan.attributes[ATTR_GEN_AI_OUTPUT_MESSAGES] as string,
);
assert.ok(Array.isArray(outputMessages));
assert.strictEqual(outputMessages.length, 1);

// Check assistant response structure
assert.strictEqual(outputMessages[0].role, "assistant");
assert.ok(Array.isArray(outputMessages[0].parts));
assert.strictEqual(outputMessages[0].parts[0].type, "text");
assert.ok(outputMessages[0].parts[0].content);
assert.ok(typeof outputMessages[0].parts[0].content === "string");
});
});
Loading
Loading