Visitar URL original
fix(ai): read Copilot reasoning tokens and content-filter categories · anomalyco/opencode@14a1826 · GitHub
Skip to content

Commit 14a1826

Browse files
exe.dev userrekram1-node
authored andcommitted
fix(ai): read Copilot reasoning tokens and content-filter categories
1 parent ddc94ca commit 14a1826

6 files changed

Lines changed: 139 additions & 4 deletions

File tree

‎packages/ai/src/protocols/open-responses.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -913,6 +913,36 @@ export const mapUsage = (
913913
})
914914
}
915915

916+
// GitHub Copilot lists the filters that ran; the blocking one names its rule in `content_filter_raw`.
917+
const decodeContentFilters = Schema.decodeUnknownOption(
918+
Schema.Array(
919+
Schema.Struct({
920+
source_type: Schema.optional(Schema.String),
921+
blocked: Schema.optional(Schema.Boolean),
922+
content_filter_raw: Schema.optional(
923+
Schema.Array(
924+
Schema.Struct({
925+
action: Schema.optional(Schema.String),
926+
label: Schema.optional(Schema.String),
927+
result: Schema.optional(Schema.Unknown),
928+
}),
929+
),
930+
),
931+
content_filter_results: Schema.optional(Schema.Unknown),
932+
}),
933+
),
934+
)
935+
936+
const contentFilterCategory = (raw: unknown) => {
937+
const blocked = Option.getOrElse(decodeContentFilters(raw), () => []).filter((filter) => filter.blocked)
938+
const filter = blocked.find((item) => item.source_type === "completion") ?? blocked[0]
939+
if (!filter) return undefined
940+
return (
941+
filter.content_filter_raw?.find((rule) => rule.action === "BLOCK" && rule.result === true)?.label ??
942+
ProviderShared.contentFilterCategory(filter.content_filter_results)
943+
)
944+
}
945+
916946
const mapFinishReason = (event: Event, hasFunctionCall: boolean): FinishReason => {
917947
const reason = event.response?.incomplete_details?.reason
918948
if (reason === undefined) {
@@ -1397,10 +1427,12 @@ const onResponseFinish = Effect.fnUntraced(function* (state: ParserState, event:
13971427
}
13981428
events.push(...pending.events)
13991429
}
1430+
const normalized = mapFinishReason(event, current.hasFunctionCall)
14001431
const lifecycle = Lifecycle.finish(current.lifecycle, events, {
14011432
reason: {
1402-
normalized: mapFinishReason(event, current.hasFunctionCall),
1433+
normalized,
14031434
raw: event.response?.incomplete_details?.reason,
1435+
...(normalized === "content-filter" ? { category: contentFilterCategory(event.response?.content_filters) } : {}),
14041436
},
14051437
// Copilot reports its billing on the terminal event, beside `response`.
14061438
usage: mapUsage(event.response?.usage, current.providerMetadataKey, event, event.response),

‎packages/ai/src/protocols/openai-chat.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,8 @@ export const OpenAIChatUsage = Schema.StructWithRest(
265265
prompt_cache_hit_tokens: optionalNull(Schema.Number),
266266
cache_read_input_tokens: optionalNull(Schema.Number),
267267
cache_created_input_tokens: optionalNull(Schema.Number),
268+
// GitHub Copilot reports reasoning at the top level instead of in `completion_tokens_details`.
269+
reasoning_tokens: optionalNull(Schema.Number),
268270
prompt_tokens_details: optionalNull(
269271
Schema.StructWithRest(
270272
Schema.Struct({
@@ -967,15 +969,22 @@ const mapUsage = (
967969
): Usage | undefined => {
968970
if (!usage) return undefined
969971
const input = usage.prompt_tokens ?? undefined
970-
const output = usage.completion_tokens ?? undefined
972+
const completion = usage.completion_tokens ?? undefined
971973
const cached =
972974
usage.prompt_tokens_details?.cached_tokens ??
973975
usage.prompt_cache_hit_tokens ??
974976
usage.cached_tokens ??
975977
usage.cache_read_input_tokens ??
976978
undefined
977979
const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? usage.cache_created_input_tokens ?? undefined
978-
const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? undefined
980+
const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? usage.reasoning_tokens ?? undefined
981+
// Copilot's Gemini counts reasoning beside `completion_tokens` rather than inside it; the total tells which.
982+
const output =
983+
completion !== undefined &&
984+
reasoning !== undefined &&
985+
usage.total_tokens === ProviderShared.sumTokens(input, completion, reasoning)
986+
? completion + reasoning
987+
: completion
979988
const nonCached = ProviderShared.subtractTokens(input, ProviderShared.sumTokens(cached, cacheWrite))
980989
return new Usage({
981990
inputTokens: input,
@@ -1128,6 +1137,14 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
11281137
? {
11291138
normalized: yield* mapFinishReason(event, rawFinishReason),
11301139
raw: choice?.native_finish_reason ?? rawFinishReason,
1140+
...(rawFinishReason === "content_filter"
1141+
? {
1142+
category: ProviderShared.contentFilterCategory(
1143+
choice?.delta?.copilot_annotations,
1144+
choice?.content_filter_results,
1145+
),
1146+
}
1147+
: {}),
11311148
}
11321149
: state.finishReason
11331150
const delta = choice?.delta

‎packages/ai/src/protocols/shared.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,15 @@ export const reportedBilling = (usage: unknown, ...frames: ReadonlyArray<unknown
7676
return nanoAiu === undefined ? {} : { cost: nanoAiu / 100_000_000_000 }
7777
}
7878

79+
/**
80+
* The filter that blocked a response, from maps keyed by filter name: Azure's `content_filter_results`
81+
* (`{ hate: { filtered: true } }`) or GitHub Copilot's `copilot_annotations` (`{ TextCopyright: [...] }`).
82+
*/
83+
export const contentFilterCategory = (...sources: ReadonlyArray<unknown>) =>
84+
sources
85+
.flatMap((source) => (isRecord(source) ? Object.entries(source) : []))
86+
.find(([, value]) => (isRecord(value) && "filtered" in value ? value.filtered === true : Boolean(value)))?.[0]
87+
7988
/**
8089
* Accepts stream frames that carry no `type`, alongside the typed event schema. Gateways inject such frames as
8190
* keep-alives, so they decode as an ignored event; one with an `error` still decodes as an error event.

‎packages/ai/test/provider/openai-chat.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,6 +1264,25 @@ describe("OpenAI Chat route", () => {
12641264
}),
12651265
)
12661266

1267+
it.effect("reads top-level reasoning tokens whether or not completion tokens include them", () =>
1268+
Effect.gen(function* () {
1269+
const generate = (usage: object) =>
1270+
LLMClient.generate(request).pipe(
1271+
Effect.provide(fixedResponse(sseEvents(deltaChunk({ content: "Hi" }, "stop"), usageChunk(usage)))),
1272+
Effect.map((response) => response.usage),
1273+
)
1274+
1275+
// Copilot's Gemini: total = prompt + completion + reasoning.
1276+
expect(
1277+
yield* generate({ prompt_tokens: 7, completion_tokens: 2, total_tokens: 62, reasoning_tokens: 53 }),
1278+
).toMatchObject({ outputTokens: 55, reasoningTokens: 53, totalTokens: 62 })
1279+
// Copilot's Kimi: total = prompt + completion, which already includes reasoning.
1280+
expect(
1281+
yield* generate({ prompt_tokens: 96, completion_tokens: 50, total_tokens: 146, reasoning_tokens: 35 }),
1282+
).toMatchObject({ outputTokens: 50, reasoningTokens: 35, totalTokens: 146 })
1283+
}),
1284+
)
1285+
12671286
it.effect("converts the GitHub Copilot billing beside usage to USD", () =>
12681287
Effect.gen(function* () {
12691288
const response = yield* LLMClient.generate(request).pipe(

‎packages/ai/test/provider/openai-compatible-chat.test.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,31 @@ describe("OpenAI-compatible Chat route", () => {
657657
}),
658658
)
659659

660+
it.effect("names the content filter that blocked a response", () =>
661+
Effect.gen(function* () {
662+
const finish = (choice: object) =>
663+
LLMClient.generate(request).pipe(
664+
Effect.provide(
665+
fixedResponse(
666+
sseEvents({ id: "chatcmpl_fixture", choices: [{ finish_reason: "content_filter", ...choice }] }),
667+
),
668+
),
669+
Effect.map((response) => response.finishReason),
670+
)
671+
672+
expect(yield* finish({ delta: { copilot_annotations: { TextCopyright: [{ id: 1 }] } } })).toMatchObject({
673+
normalized: "content-filter",
674+
category: "TextCopyright",
675+
})
676+
expect(
677+
yield* finish({
678+
delta: {},
679+
content_filter_results: { hate: { filtered: false }, violence: { filtered: true } },
680+
}),
681+
).toMatchObject({ normalized: "content-filter", category: "violence" })
682+
}),
683+
)
684+
660685
it.effect("preserves late text and reasoning for replay without changing the finished tool call", () =>
661686
Effect.gen(function* () {
662687
const details = [{ type: "reasoning.encrypted", data: "opaque", id: "late-reasoning" }]
@@ -723,7 +748,11 @@ describe("OpenAI-compatible Chat route", () => {
723748

724749
expect(response.reasoning).toBe("Read the readme first. Then the manifest.")
725750
expect(response.events.filter(LLMEvent.is.reasoningEnd)).toMatchObject([
726-
{ providerMetadata: { "github-copilot": { reasoningField: "reasoning_text", reasoningOpaque: "opaque-second" } } },
751+
{
752+
providerMetadata: {
753+
"github-copilot": { reasoningField: "reasoning_text", reasoningOpaque: "opaque-second" },
754+
},
755+
},
727756
])
728757
expect(response.toolCalls.map((call) => call.id)).toEqual(["call_first", "call_second"])
729758
const replay = yield* compileRequest(ai.request({ model: copilot, messages: [response.message] }))

‎packages/ai/test/provider/openai-compatible-responses.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,35 @@ describe("Open Responses-compatible route", () => {
444444
}),
445445
)
446446

447+
it.effect("names the GitHub Copilot content filter that blocked a response", () =>
448+
Effect.gen(function* () {
449+
const model = configure({ apiKey: "test-key", baseURL: "https://api.githubcopilot.test" }).model("gpt-5")
450+
const response = yield* LLMClient.generate(ai.request({ model, prompt: "Say hello." })).pipe(
451+
Effect.provide(
452+
fixedResponse(
453+
sseEvents({
454+
type: "response.incomplete",
455+
response: {
456+
id: "resp_1",
457+
incomplete_details: { reason: "content_filter" },
458+
content_filters: [
459+
{ source_type: "prompt", blocked: false },
460+
{
461+
source_type: "completion",
462+
blocked: true,
463+
content_filter_raw: [{ action: "BLOCK", label: "TextCopyright", result: true }],
464+
},
465+
],
466+
},
467+
}),
468+
),
469+
),
470+
)
471+
472+
expect(response.finishReason).toMatchObject({ normalized: "content-filter", category: "TextCopyright" })
473+
}),
474+
)
475+
447476
it.effect("converts the GitHub Copilot billing on the completed event to USD", () =>
448477
Effect.gen(function* () {
449478
const model = configure({ apiKey: "test-key", baseURL: "https://api.githubcopilot.test" }).model("gpt-5")

0 commit comments

Comments
 (0)