Visitar URL original
fix(core): show the start of tool output that exceeds the limit by rekram1-node · Pull Request #63272 · anomalyco/opencode · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 69 additions & 23 deletions packages/core/src/tool-output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,48 +75,56 @@ const layer = Layer.effect(
const truncate = Effect.fnUntraced(function* (result: Result) {
if (result.metadata?.truncated !== undefined) return result
const content = result.content
const text = content.flatMap((item) => (item.type === "text" ? [item.text] : [])).join("\n")
// A line break joins text parts unless the text before already ends in one.
const text = content.reduce(
(joined, item) =>
item.type !== "text"
? joined
: joined === "" || joined.endsWith("\n")
? joined + item.text
: `${joined}\n${item.text}`,
"",
)
const limits = state.get()
const lines = text.split("\n")
if (text.endsWith("\n")) lines.pop()
const totalBytes = Buffer.byteLength(text, "utf-8")
if (lines.length <= limits.maxLines && totalBytes <= limits.maxBytes)
const totalLines = countLines(text)
if (totalLines <= limits.maxLines && Buffer.byteLength(text, "utf-8") <= limits.maxBytes)
return { ...result, metadata: { ...result.metadata, truncated: false } }

const kept: string[] = []
let bytes = 0
for (const line of lines.slice(0, limits.maxLines)) {
const size = Buffer.byteLength(line, "utf-8") + (kept.length > 0 ? 1 : 0)
if (bytes + size > limits.maxBytes) break
kept.push(line)
bytes += size
}
// Cut once, at the line or byte limit, whichever comes first, so a line too long for the space left shows its
// start. The preview never ends on a line break.
const end = Math.min(afterLines(text, limits.maxLines), byteBoundary(text, limits.maxBytes))
const shown = text.slice(0, text.endsWith("\r\n", end) ? end - 2 : text.endsWith("\n", end) ? end - 1 : end)
const file = path.join(directory, fileName())
const failure = yield* fs.ensureDir(directory).pipe(
Effect.andThen(fs.writeFileString(file, text)),
Effect.as(undefined),
Effect.catch((error) =>
Effect.logWarning("failed to save full tool output", { file, cause: error }).pipe(
Effect.as(describe(error)),
),
Effect.logWarning("failed to save full tool output", { file, cause: error }).pipe(Effect.as(describe(error))),
),
)
const shown = kept.length > 0 ? `lines 1-${kept.length}` : "0 lines"
const marker = `[showing ${shown} of ${lines.length}; ${
// Counted before the final line break is stripped, so a blank last line still counts.
const shownLines = countLines(text.slice(0, end))
const lastLineCut =
shownLines > 0 &&
shown.length < text.length &&
!text.startsWith("\n", shown.length) &&
!text.startsWith("\r\n", shown.length)
const range = shownLines === 0 ? "0 lines" : `lines 1-${shownLines}`
const marker = `[showing ${range} of ${totalLines}${lastLineCut ? " (last line cut)" : ""}; ${
failure === undefined ? `full output saved to ${file}` : `full output could not be saved: ${failure}`
}]`
const bounded: Tool.Content[] = []
let remaining = kept.join("\n").length
let seenText = false
let remaining = shown.length
let separator = false
let marked = false
for (const item of content) {
if (item.type === "file") {
bounded.push(item)
continue
}
if (seenText && remaining > 0) remaining--
seenText = true
if (remaining >= item.text.length) {
if (separator && remaining > 0) remaining--
separator = item.text !== "" && !item.text.endsWith("\n")
if (!marked && remaining >= item.text.length) {
bounded.push(item)
remaining -= item.text.length
continue
Expand Down Expand Up @@ -172,3 +180,41 @@ export const node = makeLocationNode({
layer,
deps: [FSUtil.node, Global.node, cleanupNode],
})

/** How many lines `text` has, not counting an empty one after a final line break. Counts without splitting. */
function countLines(text: string) {
if (text === "") return 0
let count = 1
for (let index = text.indexOf("\n"); index !== -1 && index < text.length - 1; index = text.indexOf("\n", index + 1))
count++
return count
}

/** Where the line after the first `count` lines starts, or the end of `text`. */
function afterLines(text: string, count: number) {
let index = 0
for (let line = 0; line < count; line++) {
const next = text.indexOf("\n", index)
if (next === -1) return text.length
index = next + 1
}
return index
}

const encoder = new TextEncoder()
const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" })

/** The longest start of `text` that fits in `budget` UTF-8 bytes without splitting a character, emoji included. */
function byteBoundary(text: string, budget: number) {
// encodeInto only encodes what fits in the buffer, and stops at a whole code point.
const index = encoder.encodeInto(text, new Uint8Array(Math.max(0, budget))).read
if (index === text.length || index === 0) return index
const before = text.charCodeAt(index - 1)
const after = text.charCodeAt(index)
// Two ASCII characters always have a boundary between them, except the two halves of \r\n.
if (before < 0x80 && after < 0x80) return before === 13 && after === 10 ? index - 1 : index
// Otherwise back off to the start of the grapheme the cut is in. A line break always ends one, so segmenting from the
// start of the line keeps flag pairs and long emoji sequences exact; one code point past the cut gives the context.
const start = text.lastIndexOf("\n", index - 1) + 1
return start + (graphemes.segment(text.slice(start, index + 2)).containing(index - start)?.index ?? index - start)
}
174 changes: 169 additions & 5 deletions packages/core/test/tool-output.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,31 +59,192 @@ describe("ToolOutput", () => {
{ type: "text", text: "one\ntwo" },
{
type: "text",
text: expect.stringMatching(/^\[showing lines 1-2 of 3; full output could not be saved: E[A-Z]+: [^\n]+\]$/),
text: expect.stringMatching(
/^\[showing lines 1-2 of 3; full output could not be saved: E[A-Z]+: [^\n]+\]$/,
),
},
])
}),
{ maxLines: 2, maxBytes: 1_000 },
),
)

it.live("reports lines shown under the byte limit", () =>
it.live("cuts at the byte limit, inside a line when it falls there", () =>
withStore(
(output) =>
Effect.gen(function* () {
const result = yield* output.truncate({ content: [{ type: "text", text: "one\ntwo" }] })
expect(result.content).toEqual([
{ type: "text", text: "one" },
{ type: "text", text: "one\nt" },
{
type: "text",
text: expect.stringMatching(/^\[showing lines 1-1 of 2; full output saved to .+\]$/),
text: expect.stringMatching(/^\[showing lines 1-2 of 2 \(last line cut\); full output saved to .+\]$/),
},
])
}),
{ maxLines: 100, maxBytes: 5 },
),
)

it.live("shows the start of a single line longer than the byte limit", () =>
withStore(
(output) =>
Effect.gen(function* () {
const text = JSON.stringify({ items: Array.from({ length: 50 }, (_, index) => index) })
const result = yield* output.truncate({ content: [{ type: "text", text }] })
expect(result.content).toEqual([
{ type: "text", text: text.slice(0, 20) },
{
type: "text",
text: expect.stringMatching(
new RegExp("^\\[showing lines 1-1 of 1 \\(last line cut\\); full output saved to .+\\]$"),
),
},
])
}),
{ maxLines: 100, maxBytes: 20 },
),
)

it.live("never splits a character or an emoji at the cut", () =>
withStore(
(output) =>
Effect.gen(function* () {
const family = "👨‍👩‍👧‍👦"
const cases = [
{ text: "ab✓✓✓", shown: "ab✓" },
{ text: `ab${family}cd`, shown: "ab" },
{ text: "ab👍🏽cd", shown: "ab" },
{ text: "ab🇯🇵cd", shown: "ab" },
{ text: "ab𝒳𝒳", shown: "ab𝒳" },
]
for (const item of cases) {
const result = yield* output.truncate({ content: [{ type: "text", text: item.text }] })
const first = result.content[0]
expect(first).toEqual({ type: "text", text: item.shown })
expect(first?.type === "text" && first.text.isWellFormed()).toBe(true)
}
}),
{ maxLines: 100, maxBytes: 7 },
),
)

it.live("keeps flags and long clusters whole however long the text before them", () =>
withStore(
(output) =>
Effect.gen(function* () {
const flags = yield* output.truncate({ content: [{ type: "text", text: "a" + "🇯🇵".repeat(40) }] })
expect(flags.content[0]).toEqual({ type: "text", text: "a" + "🇯🇵".repeat(20) })
const cluster = "👍" + "🏽\u0301".repeat(60)
const long = yield* output.truncate({ content: [{ type: "text", text: `ab${cluster}z` }] })
expect(long.content[0]).toEqual({ type: "text", text: "ab" })
}),
{ maxLines: 100, maxBytes: 165 },
),
)

it.live("cuts inside a later text part and leaves out the parts after it", () =>
withStore(
(output) =>
Effect.gen(function* () {
const inside = yield* output.truncate({
content: [
{ type: "text", text: "" },
{ type: "text", text: "abc" },
{ type: "text", text: "defgh" },
{ type: "text", text: "" },
],
})
expect(inside.content).toEqual([
{ type: "text", text: "" },
{ type: "text", text: "abc" },
{ type: "text", text: "de" },
{
type: "text",
text: expect.stringMatching(/^\[showing lines 1-2 of 2 \(last line cut\);/),
},
])
const between = yield* output.truncate({
content: [
{ type: "text", text: "abcde" },
{ type: "text", text: "fgh" },
],
})
expect(between.content).toEqual([
{ type: "text", text: "abcde" },
{ type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 2;/) },
])
}),
{ maxLines: 100, maxBytes: 6 },
),
)

it.live("ends a line cut before its line break", () =>
withStore(
(output) =>
Effect.gen(function* () {
const crlf = yield* output.truncate({ content: [{ type: "text", text: "one\r\ntwo\r\nthree" }] })
expect(crlf.content).toEqual([
{ type: "text", text: "one" },
{ type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 3;/) },
])
const blank = yield* output.truncate({ content: [{ type: "text", text: "\nfoo\nbar" }] })
expect(blank.content).toEqual([{ type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 3;/) }])
}),
{ maxLines: 1, maxBytes: 1_000 },
),
)

it.live("counts a blank last line of the preview", () =>
withStore(
(output) =>
Effect.gen(function* () {
const result = yield* output.truncate({ content: [{ type: "text", text: "a\n\n\nb" }] })
expect(result.content).toEqual([
{ type: "text", text: "a\n" },
{ type: "text", text: expect.stringMatching(/^\[showing lines 1-2 of 4;/) },
])
}),
{ maxLines: 2, maxBytes: 1_000 },
),
)

it.live("shows nothing under a zero line limit", () =>
withStore(
(output) =>
Effect.gen(function* () {
const result = yield* output.truncate({ content: [{ type: "text", text: "secret" }] })
expect(result.content).toEqual([{ type: "text", text: expect.stringMatching(/^\[showing 0 lines of 1;/) }])
}),
{ maxLines: 0, maxBytes: 1_000 },
),
)

it.live("joins text parts with a line break only where one is missing", () =>
withStore(
(output, fs) =>
Effect.gen(function* () {
const result = yield* output.truncate({
content: [
{ type: "text", text: "one\n" },
{ type: "text", text: "two" },
{ type: "text", text: "three" },
],
})
const outputPath = result.metadata?.outputPath
expect(typeof outputPath).toBe("string")
if (typeof outputPath !== "string") return
expect(yield* fs.readFileString(outputPath)).toBe("one\ntwo\nthree")
expect(result.content).toEqual([
{ type: "text", text: "one\n" },
{ type: "text", text: "two" },
{ type: "text", text: expect.stringMatching(/^\[showing lines 1-2 of 3;/) },
])
}),
{ maxLines: 2, maxBytes: 1_000 },
),
)

it.live("preserves mixed content ordering", () =>
withStore(
(output) =>
Expand Down Expand Up @@ -144,7 +305,10 @@ describe("ToolOutput", () => {
const result = yield* output.truncate({ content: [{ type: "text", text: "one\n" }] })
expect(result.content).toEqual([
{ type: "text", text: "one" },
{ type: "text", text: expect.stringMatching(/^\[showing lines 1-1 of 1; full output saved to /) },
{
type: "text",
text: expect.stringMatching(/^\[showing lines 1-1 of 1; full output saved to /),
},
])
}),
{ maxLines: 2, maxBytes: 3 },
Expand Down
Loading