Visitar URL original
fix: a failed prompt leaves no workspace verdict, and the workspace t… · AltimateAI/altimate-code@749495d · GitHub
Skip to content

Commit 749495d

Browse files
fix: a failed prompt leaves no workspace verdict, and the workspace telemetry review's follow-ups (#1711)
1 parent 6d84c5a commit 749495d

9 files changed

Lines changed: 314 additions & 15 deletions

File tree

‎docs/docs/reference/telemetry.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ Other rows gain workspace fields in a linked project:
114114

115115
- Every event carries `client: "altimate-code"`, which names the product that sent it. `source` still says how the CLI was launched.
116116
- Rows of a turn in a linked project carry that project's `workspace_id`: `session_start`, `session_end`, `first_prompt_sent`, `first_job_completed`, `skill_used`, `skill_published`, `memory_injection`, `agent_outcome`, `task_classified`, `task_outcome_signal`, `tool_chain_outcome`, `core_failure`, `error_fingerprint`, `plan_revision`, `sql_quality` and `tool_call`.
117+
- With `ALTIMATE_DISABLE_WORKSPACE` set, no `first_prompt_sent` or `first_job_completed` is sent for the project's saved link, and an onboarding session's own rows name no workspace. The turn's other rows still name that link, read from the local cache with no request; a pin from the IDE extension is not read. Once the switch is off, the workspace's first rows are sent as usual.
117118
- `session_end` counts the workspace skills, memory blocks and integration calls the session used. `memory_injection` counts the workspace's blocks. `task_outcome_signal` says whether workspace context was used.
118119
- A call to a workspace integration through the engine is reported as a `tool_call` with `tool_type: "mcp"` and `via_workspace: true`. As for other tools, there are no arguments or outputs, and an error message is masked and capped at 500 characters.
119120
- An `endpoint` value is a route label with every id and path segment replaced by `:id`, for example `POST /datamate-project-bindings/bind`. It never contains an id, a file path or a query string.

‎packages/opencode/src/altimate/plugin/onboarding-telemetry.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,14 +264,20 @@ export async function OnboardingTelemetryPlugin(_input: PluginInput): Promise<Ho
264264
) {
265265
// altimate_change — the workspace of the turn that ran the tool, carried by the hook input
266266
// from the trigger site: a tool returning after its turn was cancelled and replaced still
267-
// names its own turn's workspace, not the replacement's.
267+
// names its own turn's workspace, not the replacement's. None with the kill switch set.
268+
const workspaceId = WorkspaceFirstEvents.firstEventWorkspace(input.workspaceId)
268269
void OnboardingTelemetry.emit(
269-
{ type: "first_job_completed", job, workspace_id: input.workspaceId, origin: "onboarding" },
270+
{ type: "first_job_completed", job, workspace_id: workspaceId, origin: "onboarding" },
270271
input.sessionID,
271272
)
272273
// Marked so the end of this turn, or a later session, doesn't send the workspace's row as well.
273-
if (input.workspaceId !== undefined) {
274-
await WorkspaceFirstEvents.claim("first_job_completed", input.workspaceId)
274+
// Under the credential of the turn that ran the tool, which its workspace id belongs to.
275+
if (workspaceId !== undefined) {
276+
await WorkspaceFirstEvents.claim(
277+
"first_job_completed",
278+
workspaceId,
279+
WorkspaceFirstEvents.callScope(input.callID),
280+
)
275281
}
276282
}
277283
},

‎packages/opencode/src/altimate/workspace/first-events.ts‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,13 @@
1010
// agent does the work with other tools. So a job here is a turn with at least one tool call that
1111
// didn't fail, ending `completed`, the outcome `task_outcome_signal` reports as `accepted`. It
1212
// reports as `something_else` because nothing says which job it was.
13+
//
14+
// With the workspace kill switch set, no first event names the project's saved link, so a project
15+
// linked before the switch was set isn't counted as that workspace's first prompt or job.
1316
import { createHash } from "node:crypto"
1417
import { mkdir, writeFile } from "node:fs/promises"
1518
import path from "node:path"
19+
import { Flag } from "@opencode-ai/core/flag/flag"
1620
import { Global } from "@/global"
1721
import { Log } from "@/altimate/util/log"
1822
import { Telemetry } from "@/altimate/telemetry"
@@ -80,8 +84,38 @@ export async function claim(event: FirstEvent, workspaceId: number, scope?: Scop
8084
}
8185
}
8286

87+
/** The credential scope of the turn each tool call ran in, while that call's `tool.execute.after`
88+
* plugins run. The trigger site notes it from its own turn, so a mark made there names the tenant the
89+
* call's workspace id belongs to, even when the credential has switched since, or the turn was
90+
* cancelled and replaced before the tool returned. Keyed by call id, not session: a session's later
91+
* turn has its own calls. */
92+
const callScopes = new Map<string, Scope>()
93+
94+
/** Notes `scope` for `callID` until the returned handle is disposed. */
95+
export function noteCallScope(callID: string | undefined, scope: Scope | undefined): Disposable {
96+
if (callID === undefined || !scope) return { [Symbol.dispose]() {} }
97+
callScopes.set(callID, scope)
98+
return {
99+
[Symbol.dispose]() {
100+
if (callScopes.get(callID) === scope) callScopes.delete(callID)
101+
},
102+
}
103+
}
104+
105+
/** The scope noted for a tool call, while its plugins run. */
106+
export function callScope(callID: string): Scope | undefined {
107+
return callScopes.get(callID)
108+
}
109+
110+
/** The workspace a first event may name: none while the workspace kill switch is set. Nothing is
111+
* marked then either, so the workspace's rows are still sent once the switch is off. */
112+
export function firstEventWorkspace(workspaceId: number | undefined): number | undefined {
113+
return Flag.ALTIMATE_DISABLE_WORKSPACE ? undefined : workspaceId
114+
}
115+
83116
/** A typed prompt: the onboarding row and the workspace row are one row when both apply. */
84-
export async function promptSent(sessionID: string, workspaceId: number | undefined, scope?: Scope): Promise<void> {
117+
export async function promptSent(sessionID: string, turnWorkspaceId: number | undefined, scope?: Scope): Promise<void> {
118+
const workspaceId = firstEventWorkspace(turnWorkspaceId)
85119
const onboarding = OnboardingTelemetry.isOnboardingSession(sessionID) && OnboardingTelemetry.claimFirstPrompt(sessionID)
86120
const inWorkspace = workspaceId !== undefined && (await claim("first_prompt_sent", workspaceId, scope))
87121
if (onboarding || inWorkspace) {
@@ -93,7 +127,12 @@ export async function promptSent(sessionID: string, workspaceId: number | undefi
93127
}
94128

95129
/** A turn in a workspace that ended `completed` with at least one tool call that didn't fail. */
96-
export async function jobCompleted(sessionID: string, workspaceId: number | undefined, scope?: Scope): Promise<void> {
130+
export async function jobCompleted(
131+
sessionID: string,
132+
turnWorkspaceId: number | undefined,
133+
scope?: Scope,
134+
): Promise<void> {
135+
const workspaceId = firstEventWorkspace(turnWorkspaceId)
97136
if (workspaceId === undefined) return
98137
if (await claim("first_job_completed", workspaceId, scope)) {
99138
await OnboardingTelemetry.emit(
@@ -106,5 +145,6 @@ export async function jobCompleted(sessionID: string, workspaceId: number | unde
106145
/** Test seam: drops this process's memory of the marks, as a new process would start. */
107146
export function resetForTest() {
108147
sent.clear()
148+
callScopes.clear()
109149
warnedUnwritable = false
110150
}

‎packages/opencode/src/altimate/workspace/state.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -793,7 +793,8 @@ export async function resolveTurnBinding(
793793
waitMs = TURN_PIN_WAIT_MS,
794794
): Promise<TurnBinding | null> {
795795
// With the workspace kill switch on, nothing else in the session reads the pin, so the turn sends
796-
// no validation request and waits for none. Its rows read the link, as they do on `main`.
796+
// no validation request and waits for none. Its rows read the link, as they did before the pin
797+
// was honoured.
797798
if (!workspaceEnabled()) return linkedTurnBinding(directory)
798799
let timer: ReturnType<typeof setTimeout> | undefined
799800
const deadline = new Promise<BindingOutcome>((done) => {

‎packages/opencode/src/session/prompt.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,13 @@ import { Telemetry } from "@/telemetry" // altimate_change — session telemetry
125125
import { ErrorDetail } from "@/altimate/telemetry/error-detail" // altimate_change — error_fingerprint payload
126126
// altimate_change start — the session's bound workspace rides on every telemetry row; workspace rules canonicalize bound
127127
// directories like the bindings cache does
128-
import { canonicalDirectory, endTurnBinding, turnBinding, unvalidatedTurnBinding } from "@/altimate/workspace/state"
128+
import {
129+
canonicalDirectory,
130+
endTurnBinding,
131+
turnBinding,
132+
unvalidatedTurnBinding,
133+
type AccountKey,
134+
} from "@/altimate/workspace/state"
129135

130136
/** The session's workspace use, as session_end fields. Read only: the row is tracked first, then the
131137
* turn ends its telemetry entries (`Telemetry.endSession`), so the row itself still carries the
@@ -1850,6 +1856,7 @@ export namespace SessionPrompt {
18501856
// altimate_change start — file rules append only to this tool result
18511857
lessons,
18521858
workspaceId: boundWorkspace?.datamateId,
1859+
workspaceScope: boundWorkspace?.scope,
18531860
turn: telemetryTurn,
18541861
// altimate_change end
18551862
}),
@@ -2883,6 +2890,8 @@ export namespace SessionPrompt {
28832890
/** The workspace the calling turn resolved, for the rows its MCP calls emit, and the turn itself,
28842891
* so a call landing after the turn was replaced does not count toward the replacement. */
28852892
workspaceId?: number
2893+
/** The credential that turn's workspace was resolved under, noted for each tool call's plugins. */
2894+
workspaceScope?: AccountKey
28862895
turn?: number
28872896
// altimate_change end
28882897
}) {
@@ -3008,6 +3017,8 @@ export namespace SessionPrompt {
30083017
// altimate_change start — stamp authoritative tool source so clients render the right
30093018
// badge. Shared with SessionTools.resolve (session/tools.ts) so the resolvers can't drift.
30103019
const stamped = stampRegistryToolSource(output, item)
3020+
// The turn's credential, for this call's plugins: a first-event mark names its tenant.
3021+
using _callScope = WorkspaceFirstEvents.noteCallScope(ctx.callID, input.workspaceScope)
30113022
// altimate_change end
30123023
await Plugin.trigger(
30133024
"tool.execute.after",
@@ -3233,6 +3244,14 @@ export namespace SessionPrompt {
32333244
? await unvalidatedTurnBinding(Instance.directory)
32343245
: await turnBinding(input.sessionID, Instance.directory)
32353246
)?.datamateId
3247+
// Every part settles before a failed one is rethrown. A part still running after another failed could
3248+
// otherwise read the turn's workspace once `prompt` has cleared it, and leave a verdict for the next
3249+
// loop started without a prompt (summarize, resume) to take.
3250+
function settled<T>(results: PromiseSettledResult<T>[]): T[] {
3251+
const failed = results.find((result): result is PromiseRejectedResult => result.status === "rejected")
3252+
if (failed) throw failed.reason
3253+
return results.map((result) => (result as PromiseFulfilledResult<T>).value)
3254+
}
32363255
// altimate_change end
32373256
const agentName = input.agent ?? (await Agent.defaultAgent())
32383257
const agent = await Agent.get(agentName)
@@ -3288,7 +3307,9 @@ export namespace SessionPrompt {
32883307
id: part.id ? PartID.make(part.id) : PartID.ascending(),
32893308
})
32903309

3291-
const parts = await Promise.all(
3310+
// altimate_change start — every part settles before a failure is rethrown (see `settled`)
3311+
const parts = await Promise.allSettled(
3312+
// altimate_change end
32923313
input.parts.map(async (part): Promise<Draft<MessageV2.Part>[]> => {
32933314
if (part.type === "file") {
32943315
// before checking the protocol we check if this is an mcp resource because it needs special handling
@@ -3618,7 +3639,9 @@ export namespace SessionPrompt {
36183639
},
36193640
]
36203641
}),
3621-
).then((x) => x.flat().map(assign))
3642+
// altimate_change start — every part settles before a failure is rethrown (see `settled`)
3643+
).then((x) => settled(x).flat().map(assign))
3644+
// altimate_change end
36223645

36233646
await Plugin.trigger(
36243647
"chat.message",

‎packages/opencode/test/altimate/telemetry/onboarding.test.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,14 @@ async function runTool(hooks: any, input: any, output: any) {
123123

124124
describe("activation events", () => {
125125
const SESSION = "ses_test"
126+
// The suite's preload sets the workspace kill switch. A test that needs workspaces on, as a user's
127+
// CLI has them, clears it, and it is put back after each test.
128+
const KILL_SWITCH = process.env.ALTIMATE_DISABLE_WORKSPACE
129+
const workspacesOn = () => delete process.env.ALTIMATE_DISABLE_WORKSPACE
130+
afterEach(() => {
131+
if (KILL_SWITCH === undefined) delete process.env.ALTIMATE_DISABLE_WORKSPACE
132+
else process.env.ALTIMATE_DISABLE_WORKSPACE = KILL_SWITCH
133+
})
126134

127135
async function plugin() {
128136
return OnboardingTelemetryPlugin({} as any)
@@ -221,6 +229,7 @@ describe("activation events", () => {
221229
})
222230

223231
test("the first job's row names the workspace the hook input carries", async () => {
232+
workspacesOn()
224233
const events = captureEvents()
225234
const hooks = await plugin()
226235
await startOnboarding(hooks, "skip")
@@ -240,6 +249,7 @@ describe("activation events", () => {
240249
})
241250

242251
test("the first job's row marks its workspace, so the workspace's own row isn't sent after it", async () => {
252+
workspacesOn()
243253
await withTestStateHome(async () => {
244254
spyOn(Telemetry, "isEnabled").mockImplementation(() => true)
245255
spyOn(AltimateApi, "isConfigured").mockImplementation(async () => true)
@@ -264,6 +274,70 @@ describe("activation events", () => {
264274
})
265275
})
266276

277+
test("the first job's mark is the tool's own turn's tenant, when the credential switched before a late return", async () => {
278+
workspacesOn()
279+
await withTestStateHome(async () => {
280+
spyOn(Telemetry, "isEnabled").mockImplementation(() => true)
281+
spyOn(AltimateApi, "isConfigured").mockImplementation(async () => true)
282+
// By the time the tool returns, the user has signed in to another tenant.
283+
spyOn(AltimateApi, "getCredentials").mockImplementation(
284+
async () => ({ altimateInstanceName: "other", altimateUrl: "https://api.test", altimateApiKey: "k2" }) as never,
285+
)
286+
WorkspaceFirstEvents.resetForTest()
287+
captureEvents()
288+
const hooks = await plugin()
289+
await startOnboarding(hooks, "skip")
290+
const acme = { tenant: "acme", apiUrl: "https://api.test" }
291+
const other = { tenant: "other", apiUrl: "https://api.test" }
292+
293+
// The cancelled turn's call, and its replacement's, both still running: each trigger site notes
294+
// its own turn's credential.
295+
using _late = WorkspaceFirstEvents.noteCallScope("c3late", acme)
296+
using _replacement = WorkspaceFirstEvents.noteCallScope("c9new", other)
297+
await runTool(
298+
hooks,
299+
{ tool: "sample_setup", sessionID: SESSION, callID: "c3late", args: {}, workspaceId: 7 },
300+
{ title: "", output: "", metadata: { success: true } },
301+
)
302+
await settle()
303+
304+
expect(await WorkspaceFirstEvents.claim("first_job_completed", 7, acme)).toBe(false)
305+
// The other tenant's workspace 7 is a different workspace: its own first row is still sent.
306+
expect(await WorkspaceFirstEvents.claim("first_job_completed", 7, other)).toBe(true)
307+
WorkspaceFirstEvents.resetForTest()
308+
})
309+
})
310+
311+
test("with the workspace kill switch set, the first job's row names no workspace and marks none", async () => {
312+
await withTestStateHome(async () => {
313+
process.env.ALTIMATE_DISABLE_WORKSPACE = "1"
314+
spyOn(Telemetry, "isEnabled").mockImplementation(() => true)
315+
spyOn(AltimateApi, "isConfigured").mockImplementation(async () => true)
316+
spyOn(AltimateApi, "getCredentials").mockImplementation(
317+
async () => ({ altimateInstanceName: "acme", altimateUrl: "https://api.test", altimateApiKey: "k" }) as never,
318+
)
319+
WorkspaceFirstEvents.resetForTest()
320+
const events = captureEvents()
321+
const hooks = await plugin()
322+
await startOnboarding(hooks, "skip")
323+
324+
await runTool(
325+
hooks,
326+
{ tool: "sample_setup", sessionID: SESSION, callID: "c3k", args: {}, workspaceId: 7 },
327+
{ title: "", output: "", metadata: { success: true } },
328+
)
329+
await settle()
330+
331+
const completed = events.find((e) => e.type === "first_job_completed") as any
332+
expect(completed.origin).toBe("onboarding")
333+
expect(completed.workspace_id).toBeUndefined()
334+
// Nothing was marked: with the switch off, the workspace's own first job is still sent.
335+
workspacesOn()
336+
expect(await WorkspaceFirstEvents.claim("first_job_completed", 7)).toBe(true)
337+
WorkspaceFirstEvents.resetForTest()
338+
})
339+
})
340+
267341
test("a failed sample setup counts as selected but not completed", async () => {
268342
const events = captureEvents()
269343
const hooks = await plugin()

0 commit comments

Comments
 (0)