Visitar URL original
fix(core): stop warming a session after a ping misses the cache by rekram1-node · Pull Request #62926 · anomalyco/opencode · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 32 additions & 7 deletions packages/core/src/plugin/warming.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ const Decision = Data.taggedEnum<Decision>()
// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last.
const fallbackInterval = Duration.minutes(4)

/** Whether a session's warming is sending pings. */
type Status = Data.TaggedEnum<{
Active: {}
/** A ping missed the cache, so pinging waits for the session's next non-warming request. */
StoppedAfterMiss: {}
}>
const Status = Data.taggedEnum<Status>()

export const Plugin = define({
id: "opencode.warming",
effect: Effect.fn(function* (ctx) {
Expand Down Expand Up @@ -77,7 +85,7 @@ export const Plugin = define({
})

const scope = yield* Scope.Scope
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings }>()
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings; status: Status }>()
// Sessions already told their cache outlives the window; the check runs on every request.
const outlived = new Set<Session.ID>()
// A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models.
Expand Down Expand Up @@ -126,11 +134,27 @@ export const Plugin = define({

const last = current.last
yield* Effect.logInfo("warming session", { sessionID, last })
yield* ctx.session
const result = yield* ctx.session
.generate({ sessionID, prompt: current.settings.prompt })
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
.pipe(
Effect.catchCause((cause) =>
Effect.logWarning("failed to warm session", { sessionID, cause }).pipe(Effect.as(undefined)),
),
)
const latest = sessions.get(sessionID)
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
if (latest !== current || latest.last !== last) return yield* loop(sessionID)
// A ping that wrote more to the cache than it read rebuilt the cache instead of keeping it, so later pings
// would keep paying for that. Providers that report no cache usage never count as a miss.
if (result && result.tokens.cache.read < result.tokens.cache.write) {
latest.status = Status.StoppedAfterMiss()
yield* Effect.logInfo("stopped warming session after a cache miss", {
sessionID,
read: result.tokens.cache.read,
write: result.tokens.cache.write,
})
return
}
latest.last = yield* Clock.currentTimeMillis
return yield* loop(sessionID)
})

Expand Down Expand Up @@ -180,14 +204,15 @@ export const Plugin = define({

const now = yield* Clock.currentTimeMillis
const duration = Duration.toMillis(settings.duration)
if (active) {
if (active && Status.$is("Active")(active.status)) {
active.last = now
active.expires = now + duration
active.settings = settings
return
}
sessions.set(event.sessionID, { last: now, expires: now + duration, settings })
yield* Effect.logInfo("scheduled session warming", {
// A session stopped after a miss has no running loop, so its next real request starts one.
sessions.set(event.sessionID, { last: now, expires: now + duration, settings, status: Status.Active() })
yield* Effect.logInfo(active ? "resumed session warming" : "scheduled session warming", {
sessionID: event.sessionID,
interval: settings.interval,
expires: now + duration,
Expand Down
62 changes: 55 additions & 7 deletions packages/core/test/plugin/warming.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,19 @@ const context: SessionContext = {
options: {},
}

/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */
const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) {
const hit = { read: 1000, write: 10 }
const miss = { read: 0, write: 1000 }
const unreported = { read: 0, write: 0 }

/**
* Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. Each ping
* reports the next cache usage in `usage`.
*/
const run = Effect.fnUntraced(function* (
warming: unknown,
cache?: Provider.Cache,
usage: ReadonlyArray<typeof hit> = [],
) {
const warms: number[] = []
const events = yield* Queue.unbounded<SessionEvent.Execution.Succeeded>()
let contextHook: ((input: SessionContext) => Effect.Effect<void>) | undefined
Expand All @@ -60,12 +71,9 @@ const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cach
generate: () =>
Clock.currentTimeMillis.pipe(
Effect.map((now) => {
const cache = usage[warms.length] ?? unreported
warms.push(now / 60_000)
return {
text: "OK",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
cost: Money.USD.zero,
}
return { text: "OK", tokens: { input: 0, output: 1, reasoning: 0, cache }, cost: Money.USD.zero }
}),
),
hook: (name, callback) => {
Expand Down Expand Up @@ -134,3 +142,43 @@ describe("WarmingPlugin interval", () => {
}),
)
})

describe("WarmingPlugin cache miss", () => {
it.effect("keeps warming after a ping hits the cache", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [hit, hit, hit])
yield* warming.turn
for (const minutes of [4, 8, 12]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8, 12])
}),
)

it.effect("stops warming after a ping misses the cache", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [hit, miss, hit])
yield* warming.turn
for (const minutes of [4, 8, 12, 16]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8])
}),
)

it.effect("resumes warming on the next real request after a miss", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [miss, hit, hit])
yield* warming.turn
for (const minutes of [4, 8]) yield* warming.at(minutes)
yield* warming.turn
for (const minutes of [12, 16]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 12, 16])
}),
)

it.effect("doesn't count a ping without cache usage as a miss", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [unreported, unreported])
yield* warming.turn
for (const minutes of [4, 8]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8])
}),
)
})
10 changes: 10 additions & 0 deletions services/site/src/docs/content/warming.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,16 @@ window continues to count down.
00:08 next warming attempt
```

A warming request that misses the cache, reading less from it than it writes,
stops warming for that session until its next non-warming request. Providers
that don't report cache usage never count as a miss.

```text
00:04 warming request misses the cache; warming stops
00:10 normal model request; warming resumes
00:14 warming request
```

## Costs

Warming requests are real provider requests. They can consume tokens, incur
Expand Down
Loading