Visitar URL original
feat(core): run shell commands in the process daemon across handoffs by jlongster · Pull Request #62978 · anomalyco/opencode · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions packages/cli/src/commands/handlers/service/restart.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,12 @@ import { Service } from "@opencode/client/effect/service"
import { Commands } from "../../commands"
import { Runtime } from "../../../framework/runtime"
import { ServiceConfig } from "../../../services/service-config"
import { ServerConnection } from "../../../services/server-connection"

export default Runtime.handler(
Commands.commands.service.commands.restart,
Effect.fn("cli.service.restart")(function* () {
const options = yield* ServiceConfig.options()
// Keep this explicit: automatic service replacement must preserve terminals.
yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore)
// No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes.
yield* Service.stop(options)
const transport = yield* Service.ensure(options)
process.stdout.write(transport.url + EOL)
Expand Down
3 changes: 1 addition & 2 deletions packages/cli/src/commands/handlers/service/stop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,12 @@ import { Service } from "@opencode/client/effect/service"
import { Commands } from "../../commands"
import { Runtime } from "../../../framework/runtime"
import { ServiceConfig } from "../../../services/service-config"
import { ServerConnection } from "../../../services/server-connection"

export default Runtime.handler(
Commands.commands.service.commands.stop,
Effect.fn("cli.service.stop")(function* () {
const options = yield* ServiceConfig.options()
yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore)
// No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes.
yield* Service.stop(options)
}),
)
16 changes: 15 additions & 1 deletion packages/core/src/job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ type Active = {
isBackgrounded: boolean
consumed: boolean
recovery?: Recovery
cancel?: Effect.Effect<void>
}

type State = {
Expand All @@ -59,6 +60,7 @@ type State = {
type FinishResult = {
info?: Info
done?: Deferred.Deferred<Info>
cancel?: Effect.Effect<void>
scope?: Scope.Closeable
generation?: Scope.Closeable
}
Expand Down Expand Up @@ -90,6 +92,9 @@ export type StartInput = {
recovery?: Recovery
notificationID?: SessionMessage.ID
run: Effect.Effect<string, unknown>
// Runs when the job is cancelled (`cancel`, `cancelSession`), before its work is interrupted. Teardown, such as
// a server shutdown, interrupts the work without it.
cancel?: Effect.Effect<void>
}

export type WaitInput = {
Expand Down Expand Up @@ -248,6 +253,11 @@ export const make = Effect.gen(function* () {
if (!job) return [{}, jobs]
if (job.scope !== scope) return [{}, jobs]
if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs]
// Teardown (this registry closing) is not a terminal outcome: leave the job and its durable row running for
// the next server. The scope's own finalizer interrupts its observers. A cancel has already settled the job
// as cancelled before interrupting its work.
if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && scope.state._tag === "Closed")
return [{}, jobs]
const status: Exclude<Status, "running"> = Exit.isSuccess(exit)
? "completed"
: Cause.hasInterruptsOnly(exit.cause)
Expand Down Expand Up @@ -295,6 +305,8 @@ export const make = Effect.gen(function* () {
return [{ info: snapshot(existing) }, jobs]
}
const scope = yield* Scope.fork(state.scope, "parallel")
// Closing the registry interrupts the work without settling it; interrupt its observers too.
yield* Scope.addFinalizer(scope, Deferred.interrupt(done))
const job = {
info: {
id: input.id,
Expand All @@ -312,6 +324,7 @@ export const make = Effect.gen(function* () {
isBackgrounded: false,
consumed: false,
recovery: input.recovery,
cancel: input.cancel,
}
return [{ info: snapshot(job), scope }, new Map(jobs).set(input.id, job)]
}),
Expand Down Expand Up @@ -459,11 +472,12 @@ export const make = Effect.gen(function* () {
}
if (persist) yield* persistBackground(next)
return [
{ info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope },
{ info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope, cancel: job.cancel },
new Map(jobs).set(id, next),
]
}),
)
if (result.cancel) yield* result.cancel
if (result.info && result.done) yield* Deferred.succeed(result.done, result.info)
if (result.scope) yield* Scope.close(result.scope, Exit.void)
if (result.generation) yield* consume(id, result.generation)
Expand Down
16 changes: 15 additions & 1 deletion packages/core/src/persistent-pty/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,11 @@ export interface Interface {
readonly remove: (id: Pty.ID) => Effect.Effect<void, NotFoundError | UnavailableError>
readonly shutdown: () => Effect.Effect<void, UnavailableError>
readonly handoff: () => Effect.Effect<Handoff | null, UnavailableError>
/**
* Whether a handoff to the next server is prepared and its ticket still valid: this server is being replaced, so
* the daemon's processes should be left running for the next one. A shutdown clears it.
*/
readonly handoffPrepared: () => boolean
readonly attach: (id: Pty.ID, input: SubscribeInput) => Effect.Effect<Attachment, NotFoundError | UnavailableError>
/** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */
readonly createPipe: (input: CreatePipeInput) => Effect.Effect<PipeInfo, UnavailableError | RequestError>
Expand Down Expand Up @@ -361,14 +366,22 @@ const makeLayer = (options: Options = {}) =>
return undefined
})

// When the prepared handoff's ticket expires; 0 when none is prepared.
let handoffExpiresAt = 0

const shutdown = Effect.fn("PersistentPty.shutdown")(function* () {
handoffExpiresAt = 0
const response = yield* daemon.shutdown.pipe(Effect.mapError(unavailable))
current.clear()
if (!response) return
if (response.type !== "ok") return yield* unexpected(response)
})

const handoff = Effect.fn("PersistentPty.handoff")(() => daemon.handoff.pipe(Effect.mapError(unavailable)))
const handoff = Effect.fn("PersistentPty.handoff")(function* () {
const prepared = yield* daemon.handoff.pipe(Effect.mapError(unavailable))
if (prepared) handoffExpiresAt = prepared.expiresAt
return prepared
})

const removeVisibleExit = (id: Pty.ID) => {
if (removing.has(id)) return
Expand Down Expand Up @@ -512,6 +525,7 @@ const makeLayer = (options: Options = {}) =>
remove,
shutdown,
handoff,
handoffPrepared: () => Date.now() < handoffExpiresAt,
attach,
createPipe,
waitPipe,
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/plugin/internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ export const requirements = LayerNode.group([
InstructionDiscovery.node,
Integration.node,
Job.node,
Shell.daemonNode,
KV.node,
llmClient,
Location.node,
Expand Down
154 changes: 144 additions & 10 deletions packages/core/src/session/execution/restart.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
export * as SessionRestart from "./restart.js"

import { Context, Effect, Layer } from "effect"
import { Context, Effect, Layer, Option, Schema } from "effect"
import { isReadonlyArrayNonEmpty } from "effect/Array"
import { makeGlobalNode } from "@opencode/util/effect/app-node"
import { Shell as ShellSchema } from "@opencode/schema/shell"
import { Bus } from "../../bus.js"
import { Database } from "../../database/database.js"
import { Instance } from "../../instance/service.js"
import { Job } from "../../job.js"
import { Session } from "../../session.js"
import { Shell } from "../../shell.js"
import { ShellTool } from "../../tool/plugin/shell.js"
import { SessionEvent } from "../event.js"
import { SessionExecution } from "../execution.js"
import { SessionHistory } from "../history.js"
import { SessionMessage } from "../message.js"
import { SessionSchema } from "../schema.js"
import { SessionStore } from "../store.js"
import { ImageNotice } from "../../image/notice.js"
Expand Down Expand Up @@ -72,6 +80,8 @@ export const layer = (options?: Options) =>
const bus = yield* Bus.Service
const jobs = yield* Job.Service
const sessions = yield* Session.Service
const instances = yield* Instance.Service
const db = (yield* Database.Service).db
const scope = yield* Effect.scope
const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS

Expand Down Expand Up @@ -99,10 +109,7 @@ export const layer = (options?: Options) =>
return true
})

const restartMessage = (
background: Job.Background,
recovery: Exclude<Job.Recovery, { kind: "subagent" }>,
) => {
const restartMessage = (background: Job.Background, recovery: Exclude<Job.Recovery, { kind: "subagent" }>) => {
const state = background.status === "running" ? "cancelled" : background.status
if (recovery.kind === "shell") {
const text =
Expand Down Expand Up @@ -164,6 +171,100 @@ export const layer = (options?: Options) =>
yield* jobs.completeBackground(background.notificationID)
})

// The Shell of a session's Location, booted so it takes back the commands the daemon still runs for it.
const shellOf = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) {
const session = yield* store.get(sessionID)
if (!session) return undefined
return yield* Effect.serviceOption(Shell.Service).pipe(
instances.provide(session),
Effect.map(Option.getOrUndefined),
// The session's directory is gone: its commands are reported like any lost background work.
Effect.catchTag("FileSystem.DirectoryNotFoundError", () => Effect.succeed(undefined)),
)
})

const reattach = (
shell: Shell.Interface,
sessionID: SessionSchema.ID,
info: ShellSchema.Info,
suspended: ReadonlySet<SessionSchema.ID>,
notificationID?: SessionMessage.ID,
) =>
ShellTool.reattach({
shell,
jobs,
sessions,
sessionID,
info,
notificationID,
resume: () => (suspended.has(sessionID) ? false : undefined),
})

// A shell job the previous server left running is watched again if its command is still known (running or
// finished during the restart); otherwise it is reported as lost.
const recoverShellJob = Effect.fnUntraced(function* (
background: Job.Background,
recovery: Extract<Job.Recovery, { kind: "shell" }>,
suspended: ReadonlySet<SessionSchema.ID>,
reattached: Set<string>,
) {
const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined
const info =
shell && Schema.is(ShellSchema.ID)(recovery.shellID)
? yield* shell.get(recovery.shellID).pipe(Effect.option)
: Option.none()
if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery)
reattached.add(info.value.id)
yield* reattach(shell, recovery.sessionID, info.value, suspended, background.notificationID)
})

// A server that stopped (shut down or killed) during a foreground shell call leaves the call unsettled, its
// command still running in the daemon. Settle the call as moved to the background and watch the command.
const settleBackgroundedCalls = Effect.fnUntraced(function* (
sessionID: SessionSchema.ID,
suspended: ReadonlySet<SessionSchema.ID>,
reattached: ReadonlySet<string>,
) {
const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) =>
message.type === "assistant"
? message.content.flatMap((part) =>
part.type === "tool" && part.name === ShellTool.name && part.state.status === "running"
? [{ assistantMessageID: message.id, part }]
: [],
)
: [],
)
if (calls.length === 0) return
const shell = yield* shellOf(sessionID)
if (!shell) return
yield* Effect.forEach(
calls,
Effect.fnUntraced(function* (call) {
const info = yield* shell.byToolCall({
sessionID,
assistantMessageID: call.assistantMessageID,
callID: call.part.id,
})
if (!info) return
const result = ShellTool.backgroundToolResult(info)
if (!isReadonlyArrayNonEmpty(result.content)) return
yield* bus.publish(SessionEvent.Tool.Success, {
sessionID,
assistantMessageID: call.assistantMessageID,
id: call.part.id,
content: result.content,
metadata: result.metadata,
executed: call.part.executed === true,
})
// Backgrounded but stopped before the call was settled: its job row was already reattached (and, for a
// command that already finished, may even be done).
if (reattached.has(info.id)) return
yield* reattach(shell, sessionID, info, suspended)
}),
{ discard: true },
)
})

const recoverSubagent = Effect.fnUntraced(function* (
background: Job.Background,
recovery: Extract<Job.Recovery, { kind: "subagent" }>,
Expand Down Expand Up @@ -232,13 +333,37 @@ export const layer = (options?: Options) =>
[...(yield* store.listSuspended()), ...children].filter((sessionID) => !active.has(sessionID)),
)
yield* store.releaseChildClaims(children)
const recoverable = Effect.fnUntraced(function* (background: Job.Background) {
return (yield* jobs.get(background.id))?.status !== "running"
})
// Shell and tool outcomes first, then calls a killed server left unsettled, and only then recovered children,
// whose first resume would otherwise fail their own unsettled shell calls.
const reattached = new Set<string>()
yield* Effect.forEach(
pending,
Effect.fnUntraced(function* (background) {
const recovery = background.recovery
if (recovery.kind === "subagent" || !(yield* recoverable(background))) return
if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended, reattached)
yield* recoverToolJob(background, recovery)
}),
{ discard: true },
)
yield* Effect.forEach(
suspended,
(sessionID) =>
settleBackgroundedCalls(sessionID, suspended, reattached).pipe(
Effect.catch((error) =>
Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }),
),
),
{ discard: true },
)
yield* Effect.forEach(
// Admit shell outcomes before a recovered child can start its first model request.
pending.toSorted((a, b) => Number(a.recovery.kind === "subagent") - Number(b.recovery.kind === "subagent")),
pending,
Effect.fnUntraced(function* (background) {
if ((yield* jobs.get(background.id))?.status === "running") return
const recovery = background.recovery
if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery)
if (recovery.kind !== "subagent" || !(yield* recoverable(background))) return
yield* recoverSubagent(background, recovery, suspended)
}),
{ discard: true },
Expand All @@ -264,5 +389,14 @@ export const layer = (options?: Options) =>
export const node = makeGlobalNode({
service: Service,
layer: layer(),
deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node],
deps: [
SessionStore.node,
SessionExecution.node,
Bus.node,
Job.node,
Session.node,
Instance.node,
Database.node,
Shell.daemonNode,
],
})
4 changes: 4 additions & 0 deletions packages/core/src/session/runner/llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { llmClient } from "../../effect/app-node-platform.js"
import { StepFailedError } from "../error.js"
import { SessionRunnerRetry } from "./retry.js"
import { SessionStep } from "./step.js"
import { PersistentPty } from "../../persistent-pty/index.js"
import { ToolOutput } from "../../tool-output.js"
import { Plugin } from "../../plugin.js"
import { PluginHooks } from "../../plugin/hooks.js"
Expand All @@ -54,6 +55,7 @@ const layer = Layer.effect(
const jobs = yield* Job.Service
const title = yield* SessionTitle.Service
const steps = yield* SessionStep.make
const pty = yield* PersistentPty.Service
// Title generation starts once input is visible and must not delay model execution.
const titles = yield* FiberMap.make<SessionSchema.ID, void, never>()

Expand Down Expand Up @@ -293,6 +295,7 @@ const layer = Layer.effect(
const outcome = yield* steps.attempt({
isLocationClosed: lifecycle.isClosed,
interruption,
handoffPrepared: pty.handoffPrepared,
sessionID,
assistantMessageID,
agent: loaded.agent.id,
Expand Down Expand Up @@ -460,6 +463,7 @@ export const node = makeLocationNode({
Job.node,
SessionTitle.node,
Snapshot.node,
PersistentPty.node,
ToolOutput.node,
Database.node,
],
Expand Down
Loading
Loading