Visitar URL original
feat(run-engine): release a dequeued run that never started · triggerdotdev/trigger.dev@ba51e55 · GitHub
Skip to content

Commit ba51e55

Browse files
myftijaTrigger.dev RepoOps
authored andcommitted
feat(run-engine): release a dequeued run that never started
Adds a worker action that lets a worker give back a run it dequeued but never started, so the run is requeued at once instead of waiting out the pending execution timeout. Nothing changes until a worker calls it. The supervisor's use of it ships separately, is off by default, and is turned on with `RELEASE_UNSTARTED_RUNS_ENABLED=true`. Mono-RevId: 446a8876ec8b09e0d4be87de6292ba1efd3ed65a
1 parent aa9de9d commit ba51e55

13 files changed

Lines changed: 1112 additions & 34 deletions
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@trigger.dev/core": patch
3+
---
4+
5+
Add `SupervisorHttpClient.releaseRun`, which gives back a dequeued run the worker never started so the platform requeues it at once instead of after the pending execution timeout.
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import { startTestServer, type TestServer } from "@internal/testcontainers/webapp";
2+
import { ApiClient } from "@trigger.dev/core/v3";
3+
import { SupervisorHttpClient } from "@trigger.dev/core/v3/runEngineWorker";
4+
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
5+
import { MANAGED_WORKER_SECRET, seedEngineFixtures } from "../../test/bench/lib/engineFixtures";
6+
7+
const RUNNER_ID = "runner-release-e2e";
8+
9+
let server: TestServer;
10+
11+
beforeAll(async () => {
12+
server = await startTestServer({
13+
// Only a release can requeue the run within these tests.
14+
extraEnv: { RUN_ENGINE_TIMEOUT_PENDING_EXECUTING: "600000" },
15+
// The engine worker moves runs from the master queue to the worker queue.
16+
overrideEnv: { RUN_ENGINE_WORKER_ENABLED: "1" },
17+
});
18+
}, 180_000);
19+
20+
afterAll(async () => {
21+
await server?.stop();
22+
}, 120_000);
23+
24+
describe("release a dequeued run that never started", () => {
25+
it("requeues the run without waiting for the stall timeout and gives back its slot", async () => {
26+
const { api, supervisor, environmentId, taskIdentifier } = await setup("release-requeue");
27+
const { id: runId } = await api.triggerTask(taskIdentifier, { payload: {} });
28+
const dequeued = await dequeue(supervisor);
29+
30+
const released = await supervisor.releaseRun(
31+
runId,
32+
dequeued.snapshot.friendlyId,
33+
{ reason: "PodFailed" },
34+
RUNNER_ID,
35+
environmentId
36+
);
37+
expect(released).toEqual({ success: true, data: { ok: true, requeued: true } });
38+
39+
const { execution } = ok(await supervisor.getLatestSnapshot(runId));
40+
expect(execution.snapshot.executionStatus).toBe("QUEUED");
41+
expect(execution.snapshot.description).toBe("Released before the run started: PodFailed");
42+
43+
// The queue's limit is 1, so the run is dequeued again only if its slot was given back.
44+
const redelivered = await dequeue(supervisor);
45+
expect(redelivered.run.friendlyId).toBe(runId);
46+
expect(redelivered.snapshot.executionStatus).toBe("PENDING_EXECUTING");
47+
ok(await supervisor.startRunAttempt(runId, redelivered.snapshot.friendlyId, {}));
48+
}, 60_000);
49+
50+
it("refuses a run that has started and says not to retry", async () => {
51+
const { api, supervisor, environmentId, taskIdentifier } = await setup("release-started");
52+
const { id: runId } = await api.triggerTask(taskIdentifier, { payload: {} });
53+
const dequeued = await dequeue(supervisor);
54+
ok(await supervisor.startRunAttempt(runId, dequeued.snapshot.friendlyId, {}));
55+
56+
const released = await supervisor.releaseRun(
57+
runId,
58+
dequeued.snapshot.friendlyId,
59+
{ reason: "PodFailed" },
60+
RUNNER_ID,
61+
environmentId
62+
);
63+
expect(released).toMatchObject({
64+
success: false,
65+
statusCode: 409,
66+
shouldRetry: false,
67+
error: "The run is not at this dequeued snapshot, or this worker group does not serve it",
68+
});
69+
70+
const { execution } = ok(await supervisor.getLatestSnapshot(runId));
71+
expect(execution.snapshot.executionStatus).toBe("EXECUTING");
72+
}, 60_000);
73+
74+
it("answers a run outside the caller's environment with a 409, never a 404", async () => {
75+
const { api, supervisor, taskIdentifier } = await setup("release-other-env");
76+
const { id: runId } = await api.triggerTask(taskIdentifier, { payload: {} });
77+
const dequeued = await dequeue(supervisor);
78+
79+
const released = await supervisor.releaseRun(
80+
runId,
81+
dequeued.snapshot.friendlyId,
82+
{ reason: "PodFailed" },
83+
RUNNER_ID,
84+
"env_not_the_runs"
85+
);
86+
expect(released).toMatchObject({ success: false, statusCode: 409, shouldRetry: false });
87+
88+
const { execution } = ok(await supervisor.getLatestSnapshot(runId));
89+
expect(execution.snapshot.friendlyId).toBe(dequeued.snapshot.friendlyId);
90+
}, 60_000);
91+
92+
it("accepts a release from another worker instance of the same group that never dequeued", async () => {
93+
const { api, supervisor, environmentId, taskIdentifier, workerGroupToken } =
94+
await setup("release-dequeuer");
95+
const { id: runId } = await api.triggerTask(taskIdentifier, { payload: {} });
96+
const dequeued = await dequeue(supervisor);
97+
// A process that only watches failed pods: same group token, its own instance name, no dequeue.
98+
const watcher = new SupervisorHttpClient({
99+
apiUrl: server.webapp.baseUrl,
100+
workerToken: workerGroupToken,
101+
instanceName: "release-watcher",
102+
managedWorkerSecret: MANAGED_WORKER_SECRET,
103+
});
104+
105+
const released = await watcher.releaseRun(
106+
runId,
107+
dequeued.snapshot.friendlyId,
108+
{ reason: "PodFailed" },
109+
"runner-from-the-watcher",
110+
environmentId
111+
);
112+
expect(released).toEqual({ success: true, data: { ok: true, requeued: true } });
113+
114+
const { execution } = ok(await supervisor.getLatestSnapshot(runId));
115+
expect(execution.snapshot.executionStatus).toBe("QUEUED");
116+
}, 60_000);
117+
118+
it("answers a run that does not exist with a 409, even without the environment header", async () => {
119+
const { supervisor } = await setup("release-missing-run");
120+
121+
const released = await supervisor.releaseRun(
122+
"run_doesnotexist0000000000",
123+
"snapshot_doesnotexist00000",
124+
{ reason: "PodFailed" },
125+
RUNNER_ID
126+
);
127+
expect(released).toMatchObject({ success: false, statusCode: 409, shouldRetry: false });
128+
}, 60_000);
129+
});
130+
131+
async function setup(instanceName: string) {
132+
const fixtures = await seedEngineFixtures(server.prisma, { taskCount: 1, concurrencyLimit: 1 });
133+
return {
134+
api: new ApiClient(server.webapp.baseUrl, fixtures.environmentApiKey),
135+
supervisor: new SupervisorHttpClient({
136+
apiUrl: server.webapp.baseUrl,
137+
workerToken: fixtures.workerGroupToken,
138+
instanceName,
139+
managedWorkerSecret: MANAGED_WORKER_SECRET,
140+
}),
141+
environmentId: fixtures.environmentId,
142+
taskIdentifier: fixtures.taskIdentifiers[0]!,
143+
workerGroupToken: fixtures.workerGroupToken,
144+
};
145+
}
146+
147+
function dequeue(supervisor: SupervisorHttpClient) {
148+
return vi.waitFor(
149+
async () => {
150+
const [message] = ok(await supervisor.dequeue({}));
151+
if (!message) throw new Error("Nothing dequeued yet");
152+
return message;
153+
},
154+
{ timeout: 10_000, interval: 100 }
155+
);
156+
}
157+
158+
function ok<T>(result: { success: true; data: T } | { success: false; error: string }): T {
159+
if (!result.success) throw new Error(result.error);
160+
return result.data;
161+
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import type { TypedResponse } from "@remix-run/server-runtime";
2+
import { json } from "@remix-run/server-runtime";
3+
import type { WorkerApiRunReleaseResponseBody } from "@trigger.dev/core/v3/workers";
4+
import { WorkerApiRunReleaseRequestBody } from "@trigger.dev/core/v3/workers";
5+
import {
6+
EngineServiceValidationError,
7+
LatestExecutionSnapshotNotFoundError,
8+
} from "@internal/run-engine";
9+
import { SnapshotStateExpiredError, UnknownShardKey } from "@internal/run-store";
10+
import { z } from "zod";
11+
import { logger } from "~/services/logger.server";
12+
import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server";
13+
14+
export const action = createActionWorkerApiRoute(
15+
{
16+
params: z.object({
17+
runFriendlyId: z.string(),
18+
snapshotFriendlyId: z.string(),
19+
}),
20+
body: z.compile(WorkerApiRunReleaseRequestBody),
21+
},
22+
async ({
23+
authenticatedWorker,
24+
params,
25+
body,
26+
runnerId,
27+
environmentId,
28+
}): Promise<TypedResponse<WorkerApiRunReleaseResponseBody>> => {
29+
const { runFriendlyId, snapshotFriendlyId } = params;
30+
31+
let result: Awaited<ReturnType<typeof authenticatedWorker.releaseRun>>;
32+
try {
33+
result = await authenticatedWorker.releaseRun({
34+
runFriendlyId,
35+
snapshotFriendlyId,
36+
reason: body.reason,
37+
runnerId,
38+
environmentId,
39+
});
40+
} catch (error) {
41+
if (error instanceof Response) {
42+
throw error;
43+
}
44+
const notFound =
45+
(error instanceof EngineServiceValidationError && error.status === 404) ||
46+
error instanceof LatestExecutionSnapshotNotFoundError ||
47+
error instanceof SnapshotStateExpiredError ||
48+
error instanceof UnknownShardKey;
49+
logger.warn(
50+
notFound ? "Release not applied, the run was not found" : "Failed to release run",
51+
{
52+
runFriendlyId,
53+
snapshotFriendlyId,
54+
environmentId,
55+
reason: body.reason,
56+
error,
57+
}
58+
);
59+
if (!notFound) {
60+
throw error;
61+
}
62+
// Not a 404, which clients read as a webapp without this route, nor a 500, which they retry.
63+
throw json(
64+
{ error: "Run not found" },
65+
{ status: 409, headers: { "x-should-retry": "false" } }
66+
);
67+
}
68+
69+
if (!result.ok) {
70+
logger.info("Release not applied, the run moved on or is not this worker group's", {
71+
runFriendlyId,
72+
snapshotFriendlyId,
73+
environmentId,
74+
reason: body.reason,
75+
latestExecutionStatus: result.latestExecutionStatus,
76+
});
77+
78+
// The client retries a 409 by default; a stale release will never apply, so say not to.
79+
throw json(
80+
{
81+
error: "The run is not at this dequeued snapshot, or this worker group does not serve it",
82+
},
83+
{ status: 409, headers: { "x-should-retry": "false" } }
84+
);
85+
}
86+
87+
return json({ ok: true, requeued: result.requeued });
88+
}
89+
);

‎apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,13 @@ if (workloadCreatedAtGateEnabled && !workloadTokenCutoff) {
7474
);
7575
}
7676

77-
type WorkloadGateAction = "start" | "complete" | "continue" | "snapshots_since" | "restore_outcome";
77+
type WorkloadGateAction =
78+
| "start"
79+
| "complete"
80+
| "continue"
81+
| "snapshots_since"
82+
| "restore_outcome"
83+
| "release";
7884

7985
const meter = getMeter("workload-auth-gate");
8086

@@ -799,6 +805,36 @@ export class AuthenticatedWorkerInstance extends WithRunEngine {
799805
});
800806
}
801807

808+
async releaseRun({
809+
runFriendlyId,
810+
snapshotFriendlyId,
811+
reason,
812+
runnerId,
813+
environmentId,
814+
}: {
815+
runFriendlyId: string;
816+
snapshotFriendlyId: string;
817+
reason: string;
818+
runnerId?: string;
819+
environmentId?: string;
820+
}) {
821+
await this.assertCreatedAtGate({
822+
runId: fromFriendlyId(runFriendlyId),
823+
environmentId,
824+
action: "release",
825+
});
826+
827+
return await this._engine.releaseRun({
828+
runId: fromFriendlyId(runFriendlyId),
829+
snapshotId: fromFriendlyId(snapshotFriendlyId),
830+
reason,
831+
workerId: this.workerInstanceId,
832+
runnerId,
833+
environmentId,
834+
servesWorkerQueue: (workerQueue) => workerServesQueue(this.queueConsumer, workerQueue),
835+
});
836+
}
837+
802838
async getSnapshotsSince({
803839
runFriendlyId,
804840
snapshotId,

‎internal-packages/run-engine/src/engine/index.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ import type {
105105
EngineStartRunAttemptResult,
106106
EngineWorker,
107107
HeartbeatTimeouts,
108+
ReleaseRunResult,
108109
ReportableQueue,
109110
RestoreOutcome,
110111
RestoreOutcomeResult,
@@ -2236,6 +2237,39 @@ export class RunEngine {
22362237
});
22372238
}
22382239

2240+
/**
2241+
* Requeues a run whose worker never started it, while `snapshotId` is still its dequeued
2242+
* PENDING_EXECUTING snapshot. Each release counts against the queue's dequeue budget and delays
2243+
* the run by the queue's backoff, as the pending execution timeout does.
2244+
*/
2245+
async releaseRun({
2246+
runId,
2247+
snapshotId,
2248+
reason,
2249+
workerId,
2250+
runnerId,
2251+
environmentId,
2252+
servesWorkerQueue,
2253+
}: {
2254+
runId: string;
2255+
snapshotId: string;
2256+
reason: string;
2257+
workerId?: string;
2258+
runnerId?: string;
2259+
environmentId?: string;
2260+
servesWorkerQueue?: (workerQueue: string) => boolean;
2261+
}): Promise<ReleaseRunResult> {
2262+
return this.runAttemptSystem.releaseRun({
2263+
runId,
2264+
snapshotId,
2265+
reason,
2266+
workerId,
2267+
runnerId,
2268+
environmentId,
2269+
servesWorkerQueue,
2270+
});
2271+
}
2272+
22392273
/**
22402274
Signals that the run still executes. Without heartbeats the run stalls and recovery starts.
22412275
@returns The ExecutionResult, which can name a different snapshot.

0 commit comments

Comments
 (0)