Visitar URL original
feat(webapp,cli): let self-hosted instances require deploy base images by brentshulman-silkline · Pull Request #5005 · triggerdotdev/trigger.dev · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/instance-deploy-base-images.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"@trigger.dev/core": patch
"trigger.dev": patch
---

Self-hosted instances can require custom deploy base images per runtime via the new `DEPLOY_BASE_IMAGES` and `DEPLOY_BUILD_BASE_IMAGES` webapp settings. The CLI builds on the images the instance specifies, and older CLIs are rejected with an upgrade message.
21 changes: 21 additions & 0 deletions apps/webapp/app/env.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
} from "@trigger.dev/core/v3/isomorphic";
import { BoolEnv } from "./utils/boolEnv";
import { isValidDatabaseUrl } from "./utils/db";
import { parseDeployBaseImages } from "~/v3/deployBaseImages.server";
import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server";
import { isValidRegex } from "./utils/regex";
import { isValidDuration } from "./services/realtime/duration.server";
Expand All @@ -16,6 +17,18 @@ function durationString() {
return z.string().refine(isValidDuration, "must be a duration like 7d, 30d, 365d, 1h, 1y");
}

const parseDeployBaseImagesEnv = (
value: string | undefined,
envVarName: string,
ctx: z.RefinementCtx
) => {
const { images, errors } = parseDeployBaseImages(value, envVarName);
for (const message of errors) {
ctx.addIssue({ code: z.ZodIssueCode.custom, message });
}
return errors.length > 0 ? z.NEVER : images;
};

const GithubAppEnvSchema = z.preprocess(
(val) => {
const obj = val as any;
Expand Down Expand Up @@ -856,6 +869,14 @@ const EnvironmentSchema = z
.transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY),

DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"),
DEPLOY_BASE_IMAGES: z
.string()
.optional()
.transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BASE_IMAGES", ctx)),
DEPLOY_BUILD_BASE_IMAGES: z
.string()
.optional()
.transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BUILD_BASE_IMAGES", ctx)),
DEPLOY_TIMEOUT_MS: z.coerce
.number()
.int()
Expand Down
6 changes: 6 additions & 0 deletions apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3";
import { z } from "zod";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
import { authenticateApiKeyWithScope } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";

Expand Down Expand Up @@ -65,6 +67,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
externalId: deployment.externalId ?? undefined,
externalBuildData:
deployment.externalBuildData as GetDeploymentResponseBody["externalBuildData"],
baseImages: resolveDeployBaseImages(deployment.runtime, {
base: env.DEPLOY_BASE_IMAGES,
buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
}),
errorData: deployment.errorData as GetDeploymentResponseBody["errorData"],
canceledReason: deployment.canceledReason,
worker: deployment.worker
Expand Down
6 changes: 6 additions & 0 deletions apps/webapp/app/routes/api.v1.deployments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { authenticateApiKeyWithScope } from "~/services/apiAuth.server";
import { logger } from "~/services/logger.server";
import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
import { ServiceValidationError } from "~/v3/services/baseService.server";
import { env } from "~/env.server";
import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
import { InitializeDeploymentService } from "~/v3/services/initializeDeployment.server";

export async function action({ request, params }: ActionFunctionArgs) {
Expand Down Expand Up @@ -60,6 +62,10 @@ export async function action({ request, params }: ActionFunctionArgs) {
? {
externalBuildData: result.deployment
.externalBuildData as InitializeDeploymentResponseBody["externalBuildData"],
baseImages: resolveDeployBaseImages(result.deployment.runtime, {
base: env.DEPLOY_BASE_IMAGES,
buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
}),
eventStream: result.eventStream,
canceledDeployments: result.canceledDeployments,
}
Expand Down
77 changes: 77 additions & 0 deletions apps/webapp/app/v3/deployBaseImages.server.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import { BuildRuntime, DeployBaseImageRef } from "@trigger.dev/core/v3";

type BaseImageMap = Partial<Record<BuildRuntime, string>>;

export function parseDeployBaseImages(
value: string | undefined,
envVarName: string
): { images: BaseImageMap; errors: string[] } {
const images: BaseImageMap = {};
const errors: string[] = [];

if (!value) {
return { images, errors };
}

for (const segment of value.split(",").map((s) => s.trim())) {
if (!segment) {
continue;
}

const fail = (reason: string) => errors.push(`${envVarName}: ${reason} in "${segment}"`);

const separator = segment.indexOf("=");
if (separator === -1) {
fail("expected runtime=image");
continue;
}

const runtimeName = segment.slice(0, separator).trim();
const image = segment.slice(separator + 1).trim();

if (runtimeName === "node") {
fail('runtime "node" is an alias; use the concrete runtime (node-22, node-24, node-26)');
continue;
}

const runtime = BuildRuntime.safeParse(runtimeName);
if (!runtime.success) {
fail(`unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})`);
continue;
}

if (!image) {
fail("missing image");
continue;
}

if (!DeployBaseImageRef.safeParse(image).success) {
fail("image must be image@sha256:<64 hex chars> with no whitespace before the digest");
continue;
}

if (runtime.data in images) {
fail(`duplicate runtime "${runtimeName}"`);
continue;
}

images[runtime.data] = image;
}

return { images, errors };
}

export function resolveDeployBaseImages(
runtime: string | null | undefined,
config: { base: BaseImageMap; buildBase: BaseImageMap }
): { base?: string; buildBase?: string } | undefined {
const parsedRuntime = BuildRuntime.safeParse(runtime);
if (!parsedRuntime.success) {
return undefined;
}

const base = config.base[parsedRuntime.data];
const buildBase = config.buildBase[parsedRuntime.data];

return base || buildBase ? { base, buildBase } : undefined;
}
20 changes: 20 additions & 0 deletions apps/webapp/app/v3/services/initializeDeployment.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server";
import { BaseService, ServiceValidationError } from "./baseService.server";
import { TimeoutDeploymentService } from "./timeoutDeployment.server";
import { resolveDeployBaseImages } from "../deployBaseImages.server";
import { getDeploymentImageRef } from "../getDeploymentImageRef.server";
import { tryCatch } from "@trigger.dev/core";
import { getRegistryConfig } from "../registryConfig.server";
Expand Down Expand Up @@ -147,6 +148,25 @@ export class InitializeDeploymentService extends BaseService {
throw new ServiceValidationError("UNMANAGED deployments are not supported");
}

const requiredBaseImages = resolveDeployBaseImages(runtime, {
base: env.DEPLOY_BASE_IMAGES,
buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
});
Comment thread
nicktrn marked this conversation as resolved.
Resolved

if (requiredBaseImages && payload.isNativeBuild) {
throw new ServiceValidationError(
"This instance requires custom deploy base images, which native builds cannot apply. Deploy without --native-build or --local-bundle.",
400
);
}

if (requiredBaseImages && payload.supportsInstanceBaseImages !== true) {
throw new ServiceValidationError(
"This instance requires custom deploy base images, which this version of the CLI cannot apply. Upgrade the trigger.dev CLI and deploy again.",
400
);
}

// Upgrade the project to engine "V2" if it's not already. This should cover cases where people deploy to V2 without running dev first.
if (payload.type === "MANAGED" && environment.project.engine === "V1") {
await this._prisma.project.update({
Expand Down
114 changes: 114 additions & 0 deletions apps/webapp/test/deployBaseImages.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import { describe, expect, it } from "vitest";
import { parseDeployBaseImages, resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
Comment thread
nicktrn marked this conversation as resolved.
Resolved

const digestA = `sha256:${"a".repeat(64)}`;
const digestB = `sha256:${"b".repeat(64)}`;
const digestC = `sha256:${"c".repeat(64)}`;

describe("parseDeployBaseImages", () => {
it("returns an empty map for undefined and empty values", () => {
const empty = { images: {}, errors: [] };
expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual(empty);
expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual(empty);
expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual(empty);
});

it("parses multiple entries and trims whitespace", () => {
expect(
parseDeployBaseImages(
` node-24 = acme/node-fips:24@${digestA} , bun=acme/bun:1@${digestB},`,
"DEPLOY_BASE_IMAGES"
)
).toEqual({
images: {
"node-24": `acme/node-fips:24@${digestA}`,
bun: `acme/bun:1@${digestB}`,
},
errors: [],
});
});

it("accepts a registry with a port and a tag before the digest", () => {
const image = `registry.example.com:5000/ns/img:tag@${digestA}`;
expect(parseDeployBaseImages(`node-24=${image}`, "DEPLOY_BASE_IMAGES")).toEqual({
images: { "node-24": image },
errors: [],
});
});

it.each([
["missing =", "garbage"],
["unknown runtime", `node-23=acme/node:23@${digestA}`],
["node alias", `node=acme/node:24@${digestA}`],
["empty image", "node-24="],
["missing digest", "node-24=acme/node:24"],
["flag before image", `node-24=--platform=linux/arm64 acme/node@${digestA}`],
["bare digest", `node-24=@${digestA}`],
["newline in image", `node-24=acme/node\nx@${digestA}`],
["duplicate runtime", `node-24=acme/a@${digestA},node-24=acme/b@${digestB}`],
])("reports an error naming the env var and segment: %s", (_name, value) => {
const segments = value.split(",");
const offending = segments[segments.length - 1]!.trim();

const { images, errors } = parseDeployBaseImages(
`node-22=acme/ok@${digestC},${value}`,
"DEPLOY_BUILD_BASE_IMAGES"
);

expect(images["node-22"]).toBe(`acme/ok@${digestC}`);
expect(errors).toHaveLength(1);
expect(errors[0]).toContain("DEPLOY_BUILD_BASE_IMAGES");
expect(errors[0]).toContain(offending);
});

it("explains that node is an alias", () => {
const { errors } = parseDeployBaseImages(`node=acme/node@${digestA}`, "DEPLOY_BASE_IMAGES");
expect(errors[0]).toContain('runtime "node" is an alias; use the concrete runtime');
});

it("reports every bad segment", () => {
const { errors } = parseDeployBaseImages(
`garbage,node-23=acme/node@${digestA},bun=acme/bun`,
"DEPLOY_BASE_IMAGES"
);
expect(errors).toHaveLength(3);
expect(errors[0]).toContain("garbage");
expect(errors[1]).toContain("node-23");
expect(errors[2]).toContain("bun=acme/bun");
});
});

describe("resolveDeployBaseImages", () => {
const base = { "node-26": `acme/node-fips:26@${digestA}` } as const;
const buildBase = { "node-26": `acme/node:26-dev@${digestB}` } as const;

it("returns undefined for a missing or unknown runtime", () => {
expect(resolveDeployBaseImages("node-23", { base, buildBase })).toBeUndefined();
expect(resolveDeployBaseImages(null, { base, buildBase })).toBeUndefined();
expect(resolveDeployBaseImages(undefined, { base, buildBase })).toBeUndefined();
});

it("returns undefined when the runtime has no entries", () => {
expect(resolveDeployBaseImages("bun", { base, buildBase })).toBeUndefined();
expect(resolveDeployBaseImages("node-26", { base: {}, buildBase: {} })).toBeUndefined();
});

it("returns both images", () => {
expect(resolveDeployBaseImages("node-26", { base, buildBase })).toEqual({
base: base["node-26"],
buildBase: buildBase["node-26"],
});
});

it("returns only the base image", () => {
expect(resolveDeployBaseImages("node-26", { base, buildBase: {} })).toEqual({
base: base["node-26"],
});
});

it("returns only the build base image", () => {
expect(resolveDeployBaseImages("node-26", { base: {}, buildBase })).toEqual({
buildBase: buildBase["node-26"],
});
});
});
2 changes: 2 additions & 0 deletions docs/self-hosting/env/webapp.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ mode: "wide"
| `DEPLOY_REGISTRY_NAMESPACE` | No | trigger | Deploy registry namespace. |
| `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY` | No | — | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. |
| `DEPLOY_IMAGE_PLATFORM` | No | linux/amd64 | Deploy image platform, same values as docker `--platform` flag. |
| `DEPLOY_BASE_IMAGES` | No | — | Base images every deploy must build on, as comma-separated `runtime=image@sha256:<digest>`. See [custom base images](/self-hosting/overview#custom-base-images). |
| `DEPLOY_BUILD_BASE_IMAGES` | No | — | Build-stage images, in the same format as `DEPLOY_BASE_IMAGES`. See [custom base images](/self-hosting/overview#custom-base-images). |
| `DEPLOY_TIMEOUT_MS` | No | 480000 (8m) | Deploy timeout (ms). |
| `DEPLOY_QUEUE_TIMEOUT_MS` | No | 900000 (15m) | Deploy queue timeout (ms). |
| **Object store (S3)** | | | |
Expand Down
26 changes: 26 additions & 0 deletions docs/self-hosting/overview.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,32 @@ All fields are optional. Partial overrides are supported:
}
```

## Custom base images

Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debian images. To require a different base for every deploy to your instance, such as a FIPS-validated or hardened Node image, set `DEPLOY_BASE_IMAGES` on the webapp (and optionally `DEPLOY_BUILD_BASE_IMAGES` for the build stage):

```bash
DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-character-digest>"
```

Entries are comma-separated `runtime=image@sha256:<digest>`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. With the Helm chart, set the variables through `webapp.extraEnvVars`.

Runtimes with an entry build on that image; the others keep the published images. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade, and so are `--native-build`, `--local-bundle` and `--from-bundle` deploys. This is a self-hosting setting and does not apply to Trigger.dev Cloud.

You own a custom base image. It must provide:

- `node` on `PATH` at the runtime's major version (for `bun`, both `bun` and `node`, because the final stage starts the app with `dumb-init node`)
- `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl`
- a `node` user (a `bun` user for the Bun runtime)
- glibc, so native modules built in the build stage load at runtime

A `DEPLOY_BUILD_BASE_IMAGES` image needs everything the base image provides, plus `python3`, `make` and `g++`. Build extensions that add image instructions are replayed on it. Without an entry, the build stage uses the published build image, except for projects whose build extensions add image instructions: those build from your base image and install the toolchain with `apt-get`, so that base must be Debian-based. To avoid the published images entirely, set both variables.

<Warning>
`image.pkgs` and build extensions that run `apt-get` (such as `aptGet` and `playwright`) assume a
Debian base. On other distributions, install those packages in your base image instead.
</Warning>

## Community support

It's dangerous to go alone! Join the self-hosting channel on our [Discord server](https://discord.gg/NQTxt5NA7s).
Expand Down
2 changes: 1 addition & 1 deletion packages/cli-v3/src/build/buildWorker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ async function readProjectPackageJson(packageJsonPath: string) {
return packageJson;
}

async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
export async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) {
throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]");
}
Expand Down
Loading
Loading