Visitar URL original
fix(webapp): stop the billing limits page timing out under enforcemen… · triggerdotdev/trigger.dev@bc3a33b · GitHub
Skip to content

Commit bc3a33b

Browse files
matt-aitkenmatt-aitken
authored
fix(webapp): stop the billing limits page timing out under enforcement (#4594)
## Summary Opening the billing limits page while a spend limit was being enforced could time out with no response for organizations with many preview branches. That is exactly the moment the page matters: it is the only self-serve way to raise or resolve the limit. The page now loads fast regardless of how many environments the organization has. ## Root cause and fix The loader's queued-run count ran one ClickHouse count per billable environment, sequentially, with no timeout, and the environment list included every archived preview branch ever created. Thousands of environments times one round trip each held the response open past the edge timeout. The count is now a single org-level ClickHouse query filtered on environment type, capped server-side with max_execution_time. If the count fails, the loader falls back to 0 (the page hides the count label at 0) instead of throwing, so the recovery panel stays reachable even when the count errors. The billing-limit bulk-cancel path also stops enumerating archived environments.
1 parent 622fa79 commit bc3a33b

5 files changed

Lines changed: 251 additions & 18 deletions

File tree

‎.server-changes/billing-limits-page-timeout.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
area: webapp
3+
type: fix
4+
---
5+
6+
Fixed the billing limits page timing out for organizations with many preview branches, especially while a spend limit was being enforced. The page now loads quickly, so you can raise or resolve your limit without delay.

‎apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ export const BILLING_LIMIT_RECONCILE_LOOKUP_CONCURRENCY = 10;
1616
/** Inline bulk-cancel budget for billing limit resolve (worker visibility is 10 min). */
1717
export const BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS = 8 * 60_000;
1818

19+
/** Server-side cap (seconds) on the org-level queued-run count; it renders in a loader. */
20+
export const BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S = 10;
21+
1922
export type BillingLimitConvergeTargetState = "grace" | "rejected" | "ok";
2023

2124
export function isBillableEnvironmentType(type: RuntimeEnvironmentType): boolean {

‎apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,25 @@
1+
import type { ClickHouse } from "@internal/clickhouse";
12
import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database";
23
import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
34
import { prisma } from "~/db.server";
45
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
56
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
6-
import { BILLABLE_ENVIRONMENT_TYPES } from "./billingLimitConstants";
7+
import {
8+
BILLABLE_ENVIRONMENT_TYPES,
9+
BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S,
10+
} from "./billingLimitConstants";
711

812
import { boundedIn } from "@trigger.dev/database";
913
export type BillableEnvironmentRef = {
1014
id: string;
1115
projectId: string;
1216
};
1317

18+
/**
19+
* Environments whose runs a billing limit must act on. Archived branches stay included:
20+
* archiving is a soft update that cancels nothing, so an archived branch can still hold
21+
* executing runs that enforcement has to cancel.
22+
*/
1423
export async function getBillableEnvironmentsForBillingLimit(
1524
organizationId: string,
1625
prismaClient: PrismaClient = prisma
@@ -73,27 +82,37 @@ async function countRunsForBillableEnvironment(
7382
});
7483
}
7584

76-
/** Same source as BillingLimitBulkCancelService — ClickHouse countRuns(QUEUED_STATUSES). */
85+
/**
86+
* Same table and statuses as BillingLimitBulkCancelService's per-environment counts, but a
87+
* single org-level ClickHouse query filtered on environment_type. Deliberately NOT a
88+
* per-environment loop: an org can have thousands of (mostly archived) preview environments,
89+
* and sequential per-env counts hold the billing-limits loader open past the edge timeout.
90+
* The count is display-only, so a server-side execution cap beats an unbounded query.
91+
*/
7792
export async function countBillableQueuedRunsForOrganization(
78-
organizationId: string
93+
organizationId: string,
94+
clickhouse?: ClickHouse
7995
): Promise<number> {
80-
const environments = await getBillableEnvironmentsForBillingLimit(organizationId);
96+
const client =
97+
clickhouse ??
98+
(await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard"));
8199

82-
if (environments.length === 0) {
83-
return 0;
84-
}
100+
const queryBuilder = client.taskRuns.countQueryBuilder({
101+
settings: { max_execution_time: BILLING_LIMIT_QUEUED_COUNT_MAX_EXECUTION_S },
102+
});
85103

86-
const runsRepository = await createBillingLimitRunsRepository(organizationId);
104+
queryBuilder
105+
.where("organization_id = {organizationId: String}", { organizationId })
106+
.where("environment_type IN {environmentTypes: Array(String)}", {
107+
environmentTypes: [...BILLABLE_ENVIRONMENT_TYPES],
108+
})
109+
.where("status IN {statuses: Array(String)}", { statuses: [...QUEUED_STATUSES] });
87110

88-
let total = 0;
111+
const [queryError, result] = await queryBuilder.execute();
89112

90-
for (const environment of environments) {
91-
total += await countQueuedRunsForBillableEnvironment(
92-
runsRepository,
93-
organizationId,
94-
environment
95-
);
113+
if (queryError) {
114+
throw queryError;
96115
}
97116

98-
return total;
117+
return result[0]?.count ?? 0;
99118
}

‎apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,26 @@
1+
import { tryCatch } from "@trigger.dev/core/utils";
12
import { EnvironmentPauseSource } from "@trigger.dev/database";
23
import { prisma } from "~/db.server";
4+
import { logger } from "~/services/logger.server";
35
import { countBillableQueuedRunsForOrganization } from "./billingLimitQueuedRuns.server";
46

7+
/**
8+
* Display-only count for the billing-limits page. Falls back to 0 on failure (the page hides
9+
* the count label at 0) — the recovery panel must stay reachable even when the count errors,
10+
* because it is the customer's only self-serve path out of an enforced limit.
11+
*/
512
export async function getBillingLimitQueuedRunCount(organizationId: string): Promise<number> {
6-
return countBillableQueuedRunsForOrganization(organizationId);
13+
const [error, count] = await tryCatch(countBillableQueuedRunsForOrganization(organizationId));
14+
15+
if (error) {
16+
logger.error("getBillingLimitQueuedRunCount failed, returning 0", {
17+
organizationId,
18+
error,
19+
});
20+
return 0;
21+
}
22+
23+
return count ?? 0;
724
}
825

926
export async function countBillingLimitPausedEnvironments(organizationId: string): Promise<number> {

‎apps/webapp/test/billingLimitQueuedRuns.test.ts‎

Lines changed: 189 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ import { describe, expect, vi } from "vitest";
22
import { setTimeout } from "node:timers/promises";
33
import { replicationContainerTest } from "@internal/testcontainers";
44
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
5-
import { countQueuedRunsForBillableEnvironment } from "~/v3/services/billingLimit/billingLimitQueuedRuns.server";
5+
import {
6+
countBillableQueuedRunsForOrganization,
7+
countQueuedRunsForBillableEnvironment,
8+
getBillableEnvironmentsForBillingLimit,
9+
} from "~/v3/services/billingLimit/billingLimitQueuedRuns.server";
610
import { setupClickhouseReplication } from "./utils/replicationUtils";
711

812
vi.setConfig({ testTimeout: 60_000 });
@@ -109,4 +113,188 @@ describe("billingLimitQueuedRuns", () => {
109113
expect(developmentCount).toBe(1);
110114
}
111115
);
116+
117+
replicationContainerTest(
118+
"counts queued runs org-wide with a single query, spanning billable environment types only",
119+
async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => {
120+
const { clickhouse } = await setupClickhouseReplication({
121+
prisma,
122+
databaseUrl: postgresContainer.getConnectionUri(),
123+
clickhouseUrl: clickhouseContainer.getConnectionUrl(),
124+
redisOptions,
125+
});
126+
127+
const organization = await prisma.organization.create({
128+
data: { title: "billing-limit-org-count", slug: "billing-limit-org-count" },
129+
});
130+
131+
const project = await prisma.project.create({
132+
data: {
133+
name: "billing-limit-org-count",
134+
slug: "billing-limit-org-count",
135+
organizationId: organization.id,
136+
externalRef: "billing-limit-org-count",
137+
},
138+
});
139+
140+
const productionEnv = await prisma.runtimeEnvironment.create({
141+
data: {
142+
slug: "prod",
143+
type: "PRODUCTION",
144+
projectId: project.id,
145+
organizationId: organization.id,
146+
apiKey: "prod-org-count",
147+
pkApiKey: "prod-org-count",
148+
shortcode: "prod-org-count",
149+
},
150+
});
151+
152+
const developmentEnv = await prisma.runtimeEnvironment.create({
153+
data: {
154+
slug: "dev",
155+
type: "DEVELOPMENT",
156+
projectId: project.id,
157+
organizationId: organization.id,
158+
apiKey: "dev-org-count",
159+
pkApiKey: "dev-org-count",
160+
shortcode: "dev-org-count",
161+
},
162+
});
163+
164+
const previewEnv = await prisma.runtimeEnvironment.create({
165+
data: {
166+
slug: "preview",
167+
type: "PREVIEW",
168+
projectId: project.id,
169+
organizationId: organization.id,
170+
apiKey: "preview-org-count",
171+
pkApiKey: "preview-org-count",
172+
shortcode: "preview-org-count",
173+
branchName: "feature-branch",
174+
archivedAt: new Date(),
175+
},
176+
});
177+
178+
const runRows = [
179+
{
180+
friendlyId: "run_org_prod_pending",
181+
env: productionEnv,
182+
type: "PRODUCTION",
183+
status: "PENDING",
184+
},
185+
{
186+
friendlyId: "run_org_prod_delayed",
187+
env: productionEnv,
188+
type: "PRODUCTION",
189+
status: "DELAYED",
190+
},
191+
{
192+
friendlyId: "run_org_prod_done",
193+
env: productionEnv,
194+
type: "PRODUCTION",
195+
status: "COMPLETED_SUCCESSFULLY",
196+
},
197+
{
198+
friendlyId: "run_org_dev_pending",
199+
env: developmentEnv,
200+
type: "DEVELOPMENT",
201+
status: "PENDING",
202+
},
203+
{
204+
friendlyId: "run_org_preview_pending",
205+
env: previewEnv,
206+
type: "PREVIEW",
207+
status: "PENDING",
208+
},
209+
] as const;
210+
211+
for (const row of runRows) {
212+
await prisma.taskRun.create({
213+
data: {
214+
friendlyId: row.friendlyId,
215+
taskIdentifier: "queued-task",
216+
status: row.status,
217+
payload: JSON.stringify({}),
218+
traceId: "trace",
219+
spanId: "span",
220+
queue: "main",
221+
runtimeEnvironmentId: row.env.id,
222+
projectId: project.id,
223+
organizationId: organization.id,
224+
environmentType: row.type,
225+
engine: "V2",
226+
},
227+
});
228+
}
229+
230+
await setTimeout(1000);
231+
232+
const count = await countBillableQueuedRunsForOrganization(organization.id, clickhouse);
233+
234+
expect(count).toBe(3);
235+
}
236+
);
237+
238+
replicationContainerTest(
239+
"keeps archived environments in the billable environment list so enforcement can cancel their runs",
240+
async ({ prisma }) => {
241+
const organization = await prisma.organization.create({
242+
data: { title: "billing-limit-archived", slug: "billing-limit-archived" },
243+
});
244+
245+
const project = await prisma.project.create({
246+
data: {
247+
name: "billing-limit-archived",
248+
slug: "billing-limit-archived",
249+
organizationId: organization.id,
250+
externalRef: "billing-limit-archived",
251+
},
252+
});
253+
254+
const activeEnv = await prisma.runtimeEnvironment.create({
255+
data: {
256+
slug: "preview-active",
257+
type: "PREVIEW",
258+
projectId: project.id,
259+
organizationId: organization.id,
260+
apiKey: "preview-active-test",
261+
pkApiKey: "preview-active-test",
262+
shortcode: "preview-active-test",
263+
branchName: "live-branch",
264+
},
265+
});
266+
267+
const archivedEnv = await prisma.runtimeEnvironment.create({
268+
data: {
269+
slug: "preview-archived",
270+
type: "PREVIEW",
271+
projectId: project.id,
272+
organizationId: organization.id,
273+
apiKey: "preview-archived-test",
274+
pkApiKey: "preview-archived-test",
275+
shortcode: "preview-archived-test",
276+
branchName: "old-branch",
277+
archivedAt: new Date(),
278+
},
279+
});
280+
281+
const developmentEnv = await prisma.runtimeEnvironment.create({
282+
data: {
283+
slug: "dev",
284+
type: "DEVELOPMENT",
285+
projectId: project.id,
286+
organizationId: organization.id,
287+
apiKey: "dev-archived-test",
288+
pkApiKey: "dev-archived-test",
289+
shortcode: "dev-archived-test",
290+
},
291+
});
292+
293+
const environments = await getBillableEnvironmentsForBillingLimit(organization.id, prisma);
294+
const environmentIds = environments.map((environment) => environment.id).sort();
295+
296+
expect(environmentIds).toEqual([activeEnv.id, archivedEnv.id].sort());
297+
expect(environmentIds).not.toContain(developmentEnv.id);
298+
}
299+
);
112300
});

0 commit comments

Comments
 (0)