Visitar URL original
fix(webhooks): stop admission-refusal retry loops and per-retry log writes by waleedlatif1 · Pull Request #8870 · simstudioai/sim · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
2fc4095
fix(execution): tag deterministic admission rejections and throttle b…
waleedlatif1 Oct 10, 2026
474c73c
fix(webhooks): acknowledge deterministic admission rejections for Tel…
waleedlatif1 Oct 10, 2026
a4d0689
fix(webhooks): skip polls for over-limit payers and back off failing …
waleedlatif1 Oct 10, 2026
46beb99
fix(webhooks): stop Slack redelivering to deleted trigger paths and l…
waleedlatif1 Oct 10, 2026
0de6619
fix(telegram): verify webhook deliveries with a per-webhook secret token
waleedlatif1 Oct 9, 2026
1ec25a8
refactor(webhooks): declare the Telegram admission opt-in on its handler
waleedlatif1 Oct 10, 2026
36f8a58
fix(execution): only treat billing and account refusals as deterministic
waleedlatif1 Oct 10, 2026
2da70cf
fix(webhooks): keep a fan-out target's retryable failure visible past…
waleedlatif1 Oct 10, 2026
5a9a90b
fix(telegram): match the active bot through env-var token references
waleedlatif1 Oct 10, 2026
46fe011
fix(webhooks): skip polls only after a recorded refusal and back off …
waleedlatif1 Oct 10, 2026
a36665f
refactor(webhooks): route every poller's source failures through one …
waleedlatif1 Oct 10, 2026
27a04be
chore(webhooks): tighten poll comments and backoff tests
waleedlatif1 Oct 10, 2026
011e4bb
fix(webhooks): stop every poller's batch on a deterministic admission…
waleedlatif1 Oct 10, 2026
0cedd07
fix(webhooks): never replay completed poll work or mask a retryable f…
waleedlatif1 Oct 10, 2026
317463a
fix(execution): drop the unreachable billing-account admission code
waleedlatif1 Oct 10, 2026
3a141dd
chore(webhooks): key blocked-run claims by gate and centralize the po…
waleedlatif1 Oct 10, 2026
5fc969c
fix(telegram): match the active bot with the env the caller resolved …
waleedlatif1 Oct 10, 2026
6080fb0
chore(testing): stub IdempotencyService.createWebhookIdempotencyKey i…
waleedlatif1 Oct 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix(webhooks): stop every poller's batch on a deterministic admission…
… refusal

Only RSS stopped at a refused item; Gmail, Outlook, and IMAP advanced their
cursors past refused emails, and every poller counted the refusals toward
auto-disable. A shared PollAdmissionRefusedError now leaves the idempotency
callback, stops the batch, and returns skipped before any cursor update or
failure count; items that already ran replay as idempotent no-ops.

Source backoff goes back to RSS only, where the rate-limited feed was: the other
pollers' fetch helpers do not carry status or Retry-After, so routing their
failures through it would back off on a guess. The block-missing 404 also tells
Slack not to redeliver.
  • Loading branch information
waleedlatif1 committed Oct 10, 2026
commit 011e4bb47de413a41685cc2868134447ca555a43
22 changes: 12 additions & 10 deletions apps/sim/lib/webhooks/polling/gmail.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@ import { pollingIdempotency } from '@/lib/core/idempotency/service'
import {
getProviderConfig,
type PollingProviderHandler,
type PollOutcome,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
recordPollSourceFailure,
PollAdmissionRefusedError,
resolveOAuthCredential,
skipAdmissionRefusedPoll,
throwIfAdmissionRefused,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
Expand Down Expand Up @@ -64,10 +67,9 @@ export const gmailPollingHandler: PollingProviderHandler = {
provider: 'gmail',
label: 'Gmail',

async pollWebhook(ctx: PollWebhookContext) {
async pollWebhook(ctx: PollWebhookContext): Promise<PollOutcome> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
const pollStartedAt = Date.now()

try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-email', requestId)
Expand Down Expand Up @@ -135,13 +137,11 @@ export const gmailPollingHandler: PollingProviderHandler = {
)
return 'success'
} catch (error) {
await recordPollSourceFailure(
webhookData,
pollStartedAt,
error,
`[${requestId}] Error polling Gmail webhook ${webhookId}`,
logger
)
if (error instanceof PollAdmissionRefusedError) {
return skipAdmissionRefusedPoll(logger, requestId, webhookId)
}
logger.error(`[${requestId}] Error processing Gmail webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
Expand Down Expand Up @@ -546,6 +546,7 @@ async function processEmails(
)

if (!result.success) {
throwIfAdmissionRefused(result)
logger.error(
`[${requestId}] Failed to process webhook for email ${email.id}:`,
result.statusCode,
Expand All @@ -567,6 +568,7 @@ async function processEmails(
)
processedCount++
} catch (error) {
if (error instanceof PollAdmissionRefusedError) throw error
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing email ${email.id}:`, errorMessage)
failedCount++
Expand Down
101 changes: 101 additions & 0 deletions apps/sim/lib/webhooks/polling/google-calendar.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
import { createLogger } from '@sim/logger'
import { createWorkflowRecord } from '@sim/testing'
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
import { jsonResponse } from '@sim/testing/helpers/http'
import {
webhooksProcessorMock,
webhooksProcessorMockFns,
} from '@sim/testing/mocks/webhooks-processor.mock'
import { beforeEach, describe, expect, it, vi } from 'vitest'

const { mockUpdateConfig, mockMarkFailed } = vi.hoisted(() => ({
mockUpdateConfig: vi.fn(),
mockMarkFailed: vi.fn(),
}))

vi.mock('@/lib/core/idempotency/service', () => ({
pollingIdempotency: {
executeWithIdempotency: vi.fn(
async (_provider: string, _key: string, execute: () => Promise<unknown>) => execute()
),
},
}))
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved

vi.mock('@/lib/webhooks/processor', () => webhooksProcessorMock)

vi.mock('@/lib/webhooks/polling/utils', async (importOriginal) => ({
...(await importOriginal<typeof import('@/lib/webhooks/polling/utils')>()),
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
resolveOAuthCredential: vi.fn().mockResolvedValue('access-token'),
markWebhookSuccess: vi.fn(),
markWebhookFailed: mockMarkFailed,
updateWebhookProviderConfig: mockUpdateConfig,
}))

import { ADMISSION_REJECTION_CODE } from '@/lib/core/admission/rejection'
import { googleCalendarPollingHandler } from '@/lib/webhooks/polling/google-calendar'
import type { PollWebhookContext, WebhookRecord } from '@/lib/webhooks/polling/types'

const mockProcessEvent = webhooksProcessorMockFns.mockProcessPolledWebhookEvent

function context(): PollWebhookContext {
const webhookData: WebhookRecord = {
id: 'calendar-webhook',
workflowId: 'calendar-listener',
deploymentVersionId: null,
registrationStatus: null,
registrationGeneration: null,
configFingerprint: null,
preparedAt: null,
blockId: null,
path: 'calendar-listener',
routingKey: null,
provider: 'google-calendar',
providerConfig: {
calendarId: 'primary',
lastCheckedTimestamp: '2026-10-09T11:00:00.000Z',
},
isActive: true,
failedCount: 0,
lastFailedAt: null,
archivedAt: null,
createdAt: new Date('2026-10-01T00:00:00.000Z'),
updatedAt: new Date('2026-10-09T11:00:00.000Z'),
}
return {
webhookData,
workflowData: createWorkflowRecord({
id: 'calendar-listener',
}) as PollWebhookContext['workflowData'],
requestId: 'calendar-request',
logger: createLogger('GoogleCalendarTest'),
}
}

describe('Google Calendar polling when execution admission refuses events', () => {
beforeEach(() => {
const events = ['event-1', 'event-2'].map((id) => ({
id,
status: 'confirmed',
created: '2026-10-09T11:30:00.000Z',
updated: '2026-10-09T11:30:00.000Z',
}))
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(jsonResponse({ items: events }, 200)))
mockProcessEvent.mockResolvedValue({
success: false,
statusCode: 402,
error: 'Usage limit exceeded',
code: ADMISSION_REJECTION_CODE.USAGE_LIMIT_EXCEEDED,
retryable: false,
})
})

it('stops the batch without advancing its cursor or counting a failure', async () => {
expect(await googleCalendarPollingHandler.pollWebhook(context())).toBe('skipped')

expect(mockProcessEvent).toHaveBeenCalledOnce()
const cursorUpdates = mockUpdateConfig.mock.calls.filter(
([, update]) => 'lastCheckedTimestamp' in (update as Record<string, unknown>)
)
expect(cursorUpdates).toEqual([])
expect(mockMarkFailed).not.toHaveBeenCalled()
})
})
22 changes: 12 additions & 10 deletions apps/sim/lib/webhooks/polling/google-calendar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
import {
getProviderConfig,
type PollingProviderHandler,
type PollOutcome,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
recordPollSourceFailure,
PollAdmissionRefusedError,
resolveOAuthCredential,
skipAdmissionRefusedPoll,
throwIfAdmissionRefused,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
Expand Down Expand Up @@ -95,10 +98,9 @@ export const googleCalendarPollingHandler: PollingProviderHandler = {
provider: 'google-calendar',
label: 'Google Calendar',

async pollWebhook(ctx: PollWebhookContext) {
async pollWebhook(ctx: PollWebhookContext): Promise<PollOutcome> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
const pollStartedAt = Date.now()

try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-calendar', requestId)
Expand Down Expand Up @@ -172,13 +174,11 @@ export const googleCalendarPollingHandler: PollingProviderHandler = {
)
return 'success'
} catch (error) {
await recordPollSourceFailure(
webhookData,
pollStartedAt,
error,
`[${requestId}] Error polling Google Calendar webhook ${webhookId}`,
logger
)
if (error instanceof PollAdmissionRefusedError) {
return skipAdmissionRefusedPoll(logger, requestId, webhookId)
}
logger.error(`[${requestId}] Error processing Google Calendar webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
Expand Down Expand Up @@ -336,6 +336,7 @@ async function processEvents(
)

if (!result.success) {
throwIfAdmissionRefused(result)
logger.error(
`[${requestId}] Failed to process webhook for event ${event.id}:`,
result.statusCode,
Expand All @@ -353,6 +354,7 @@ async function processEvents(
)
processedCount++
} catch (error) {
if (error instanceof PollAdmissionRefusedError) throw error
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing event ${event.id}:`, errorMessage)
failedCount++
Expand Down
22 changes: 12 additions & 10 deletions apps/sim/lib/webhooks/polling/google-drive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
import {
getProviderConfig,
type PollingProviderHandler,
type PollOutcome,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
recordPollSourceFailure,
PollAdmissionRefusedError,
resolveOAuthCredential,
skipAdmissionRefusedPoll,
throwIfAdmissionRefused,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
Expand Down Expand Up @@ -83,10 +86,9 @@ export const googleDrivePollingHandler: PollingProviderHandler = {
provider: 'google-drive',
label: 'Google Drive',

async pollWebhook(ctx: PollWebhookContext) {
async pollWebhook(ctx: PollWebhookContext): Promise<PollOutcome> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
const pollStartedAt = Date.now()

try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-drive', requestId)
Expand Down Expand Up @@ -171,6 +173,9 @@ export const googleDrivePollingHandler: PollingProviderHandler = {
)
return 'success'
} catch (error) {
if (error instanceof PollAdmissionRefusedError) {
return skipAdmissionRefusedPoll(logger, requestId, webhookId)
}
if (error instanceof Error && error.name === 'DrivePageTokenInvalidError') {
await updateWebhookProviderConfig(webhookId, { pageToken: undefined }, logger)
await markWebhookSuccess(webhookId, logger)
Expand All @@ -186,13 +191,8 @@ export const googleDrivePollingHandler: PollingProviderHandler = {
)
return 'success'
}
await recordPollSourceFailure(
webhookData,
pollStartedAt,
error,
`[${requestId}] Error polling Google Drive webhook ${webhookId}`,
logger
)
logger.error(`[${requestId}] Error processing Google Drive webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
Expand Down Expand Up @@ -404,6 +404,7 @@ async function processChanges(
)

if (!result.success) {
throwIfAdmissionRefused(result)
logger.error(
`[${requestId}] Failed to process webhook for file ${change.fileId}:`,
result.statusCode,
Expand All @@ -420,6 +421,7 @@ async function processChanges(
)
processedCount++
} catch (error) {
if (error instanceof PollAdmissionRefusedError) throw error
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(
`[${requestId}] Error processing change for file ${change.fileId}:`,
Expand Down
22 changes: 12 additions & 10 deletions apps/sim/lib/webhooks/polling/google-sheets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical'
import {
getProviderConfig,
type PollingProviderHandler,
type PollOutcome,
type PollWebhookContext,
} from '@/lib/webhooks/polling/types'
import {
markWebhookFailed,
markWebhookSuccess,
recordPollSourceFailure,
PollAdmissionRefusedError,
resolveOAuthCredential,
skipAdmissionRefusedPoll,
throwIfAdmissionRefused,
updateWebhookProviderConfig,
} from '@/lib/webhooks/polling/utils'
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
Expand Down Expand Up @@ -52,10 +55,9 @@ export const googleSheetsPollingHandler: PollingProviderHandler = {
provider: 'google-sheets',
label: 'Google Sheets',

async pollWebhook(ctx: PollWebhookContext) {
async pollWebhook(ctx: PollWebhookContext): Promise<PollOutcome> {
const { webhookData, workflowData, requestId, logger } = ctx
const webhookId = webhookData.id
const pollStartedAt = Date.now()

try {
const accessToken = await resolveOAuthCredential(webhookData, 'google-sheets', requestId)
Expand Down Expand Up @@ -229,13 +231,11 @@ export const googleSheetsPollingHandler: PollingProviderHandler = {
)
return 'success'
} catch (error) {
await recordPollSourceFailure(
webhookData,
pollStartedAt,
error,
`[${requestId}] Error polling Google Sheets webhook ${webhookId}`,
logger
)
if (error instanceof PollAdmissionRefusedError) {
return skipAdmissionRefusedPoll(logger, requestId, webhookId)
}
logger.error(`[${requestId}] Error processing Google Sheets webhook ${webhookId}:`, error)
await markWebhookFailed(webhookId, logger)
return 'failure'
}
},
Expand Down Expand Up @@ -433,6 +433,7 @@ async function processRows(
)

if (!result.success) {
throwIfAdmissionRefused(result)
logger.error(
`[${requestId}] Failed to process webhook for row ${rowNumber}:`,
result.statusCode,
Expand All @@ -450,6 +451,7 @@ async function processRows(
)
processedCount++
} catch (error) {
if (error instanceof PollAdmissionRefusedError) throw error
const errorMessage = getErrorMessage(error, 'Unknown error')
logger.error(`[${requestId}] Error processing row ${rowNumber}:`, errorMessage)
failedCount++
Expand Down
Loading