From 478943a0eb71179edbd7d015cfe2ed45ba37949d Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Thu, 18 Jun 2026 17:09:18 +0200 Subject: [PATCH 1/2] fix(server): treat released query runner as a transient import error A torn-down DB connection (e.g. node-postgres query_timeout destroying a pooled connection mid-transaction) surfaces as TypeORM's QueryRunnerAlreadyReleasedError. During calendar/message import this fell through to the unknown-error path, marking the channel FAILED, flushing pending events, and firing a Sentry alert for what is a transient issue. Map QueryRunnerAlreadyReleasedError to a new retryable TwentyORMExceptionCode.QUERY_RUNNER_RELEASED in computeTwentyORMException and handle it as a temporary error in the calendar and messaging import exception handlers, mirroring the existing QUERY_READ_TIMEOUT handling. --- .../compute-twenty-orm-exception.spec.ts | 28 +++++++++++++++++++ .../compute-twenty-orm-exception.ts | 13 ++++++++- .../exceptions/twenty-orm.exception.ts | 3 ++ ...-event-import-exception-handler.service.ts | 1 + ...saging-import-exception-handler.service.ts | 1 + 5 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts diff --git a/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts b/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts new file mode 100644 index 0000000000000..8f5f9a62265ff --- /dev/null +++ b/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts @@ -0,0 +1,28 @@ +import { QueryRunnerAlreadyReleasedError } from 'typeorm'; + +import { computeTwentyORMException } from 'src/engine/twenty-orm/error-handling/compute-twenty-orm-exception'; +import { + TwentyORMException, + TwentyORMExceptionCode, +} from 'src/engine/twenty-orm/exceptions/twenty-orm.exception'; + +describe('computeTwentyORMException', () => { + it('should map a released query runner error to a retryable QUERY_RUNNER_RELEASED exception', async () => { + const error = new QueryRunnerAlreadyReleasedError(); + + const result = await computeTwentyORMException(error); + + expect(result).toBeInstanceOf(TwentyORMException); + expect((result as TwentyORMException).code).toBe( + TwentyORMExceptionCode.QUERY_RUNNER_RELEASED, + ); + }); + + it('should return unrelated errors unchanged', async () => { + const error = new Error('some other error'); + + const result = await computeTwentyORMException(error); + + expect(result).toBe(error); + }); +}); diff --git a/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts b/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts index f9fad80e6b1b2..b0f5e45c069a0 100644 --- a/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts +++ b/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts @@ -1,6 +1,6 @@ import { msg } from '@lingui/core/macro'; import { isDefined } from 'twenty-shared/utils'; -import { QueryFailedError } from 'typeorm'; +import { QueryFailedError, QueryRunnerAlreadyReleasedError } from 'typeorm'; import { type WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface'; @@ -24,6 +24,17 @@ export const computeTwentyORMException = async ( entityManager?: WorkspaceEntityManager, internalContext?: WorkspaceInternalContext, ): Promise => { + // A released query runner means the underlying connection was torn down + // mid-flight (e.g. node-postgres `query_timeout` destroying the pooled + // connection during a transaction). It is transient, so we surface it as a + // retryable error rather than an opaque failure. + if (error instanceof QueryRunnerAlreadyReleasedError) { + return new TwentyORMException( + error.message, + TwentyORMExceptionCode.QUERY_RUNNER_RELEASED, + ); + } + if (error instanceof QueryFailedError) { if (error.message.includes('Query read timeout')) { return new TwentyORMException( diff --git a/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts b/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts index 95dd81e3e3fd1..cca387fc4bfc1 100644 --- a/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts +++ b/packages/twenty-server/src/engine/twenty-orm/exceptions/twenty-orm.exception.ts @@ -21,6 +21,7 @@ export enum TwentyORMExceptionCode { METHOD_NOT_ALLOWED = 'METHOD_NOT_ALLOWED', ENUM_TYPE_NAME_NOT_FOUND = 'ENUM_TYPE_NAME_NOT_FOUND', QUERY_READ_TIMEOUT = 'QUERY_READ_TIMEOUT', + QUERY_RUNNER_RELEASED = 'QUERY_RUNNER_RELEASED', DUPLICATE_ENTRY_DETECTED = 'DUPLICATE_ENTRY_DETECTED', TOO_MANY_RECORDS_TO_UPDATE = 'TOO_MANY_RECORDS_TO_UPDATE', INVALID_INPUT = 'INVALID_INPUT', @@ -61,6 +62,8 @@ const getTwentyORMExceptionUserFriendlyMessage = ( return msg`This operation is not allowed.`; case TwentyORMExceptionCode.QUERY_READ_TIMEOUT: return msg`Query timed out. Please try again.`; + case TwentyORMExceptionCode.QUERY_RUNNER_RELEASED: + return msg`We are experiencing a temporary issue with our database. Please try again later.`; case TwentyORMExceptionCode.DUPLICATE_ENTRY_DETECTED: return msg`A duplicate entry was detected.`; case TwentyORMExceptionCode.TOO_MANY_RECORDS_TO_UPDATE: diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts index 5176809fcea0d..651ac4335b668 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts @@ -57,6 +57,7 @@ export class CalendarEventImportErrorHandlerService { ); break; case TwentyORMExceptionCode.QUERY_READ_TIMEOUT: + case TwentyORMExceptionCode.QUERY_RUNNER_RELEASED: case CalendarEventImportDriverExceptionCode.TEMPORARY_ERROR: case ConnectedAccountRefreshAccessTokenExceptionCode.TEMPORARY_NETWORK_ERROR: await this.handleTemporaryException( diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts index 893b9b207ba66..5e5f7eaa5f309 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts @@ -72,6 +72,7 @@ export class MessageImportExceptionHandlerService { ); break; case TwentyORMExceptionCode.QUERY_READ_TIMEOUT: + case TwentyORMExceptionCode.QUERY_RUNNER_RELEASED: case MessageImportDriverExceptionCode.TEMPORARY_ERROR: case ConnectedAccountRefreshAccessTokenExceptionCode.TEMPORARY_NETWORK_ERROR: case MessageNetworkExceptionCode.ECONNABORTED: From 51236d30a77509354903dad1f817fc7702e7ec7c Mon Sep 17 00:00:00 2001 From: Thomas Trompette Date: Thu, 18 Jun 2026 17:41:40 +0200 Subject: [PATCH 2/2] fix(server): preserve stack and report released-runner error to locate root cause The released-runner error is a transient lifecycle race, so it is retried like other temporary errors instead of hard-failing the channel. The exact operation that escapes its transaction boundary could not be pinned statically, and the call site was masked because the error was re-wrapped twice, discarding the original stack. Preserve the original stack and cause when wrapping in computeTwentyORMException, and report the released-runner error to the exception handler on every occurrence so the originating query surfaces in monitoring and the root cause can be located. --- .../compute-twenty-orm-exception.spec.ts | 9 +++++++++ .../compute-twenty-orm-exception.ts | 14 +++++++++----- ...r-event-import-exception-handler.service.ts | 18 +++++++++++++++++- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts b/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts index 8f5f9a62265ff..19af39367a797 100644 --- a/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts +++ b/packages/twenty-server/src/engine/twenty-orm/error-handling/__tests__/compute-twenty-orm-exception.spec.ts @@ -18,6 +18,15 @@ describe('computeTwentyORMException', () => { ); }); + it('should preserve the original stack and cause when wrapping a released runner error', async () => { + const error = new QueryRunnerAlreadyReleasedError(); + + const result = await computeTwentyORMException(error); + + expect(result.stack).toBe(error.stack); + expect((result as TwentyORMException).cause).toBe(error); + }); + it('should return unrelated errors unchanged', async () => { const error = new Error('some other error'); diff --git a/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts b/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts index b0f5e45c069a0..80a3971e0c76b 100644 --- a/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts +++ b/packages/twenty-server/src/engine/twenty-orm/error-handling/compute-twenty-orm-exception.ts @@ -24,15 +24,19 @@ export const computeTwentyORMException = async ( entityManager?: WorkspaceEntityManager, internalContext?: WorkspaceInternalContext, ): Promise => { - // A released query runner means the underlying connection was torn down - // mid-flight (e.g. node-postgres `query_timeout` destroying the pooled - // connection during a transaction). It is transient, so we surface it as a - // retryable error rather than an opaque failure. if (error instanceof QueryRunnerAlreadyReleasedError) { - return new TwentyORMException( + const releasedRunnerException = new TwentyORMException( error.message, TwentyORMExceptionCode.QUERY_RUNNER_RELEASED, ); + + // Keep the original stack: it points at the query that ran on the + // released runner, which is the only way to locate where the operation + // escaped its transaction boundary. + releasedRunnerException.stack = error.stack; + releasedRunnerException.cause = error; + + return releasedRunnerException; } if (error instanceof QueryFailedError) { diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts index 651ac4335b668..472974e9cfa0c 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-event-import-exception-handler.service.ts @@ -56,8 +56,24 @@ export class CalendarEventImportErrorHandlerService { workspaceId, ); break; - case TwentyORMExceptionCode.QUERY_READ_TIMEOUT: case TwentyORMExceptionCode.QUERY_RUNNER_RELEASED: + // Transient, so retried like any temporary error - but also reported + // every time (with the original stack preserved upstream) so the + // operation that escapes its transaction boundary can be located. + this.exceptionHandlerService.captureExceptions([exception], { + additionalData: { + calendarChannelId: calendarChannel.id, + syncStep, + }, + workspace: { id: workspaceId }, + }); + await this.handleTemporaryException( + syncStep, + calendarChannel, + workspaceId, + ); + break; + case TwentyORMExceptionCode.QUERY_READ_TIMEOUT: case CalendarEventImportDriverExceptionCode.TEMPORARY_ERROR: case ConnectedAccountRefreshAccessTokenExceptionCode.TEMPORARY_NETWORK_ERROR: await this.handleTemporaryException(