https://linen.dev logo
Join Slack
Powered by
Hi guys, i am having some real trouble with the MS...
# replication-troubleshooting
p
Hi guys, i am having some real trouble with the MS SQL Destination and large loads. Its randomly giving the below error, and then it starts another attempt. And it just keeps going and going.
Copy code
2026-04-07 10:51:21 replication-orchestrator INFO failures: [ {
  "failureOrigin" : "destination",
  "internalMessage" : "Destination process message delivery failed",
  "externalMessage" : "Something went wrong within the destination connector",
  "metadata" : {
    "attemptNumber" : 1,
    "jobId" : 139039,
    "connector_command" : "write"
  },
  "stacktrace" : "io.airbyte.workers.internal.exception.DestinationException: Destination process message delivery failed\n\tat io.airbyte.workers.general.BufferedReplicationWorker.writeToDestination(BufferedReplicationWorker.java:454)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.lambda$runAsyncWithTimeout$5(BufferedReplicationWorker.java:244)\n\tat java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\nCaused by: java.io.IOException: Broken pipe\n\tat java.base/sun.nio.ch.UnixFileDispatcherImpl.write0(Native Method)\n\tat java.base/sun.nio.ch.UnixFileDispatcherImpl.write(UnixFileDispatcherImpl.java:65)\n\tat java.base/sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:137)\n\tat java.base/sun.nio.ch.IOUtil.write(IOUtil.java:102)\n\tat java.base/sun.nio.ch.IOUtil.write(IOUtil.java:72)\n\tat java.base/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:300)\n\tat java.base/sun.nio.ch.ChannelOutputStream.writeFully(ChannelOutputStream.java:68)\n\tat java.base/sun.nio.ch.ChannelOutputStream.write(ChannelOutputStream.java:105)\n\tat java.base/sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:309)\n\tat java.base/sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:381)\n\tat java.base/sun.nio.cs.StreamEncoder.implWrite(StreamEncoder.java:357)\n\tat java.base/sun.nio.cs.StreamEncoder.lockedWrite(StreamEncoder.java:158)\n\tat java.base/sun.nio.cs.StreamEncoder.write(StreamEncoder.java:139)\n\tat java.base/java.io.OutputStreamWriter.write(OutputStreamWriter.java:219)\n\tat java.base/java.io.BufferedWriter.implFlushBuffer(BufferedWriter.java:178)\n\tat java.base/java.io.BufferedWriter.flushBuffer(BufferedWriter.java:163)\n\tat java.base/java.io.BufferedWriter.implWrite(BufferedWriter.java:334)\n\tat java.base/java.io.BufferedWriter.write(BufferedWriter.java:313)\n\tat java.base/java.io.Writer.write(Writer.java:278)\n\tat io.airbyte.workers.internal.VersionedAirbyteMessageBufferedWriter.write(VersionedAirbyteMessageBufferedWriter.java:39)\n\tat io.airbyte.workers.internal.LocalContainerAirbyteDestination.acceptWithNoTimeoutMonitor(LocalContainerAirbyteDestination.kt:137)\n\tat io.airbyte.workers.internal.LocalContainerAirbyteDestination.accept(LocalContainerAirbyteDestination.kt:96)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.writeToDestination(BufferedReplicationWorker.java:439)\n\t... 5 more\n",
  "timestamp" : 1775522960990
}, {
  "failureOrigin" : "replication",
  "internalMessage" : "No exit code found.",
  "externalMessage" : "Something went wrong during replication",
  "metadata" : {
    "attemptNumber" : 1,
    "jobId" : 139039
  },
  "stacktrace" : "java.lang.IllegalStateException: No exit code found.\n\tat io.airbyte.workers.internal.ContainerIOHandle.getExitCode(ContainerIOHandle.kt:101)\n\tat io.airbyte.workers.internal.LocalContainerAirbyteDestination.getExitValue(LocalContainerAirbyteDestination.kt:118)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.readFromDestination(BufferedReplicationWorker.java:496)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.lambda$runAsync$2(BufferedReplicationWorker.java:216)\n\tat java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n",
  "timestamp" : 1775522960989
}, {
  "failureOrigin" : "source",
  "internalMessage" : "Source process read attempt failed",
  "externalMessage" : "Something went wrong within the source connector",
  "metadata" : {
    "attemptNumber" : 1,
    "jobId" : 139039,
    "connector_command" : "read"
  },
  "stacktrace" : "io.airbyte.workers.internal.exception.SourceException: Source process read attempt failed\n\tat io.airbyte.workers.general.BufferedReplicationWorker.readFromSource(BufferedReplicationWorker.java:376)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.lambda$runAsyncWithHeartbeatCheck$3(BufferedReplicationWorker.java:223)\n\tat java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\nCaused by: java.lang.IllegalStateException: No exit code found.\n\tat io.airbyte.workers.internal.ContainerIOHandle.getExitCode(ContainerIOHandle.kt:101)\n\tat io.airbyte.workers.internal.LocalContainerAirbyteSource.getExitValue(LocalContainerAirbyteSource.kt:89)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.readFromSource(BufferedReplicationWorker.java:356)\n\t... 5 more\n",
  "timestamp" : 1775522961011
}, {
  "failureOrigin" : "replication",
  "internalMessage" : "io.airbyte.workers.exception.WorkerException: Destination has not terminated.  This warning is normal if the job was cancelled.",
  "externalMessage" : "Something went wrong during replication",
  "metadata" : {
    "attemptNumber" : 1,
    "jobId" : 139039
  },
  "stacktrace" : "java.lang.RuntimeException: io.airbyte.workers.exception.WorkerException: Destination has not terminated.  This warning is normal if the job was cancelled.\n\tat io.airbyte.workers.general.BufferedReplicationWorker$CloseableWithTimeout.lambda$close$0(BufferedReplicationWorker.java:548)\n\tat io.airbyte.workers.general.BufferedReplicationWorker.lambda$runAsyncWithTimeout$5(BufferedReplicationWorker.java:244)\n\tat java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\nCaused by: io.airbyte.workers.exception.WorkerException: Destination has not terminated.  This warning is normal if the job was cancelled.\n\tat io.airbyte.workers.internal.LocalContainerAirbyteDestination.close(LocalContainerAirbyteDestination.kt:64)\n\tat io.airbyte.workers.general.BufferedReplicationWorker$CloseableWithTimeout.lambda$close$0(BufferedReplicationWorker.java:546)\n\t... 5 more\n",
  "timestamp" : 1775523021083
} ]
m
Hi Paul, The error pattern you're seeing (broken pipe, no exit code, infinite retries) is a known issue where the destination connector crashes mid-sync, most likely from running out of memory. Two things to try: 1. Upgrade to the latest Airbyte version (v2.1+): Your logs reference 
BufferedReplicationWorker
, which means you might be on an older version. v2.1 fixes this scenario so the sync fails cleanly instead of retrying forever. 2. Increase destination memory: The default memory for the MSSQL destination may not be enough for large loads. You can increase it by updating the actor_definition against your Airbyte database. Let us know how it goes!
p
Thanks @Mark Grivnin I did try increasing the memory and CPU limits to something extremely high, but sadly this didn't resolve my issue. I attempted to do an upgrade in my test environment for my current version to 2.1+ but ran into lots of issues - do you have a clear migration path here?
m
Hey Paul, have you checked out this migration doc for Helm chart v2 and best practices? If you're using Helm, would suggest upgrading to v2.0 before trying v2.1 (as that's experimental).
k
Hi Mark,Paul, I'm experiencing the similar issue with MS SQL Destination v2.2.15. Running Airbyte 2.1.0 on a D4 machine (4 vCPU, 16GB RAM). Same pattern: Extract from Salesforce completes successfully (72k+ records)
Copy code
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.s.PipelineEventBookkeepingRouter(handleStreamMessage):126 Read COMPLETE for stream Descriptor(namespace=null, name=Account)
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.s.PipelineEventBookkeepingRouter(handleStreamMessage):126 Read COMPLETE for stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.t.i.ReservingDeserializingInputFlow(collect):55 Finished processing input
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.s.PipelineEventBookkeepingRouter(close):277 Maybe closing bookkeeping router 1
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.s.PipelineEventBookkeepingRouter(close):296 Closing internal control channels
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.m.MultiProducerChannel(close):26 Closing producer fileMessageQueue (active count=0, initialized count: 1)
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.m.MultiProducerChannel(close):30 Closing underlying queue
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-4 i.a.c.l.t.TaskScopeProvider$launch$job$1(invokeSuspend):36 Task TaskWrapper(io.airbyte.cdk.load.task.implementor.OpenStreamTask@65079351) completed
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-5 i.a.c.l.t.TaskScopeProvider$launch$job$1(invokeSuspend):36 Task TaskWrapper(io.airbyte.cdk.load.task.internal.InputConsumerTask@683bae53) completed
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-3 i.a.c.l.t.i.UpdateCheckpointsTask(execute):68 All checkpoints (state) updated
2026-04-15 23:49:55 destination INFO DefaultDispatcher-worker-3 i.a.c.l.t.TaskScopeProvider$launch$job$1(invokeSuspend):36 Task TaskWrapper(io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask@375b4f72) completed
2026-04-15 23:50:38 destination INFO DefaultDispatcher-worker-6 i.a.i.d.m.v.MSSQLDirectLoader(finish):76 Finishing batch 1 for stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:51:01 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(close):91 Closing connection for batch 1
2026-04-15 23:51:01 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoaderFactory(create):113 Creating query builder for batch 2 of stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:51:01 destination INFO DefaultDispatcher-worker-3 i.a.c.l.s.CheckpointManager(flushStreamCheckpoints):366 Flushed checkpoint for stream: Descriptor(namespace=null, name=Contact) at index: CheckpointKey(checkpointIndex=CheckpointIndex(value=1), checkpointId=CheckpointId(value=1)) (records=20, bytes=32499)
2026-04-15 23:52:16 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(finish):76 Finishing batch 2 for stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:52:39 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(close):91 Closing connection for batch 2
2026-04-15 23:52:39 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoaderFactory(create):113 Creating query builder for batch 3 of stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:53:53 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(finish):76 Finishing batch 3 for stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:54:16 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(close):91 Closing connection for batch 3
2026-04-15 23:54:16 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoaderFactory(create):113 Creating query builder for batch 4 of stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:55:30 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(finish):76 Finishing batch 4 for stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:55:52 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoader(close):91 Closing connection for batch 4
2026-04-15 23:55:52 destination INFO DefaultDispatcher-worker-2 i.a.i.d.m.v.MSSQLDirectLoaderFactory(create):113 Creating query builder for batch 5 of stream Descriptor(namespace=null, name=Contact)
2026-04-15 23:56:03 replication-orchestrator ERROR No exit code file found after waiting 10s. Container may have been killed (OOM or other signal). Returning fallback exit code 1.
2026-04-15 23:56:03 replication-orchestrator ERROR DestinationReader error: 
2026-04-15 23:56:03 replication-orchestrator INFO DestinationReader finished.
2026-04-15 23:56:03 replication-orchestrator ERROR runJobs failed; recording failure but continuing to finish.
2026-04-15 23:56:03 replication-orchestrator INFO Closing StateCheckSumCountEventHandler

2026-04-15 23:29:56 replication-orchestrator INFO Sync summary: {
  "status" : "failed",
  "recordsSynced" : 21,
  "bytesSynced" : 34510,
  "startTime" : 1776288172513,
  "endTime" : 1776288596512,
  "totalStats" : {
    "additionalStats" : { },
    "bytesCommitted" : 34510,
    "bytesEmitted" : 48176383,
    "destinationStateMessagesEmitted" : 1,
    "destinationWriteEndTime" : 0,
    "destinationWriteStartTime" : 1776288173015,
    "meanSecondsBeforeSourceStateMessageEmitted" : 29,
    "maxSecondsBeforeSourceStateMessageEmitted" : 1,
    "maxSecondsBetweenStateMessageEmittedandCommitted" : 108,
    "meanSecondsBetweenStateMessageEmittedandCommitted" : 108,
    "recordsEmitted" : 31612,
    "recordsCommitted" : 21,
    "recordsFilteredOut" : 0,
    "bytesFilteredOut" : 0,
    "replicationEndTime" : 1776288596511,
    "replicationStartTime" : 1776288172513,
    "sourceReadEndTime" : 1776288245973,
    "sourceReadStartTime" : 1776288173017,
    "sourceStateMessagesEmitted" : 5
  },
  "streamStats" : [ {
    "streamName" : "Contact",
    "stats" : {
      "additionalStats" : {
        "sourceFieldsPopulated" : 1572253
      },
      "bytesCommitted" : 34510,
      "bytesEmitted" : 48080073,
      "estimatedBytes" : 0,
      "estimatedRecords" : 0,
      "recordsEmitted" : 31574,
      "recordsCommitted" : 21,
      "recordsFilteredOut" : 0,
      "bytesFilteredOut" : 0
    }
  }, {
    "streamName" : "Account",
    "stats" : {
      "additionalStats" : {
        "sourceFieldsPopulated" : 2783
      },
      "bytesCommitted" : 0,
      "bytesEmitted" : 96310,
      "estimatedBytes" : 0,
      "estimatedRecords" : 0,
      "recordsEmitted" : 38,
      "recordsCommitted" : 0,
      "recordsFilteredOut" : 0,
      "bytesFilteredOut" : 0
    }
  } ],
  "performanceMetrics" : { },
  "streamCount" : 2
}
2026-04-15 23:29:56 replication-orchestrator INFO Failures: [ {
  "failureOrigin" : "destination",
  "internalMessage" : "Destination process exited with non-zero exit code 1",
  "externalMessage" : "Something went wrong within the destination connector",
  "metadata" : {
    "attemptNumber" : 0,
    "jobId" : 13,
    "connector_command" : "write"
  },
  "stacktrace" : "io.airbyte.workers.internal.exception.DestinationException: Destination process exited with non-zero exit code 1\n\tat io.airbyte.container.orchestrator.worker.DestinationReader.run(ReplicationTask.kt:55)\n\tat io.airbyte.container.orchestrator.worker.ReplicationWorker$runJobs$2$tasks$1$1.invokeSuspend(ReplicationWorker.kt:152)\n\tat io.airbyte.container.orchestrator.worker.ReplicationWorker$runJobs$2$tasks$1$1.invoke(ReplicationWorker.kt)\n\tat io.airbyte.container.orchestrator.worker.ReplicationWorker$runJobs$2$tasks$1$1.invoke(ReplicationWorker.kt)\n\tat io.airbyte.container.orchestrator.worker.util.AsyncUtils$runAsync$1$1.invokeSuspend(AsyncUtils.kt:22)\n\tat io.airbyte.container.orchestrator.worker.util.AsyncUtils$runAsync$1$1.invoke(AsyncUtils.kt)\n\tat io.airbyte.container.orchestrator.worker.util.AsyncUtils$runAsync$1$1.invoke(AsyncUtils.kt)\n\tat kotlinx.coroutines.intrinsics.UndispatchedKt.startUndspatched(Undispatched.kt:66)\n\tat kotlinx.coroutines.intrinsics.UndispatchedKt.startUndispatchedOrReturn(Undispatched.kt:43)\n\tat kotlinx.coroutines.BuildersKt__Builders_commonKt.withContext(Builders.common.kt:165)\n\tat kotlinx.coroutines.BuildersKt.withContext(Unknown Source)\n\tat io.airbyte.container.orchestrator.worker.util.AsyncUtils$runAsync$1.invokeSuspend(AsyncUtils.kt:21)\n\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:34)\n\tat kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:100)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1583)\n",
  "timestamp" : 1776288596511
} ]
3 Views
Open in Slack
PreviousNext
Powered by