Artem Astapenko
03/24/2021, 4:31 PMairbyte-temporal | {"level":"error","ts":"2021-03-24T16:09:19.365Z","msg":"Operation failed with internal error.","service":"matching","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","metric-scope":38,"logging-call-at":"persistenceMetricClients.go:804","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:09:19.365Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/temporal-sys-tq-scanner-taskqueue-0/1","wf-task-queue-type":"Activity","store-operation":"update-task-queue","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","logging-call-at":"taskReader.go:166","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166>"}
airbyte-scheduler | 2021-03-24 16:09:23 INFO i.a.s.JobRetrier(run):53 - {workspace_app_root=/tmp/workspace/scheduler/logs} - Running Job Retrier...
maybe thats because my Internet stopped working, but the main problem is next: CPU usage increased a lot ( and probably memory leak too ), so my pc almost got freezed. Is it possible to limit usage somehow? I suppose I am not the only one who can face such issue.user
03/24/2021, 4:37 PMuser
03/24/2021, 4:37 PMuser
03/24/2021, 4:38 PMuser
03/24/2021, 5:23 PMuser
03/24/2021, 5:23 PMairbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.258Z","msg":"Operation failed with internal error.","service":"matching","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","metric-scope":38,"logging-call-at":"persistenceMetricClients.go:804","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.270Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/CHECK_CONNECTION/3","wf-task-queue-type":"Activity","store-operation":"update-task-queue","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","logging-call-at":"taskReader.go:166","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.304Z","msg":"Operation failed with internal error.","service":"matching","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","metric-scope":38,"logging-call-at":"persistenceMetricClients.go:804","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.304Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"ff8c08c10fbe:3492e393-c285-45b7-9976-307332ec9a55","wf-task-queue-type":"Workflow","store-operation":"update-task-queue","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","logging-call-at":"taskReader.go:166","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.824Z","msg":"Operation failed with internal error.","service":"matching","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","metric-scope":38,"logging-call-at":"persistenceMetricClients.go:804","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.824Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"/_sys/temporal-sys-processor-parent-close-policy/2","wf-task-queue-type":"Activity","store-operation":"update-task-queue","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","logging-call-at":"taskReader.go:166","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.857Z","msg":"Operation failed with internal error.","service":"matching","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","metric-scope":38,"logging-call-at":"persistenceMetricClients.go:804","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).updateErrorMetric\n\t/temporal/common/persistence/persistenceMetricClients.go:804\ngo.temporal.io/server/common/persistence.(*taskPersistenceClient).UpdateTaskQueue\n\t/temporal/common/persistence/persistenceMetricClients.go:787\ngo.temporal.io/server/service/matching.(*taskQueueDB).UpdateState\n\t/temporal/service/matching/db.go:109\ngo.temporal.io/server/service/matching.(*taskReader).persistAckLevel\n\t/temporal/service/matching/taskReader.go:272\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:160>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.858Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"ff8c08c10fbe:85d97e18-48b4-42f6-b9f9-c729ad638dc2","wf-task-queue-type":"Workflow","store-operation":"update-task-queue","error":"UpdateTaskQueue failed. Failed to start transaction. Error: context deadline exceeded","logging-call-at":"taskReader.go:166","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/temporal/service/matching/taskReader.go:166>"}
airbyte-temporal | {"level":"error","ts":"2021-03-24T16:08:11.907Z","msg":"transaction rollback error","service":"matching","error":"sql: transaction has already been committed or rolled back","logging-call-at":"common.go:68","stacktrace":"<http://go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence/sql|go.temporal.io/server/common/log/loggerimpl.(*loggerImpl).Error\n\t/temporal/common/log/loggerimpl/logger.go:138\ngo.temporal.io/server/common/persistence/sql>