Hi, any idea that I'm getting these errors and a failed status for the mssql ingestion job when Enabling Stateful Ingestion
~~ Execution Summary - RUN_INGEST ~~
Execution finished with errors.
{'exec_id': 'f54bc15d-808f-4957-9e62-731d12cfa652',
'infos': ['2023-04-27 160731.851823 INFO: Starting execution for task with name=RUN_INGEST',
"2023-04-27 160803.582169 INFO: Failed to execute 'datahub ingest'",
'2023-04-27 160803.583391 INFO: Caught exception EXECUTING task_id=f54bc15d-808f-4957-9e62-731d12cfa652, name=RUN_INGEST, '
'stacktrace=Traceback (most recent call last):\n'
' File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/default_executor.py", line 122, in execute_task\n'
' task_event_loop.run_until_complete(task_future)\n'
' File "/usr/local/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete\n'
' return future.result()\n'
' File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/sub_process_ingestion_task.py", line 231, in execute\n'
' raise TaskError("Failed to execute \'datahub ingest\'")\n'
"acryl.executor.execution.task.TaskError: Failed to execute 'datahub ingest'\n"],
'errors': []}
~~ Ingestion Report ~~
{
"cli": {
"cli_version": "0.10.0.7",
"cli_entry_location": "/usr/local/lib/python3.10/site-packages/datahub/__init__.py",
"py_version": "3.10.10 (main, Mar 14 2023, 023711) [GCC 10.2.1 20210110]",
"py_exec_path": "/usr/local/bin/python",
"os_details": "Linux-5.15.0-60-generic-x86_64-with-glibc2.31",
"peak_memory_usage": "332.39 MB",
"mem_info": "332.39 MB"
},
'aspects': {'container': {'containerProperties': 2, 'status': 2, 'dataPlatformInstance': 2, 'subTypes': 2, 'container': 1},
'dataset': {'container': 23, 'status': 23, 'datasetProperties': 23, 'schemaMetadata': 23, 'subTypes': 23, 'datasetProfile': 23}},
'warnings': {},
'failures': {},
'soft_deleted_stale_entities': [],
'tables_scanned': 23,
'views_scanned': 0,
'entities_profiled': 23,
'filtered': ['db_accessadmin.*',
'db_backupoperator.*',
'db_datareader.*',
'db_datawriter.*',
'db_ddladmin.*',
'db_denydatareader.*',
'db_denydatawriter.*',
'db_owner.*',
'db_securityadmin.*',
'INFORMATION_SCHEMA.*',
'... sampled of 12 total elements'],
'query_combiner': {'total_queries': 1145,
'uncombined_queries_issued': 494,
'combined_queries_issued': 71,
'queries_combined': 337,
'query_exceptions': 314},
'start_time': '2023-04-27 160733.888433 (27.11 seconds ago)',
'running_time': '27.11 seconds'}
[2023-04-27 160801,000] INFO {datahub.cli.ingest_cli:137} - Sink (datahub-rest) report:
{'total_records_written': 101,
'records_written_per_second': 3,
'warnings': [],
'failures': [],
'start_time': '2023-04-27 160733.529165 (27.47 seconds ago)',
'current_time': '2023-04-27 160801.000169 (now)',
'total_duration_in_seconds': 27.47,
'gms_version': 'v0.10.2',
'pending_requests': 0}
[2023-04-27 160801,008] ERROR {datahub.entrypoints:188} - Command failed:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/datahub/entrypoints.py", line 175, in main
sys.exit(datahub(standalone_mode=False, **kwargs))
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1130, in call
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/click/decorators.py", line 26, in new_func
return f(get_current_context(), *args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/datahub/telemetry/telemetry.py", line 379, in wrapper
raise e
File "/usr/local/lib/python3.10/site-packages/datahub/telemetry/telemetry.py", line 334, in wrapper
res = func(*args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/datahub/utilities/memory_leak_detector.py", line 95, in wrapper
return func(ctx, *args, **kwargs)
File "/usr/local/lib/python3.10/site-packages/datahub/cli/ingest_cli.py", line 198, in run
loop.run_until_complete(run_func_check_upgrade(pipeline))
File "/usr/local/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
return future.result()
File "/usr/local/lib/python3.10/site-packages/datahub/cli/ingest_cli.py", line 158, in run_func_check_upgrade
ret = await the_one_future
File "/usr/local/lib/python3.10/site-packages/datahub/cli/ingest_cli.py", line 149, in run_pipeline_async
return await loop.run_in_executor(
File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
File "/usr/local/lib/python3.10/site-packages/datahub/cli/ingest_cli.py", line 140, in run_pipeline_to_completion
raise e
File "/usr/local/lib/python3.10/site-packages/datahub/cli/ingest_cli.py", line 132, in run_pipeline_to_completion
pipeline.run()
File "/usr/local/lib/python3.10/site-packages/datahub/ingestion/run/pipeline.py", line 339, in run
for wu in itertools.islice(
File "/usr/local/lib/python3.10/site-packages/datahub/utilities/source_helpers.py", line 98, in auto_stale_entity_removal
yield from stale_entity_removal_handler.gen_removed_entity_workunits()
File "/usr/local/lib/python3.10/site-packages/datahub/ingestion/source/state/stale_entity_removal_handler.py", line 267, in gen_removed_entity_workunits
last_checkpoint: Optional[Checkpoint] = self.source.get_last_checkpoint(
File "/usr/local/lib/python3.10/site-packages/datahub/ingestion/source/state/stateful_ingestion_base.py", line 337, in get_last_checkpoint
self.last_checkpoints[job_id] = self._get_last_checkpoint(
File "/usr/local/lib/python3.10/site-packages/datahub/ingestion/source/state/stateful_ingestion_base.py", line 301, in _get_last_checkpoint
last_checkpoint_aspect = self.ingestion_checkpointing_state_provider.get_latest_checkpoint( # type: ignore
File "/usr/local/lib/python3.10/site-packages/datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py", line 85, in get_latest_checkpoint
] = self.graph.get_latest_timeseries_value(
File "/usr/local/lib/python3.10/site-packages/datahub/ingestion/graph/client.py", line 292, in get_latest_timeseries_value
assert len(values) == 1
AssertionError