faint-boots-14622
04/20/2023, 1:11 AM~~~~ Execution Summary - RUN_INGEST ~~~~
Execution finished with errors.
{'exec_id': 'ed590018-cf94-455f-8815-3a9835f40d80',
'infos': ['2023-04-20 00:27:45.091260 INFO: Starting execution for task with name=RUN_INGEST',
"2023-04-20 00:28:51.873290 INFO: Failed to execute 'datahub ingest'",
'2023-04-20 00:28:51.875247 INFO: Caught exception EXECUTING task_id=ed590018-cf94-455f-8815-3a9835f40d80, name=RUN_INGEST, '
'stacktrace=Traceback (most recent call last):\n'
' File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/default_executor.py", line 122, in execute_task\n'
' task_event_loop.run_until_complete(task_future)\n'
' File "/usr/local/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete\n'
' return future.result()\n'
' File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/sub_process_ingestion_task.py", line 231, in execute\n'
' raise TaskError("Failed to execute \'datahub ingest\'")\n'
"acryl.executor.execution.task.TaskError: Failed to execute 'datahub ingest'\n"],
'errors': []}
~~~~ Ingestion Report ~~~~
{
"cli": {
"cli_version": "0.10.1",
"cli_entry_location": "/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/__init__.py",
"py_version": "3.10.10 (main, Mar 14 2023, 02:37:11) [GCC 10.2.1 20210110]",
"py_exec_path": "/tmp/datahub/ingest/venv-bigquery-0.10.1/bin/python3",
"os_details": "Linux-5.15.0-1030-gcp-x86_64-with-glibc2.31",
"peak_memory_usage": "308.13 MB",
"mem_info": "308.13 MB"
},
"source": {
"type": "bigquery",
"report": {
"events_produced": 68,
"events_produced_per_sec": 1,
"entities": {
"container": [
"urn:li:container:770462b97050e805bf0bbf39a8ba0334",
],
"dataset": [
"urn:li:dataset:(urn:li:dataPlatform:bigquery,)",
"urn:li:dataset:(urn:li:dataPlatform:bigquery)",
"... sampled of 20 total elements"
]
},
"aspects": {
"container": {
"containerProperties": 4,
"status": 9,
"dataPlatformInstance": 4,
"subTypes": 4,
"domains": 4,
"container": 2
},
"dataset": {
"status": 20,
"schemaMetadata": 3,
"datasetProperties": 2,
"container": 4,
"subTypes": 2,
"domains": 4,
"datasetUsageStatistics": 4,
"datasetProfile": 1,
"upstreamLineage": 1
}
},
"warnings": {
"profile skipped as partitioned table is empty or partition id was invalid": [
table
]
},
"failures": {
"metadata-extraction": [
"tableid" - Unable to get tables for dataset ATE in project jade-2022-ee, skipping. Does your service account has bigquery.tables.list, bigquery.routines.get, bigquery.routines.list permission, bigquery.tables.getData permission? The error was: 'int' object has no attribute 'timestamp' - Traceback (most recent call last):\n File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 626, in _process_project\n yield from self._process_schema(\n File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 782, in _process_schema\n yield from self._process_view(\n File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 882, in _process_view\n yield from self.gen_view_dataset_workunits(\n File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 951, in gen_view_dataset_workunits\n yield from self.gen_dataset_workunits(\n File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 1007, in gen_dataset_workunits\n lastModified=TimeStamp(time=int(table.last_altered.timestamp() * 1000))\nAttributeError: 'int' object has no attribute 'timestamp'\n"
]
},
"soft_deleted_stale_entities": [
"urn:li:dataset:(urn:li",
],
"tables_scanned": 2,
"views_scanned": 1,
"entities_profiled": 1,
"filtered": [
"... sampled of 33 total elements"
],
"query_combiner": {
"total_queries": 23,
"uncombined_queries_issued": 10,
"combined_queries_issued": 3,
"queries_combined": 13,
"query_exceptions": 0
},
"profiling_skipped_not_updated": {},
"profiling_skipped_size_limit": {},
"profiling_skipped_row_limit": {},
"num_tables_not_eligible_profiling": {},
"num_total_lineage_entries": {
},
"num_skipped_lineage_entries_missing_data": {},
"num_skipped_lineage_entries_not_allowed": {
},
"num_lineage_entries_sql_parser_failure": {},
"num_lineage_entries_sql_parser_success": {},
"num_skipped_lineage_entries_other": {},
"num_total_log_entries": {
},
"num_parsed_log_entries": {
},
"num_total_audit_entries": {},
"num_parsed_audit_entries": {},
"lineage_failed_extraction": [],
"lineage_metadata_entries": {
},
"lineage_mem_size": {
},
"include_table_lineage": true,
"use_date_sharded_audit_log_tables": false,
"log_page_size": 1000,
"use_exported_bigquery_audit_metadata": false,
"log_entry_start_time": "2023-04-18T23:45:00Z",
"log_entry_end_time": "2023-04-20T00:42:54Z",
"upstream_lineage": {},
"partition_info": {},
"profile_table_selection_criteria": {},
"selected_profile_tables": {},
"invalid_partition_ids": {},
"num_usage_workunits_emitted": 4,
"total_query_log_entries": 507,
"num_read_events": 34,
"num_query_events": 220,
"num_filtered_read_events": 253,
"num_filtered_query_events": 0,
"num_operational_stats_workunits_emitted": 0,
"read_reasons_stat": {
"JOB": 32,
"TABLEDATA_LIST_REQUEST": 2
},
"operation_types_stat": {
"SELECT": 24
},
"current_project_status": {
"ultimate-bit-334500": {
"Lineage Extraction": "2023-04-20 00:28:45.591964 (3.96 seconds ago)"
}
},
"start_time": "2023-04-20 00:27:54.786969 (54.76 seconds ago)",
"running_time": "54.76 seconds"
}
},
"sink": {
"type": "datahub-rest",
"report": {
"total_records_written": 54,
"records_written_per_second": 0,
"warnings": [],
"failures": [],
"start_time": "2023-04-20 00:27:47.753399 (1 minute and 1.8 seconds ago)",
"current_time": "2023-04-20 00:28:49.549549 (now)",
"total_duration_in_seconds": 61.8,
"gms_version": "v0.10.1",
"pending_requests": 0
}
}
}lively-cat-88289
04/20/2023, 1:11 AMastonishing-answer-96712
04/20/2023, 9:18 PM