Hey all I have just started implementing DataHub t...
# troubleshoot
f
Hey all I have just started implementing DataHub to our data stack and using for cataloging mainly BigQuery data, and got some issues while data ingestion. All the tables are being stored, but no metadata is being imported for the views and materialized views. And also getting some errors when ingesting data sets with views or m views. It would be great if you guys can help me out. The log is as below:
Copy code
~~~~ Execution Summary - RUN_INGEST ~~~~
Execution finished with errors.
{'exec_id': 'ed590018-cf94-455f-8815-3a9835f40d80',
 'infos': ['2023-04-20 00:27:45.091260 INFO: Starting execution for task with name=RUN_INGEST',
           "2023-04-20 00:28:51.873290 INFO: Failed to execute 'datahub ingest'",
           '2023-04-20 00:28:51.875247 INFO: Caught exception EXECUTING task_id=ed590018-cf94-455f-8815-3a9835f40d80, name=RUN_INGEST, '
           'stacktrace=Traceback (most recent call last):\n'
           '  File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/default_executor.py", line 122, in execute_task\n'
           '    task_event_loop.run_until_complete(task_future)\n'
           '  File "/usr/local/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete\n'
           '    return future.result()\n'
           '  File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/sub_process_ingestion_task.py", line 231, in execute\n'
           '    raise TaskError("Failed to execute \'datahub ingest\'")\n'
           "acryl.executor.execution.task.TaskError: Failed to execute 'datahub ingest'\n"],
 'errors': []}

~~~~ Ingestion Report ~~~~
{
  "cli": {
    "cli_version": "0.10.1",
    "cli_entry_location": "/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/__init__.py",
    "py_version": "3.10.10 (main, Mar 14 2023, 02:37:11) [GCC 10.2.1 20210110]",
    "py_exec_path": "/tmp/datahub/ingest/venv-bigquery-0.10.1/bin/python3",
    "os_details": "Linux-5.15.0-1030-gcp-x86_64-with-glibc2.31",
    "peak_memory_usage": "308.13 MB",
    "mem_info": "308.13 MB"
  },
  "source": {
    "type": "bigquery",
    "report": {
      "events_produced": 68,
      "events_produced_per_sec": 1,
      "entities": {
        "container": [
          "urn:li:container:770462b97050e805bf0bbf39a8ba0334",
   
        ],
        "dataset": [
          "urn:li:dataset:(urn:li:dataPlatform:bigquery,)",
          "urn:li:dataset:(urn:li:dataPlatform:bigquery)",
 
          "... sampled of 20 total elements"
        ]
      },
      "aspects": {
        "container": {
          "containerProperties": 4,
          "status": 9,
          "dataPlatformInstance": 4,
          "subTypes": 4,
          "domains": 4,
          "container": 2
        },
        "dataset": {
          "status": 20,
          "schemaMetadata": 3,
          "datasetProperties": 2,
          "container": 4,
          "subTypes": 2,
          "domains": 4,
          "datasetUsageStatistics": 4,
          "datasetProfile": 1,
          "upstreamLineage": 1
        }
      },
      "warnings": {
        "profile skipped as partitioned table is empty or partition id was invalid": [
          table
        ]
      },
      "failures": {
        "metadata-extraction": [
          "tableid" - Unable to get tables for dataset ATE in project jade-2022-ee, skipping. Does your service account has bigquery.tables.list, bigquery.routines.get, bigquery.routines.list permission, bigquery.tables.getData permission? The error was: 'int' object has no attribute 'timestamp' - Traceback (most recent call last):\n  File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 626, in _process_project\n    yield from self._process_schema(\n  File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 782, in _process_schema\n    yield from self._process_view(\n  File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 882, in _process_view\n    yield from self.gen_view_dataset_workunits(\n  File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 951, in gen_view_dataset_workunits\n    yield from self.gen_dataset_workunits(\n  File \"/tmp/datahub/ingest/venv-bigquery-0.10.1/lib/python3.10/site-packages/datahub/ingestion/source/bigquery_v2/bigquery.py\", line 1007, in gen_dataset_workunits\n    lastModified=TimeStamp(time=int(table.last_altered.timestamp() * 1000))\nAttributeError: 'int' object has no attribute 'timestamp'\n"
        ]
      },
      "soft_deleted_stale_entities": [
        "urn:li:dataset:(urn:li",

      ],
      "tables_scanned": 2,
      "views_scanned": 1,
      "entities_profiled": 1,
      "filtered": [
        "... sampled of 33 total elements"
      ],
      "query_combiner": {
        "total_queries": 23,
        "uncombined_queries_issued": 10,
        "combined_queries_issued": 3,
        "queries_combined": 13,
        "query_exceptions": 0
      },
      "profiling_skipped_not_updated": {},
      "profiling_skipped_size_limit": {},
      "profiling_skipped_row_limit": {},
      "num_tables_not_eligible_profiling": {},
      "num_total_lineage_entries": {
      },
      "num_skipped_lineage_entries_missing_data": {},
      "num_skipped_lineage_entries_not_allowed": {
      },
      "num_lineage_entries_sql_parser_failure": {},
      "num_lineage_entries_sql_parser_success": {},
      "num_skipped_lineage_entries_other": {},
      "num_total_log_entries": {
      },
      "num_parsed_log_entries": {
      },
      "num_total_audit_entries": {},
      "num_parsed_audit_entries": {},
      "lineage_failed_extraction": [],
      "lineage_metadata_entries": {
      },
      "lineage_mem_size": {

      },
      "include_table_lineage": true,
      "use_date_sharded_audit_log_tables": false,
      "log_page_size": 1000,
      "use_exported_bigquery_audit_metadata": false,
      "log_entry_start_time": "2023-04-18T23:45:00Z",
      "log_entry_end_time": "2023-04-20T00:42:54Z",
      "upstream_lineage": {},
      "partition_info": {},
      "profile_table_selection_criteria": {},
      "selected_profile_tables": {},
      "invalid_partition_ids": {},
      "num_usage_workunits_emitted": 4,
      "total_query_log_entries": 507,
      "num_read_events": 34,
      "num_query_events": 220,
      "num_filtered_read_events": 253,
      "num_filtered_query_events": 0,
      "num_operational_stats_workunits_emitted": 0,
      "read_reasons_stat": {
        "JOB": 32,
        "TABLEDATA_LIST_REQUEST": 2
      },
      "operation_types_stat": {
        "SELECT": 24
      },
      "current_project_status": {
        "ultimate-bit-334500": {
          "Lineage Extraction": "2023-04-20 00:28:45.591964 (3.96 seconds ago)"
        }
      },
      "start_time": "2023-04-20 00:27:54.786969 (54.76 seconds ago)",
      "running_time": "54.76 seconds"
    }
  },
  "sink": {
    "type": "datahub-rest",
    "report": {
      "total_records_written": 54,
      "records_written_per_second": 0,
      "warnings": [],
      "failures": [],
      "start_time": "2023-04-20 00:27:47.753399 (1 minute and 1.8 seconds ago)",
      "current_time": "2023-04-20 00:28:49.549549 (now)",
      "total_duration_in_seconds": 61.8,
      "gms_version": "v0.10.1",
      "pending_requests": 0
    }
  }
}
l
This content can't be displayed.
a
Hery, what version are you on and what source are you trying to ingest from? A recipe would help as well