dbt

dbt

source:
  type: "dbt"
  config:
    # Coordinates
    manifest_path: "<s3://bucket_name/manifest.json>"
    catalog_path: "<s3://bucket_name/catalog.json>"
    sources_path: "<s3://bucket_name/sources.json>"

    aws_connection:
      aws_region: "eu-west-2"

    # Options
    target_platform: "snowflake"
    load_schemas: True
    env: STG
    node_name_pattern:
      allow:
        - ".*branch_activities.*"
      deny:
        - ".*test.*"

sink:
  type: "datahub-rest"
  config:
    server: "<http://datahub-datahub-gms:8080>"

/relationships

curl --location --request GET --header 'X-RestLi-Protocol-Version: 2.0.0' '<http://localhost:8080/relationships?direction=OUTGOING&urn=urn%3Ali%3Achart%3Acustomers&types=List(OwnedBy)>'

dataHubPolicy

great_expectations

redshift-usage

'[2022-06-17 03:11:32,443] INFO     {datahub.cli.ingest_cli:97} - DataHub CLI version: 0.8.36\n'
           '[2022-06-17 03:11:36,611] INFO     {datahub.cli.ingest_cli:113} - Starting metadata ingestion\n'
           '/usr/local/bin/run_ingest.sh: line 26:   320 Killed                  ( python3 -m datahub ingest -c "$4/$1.yml" )\n',
           "2022-06-17 03:14:45.501901 [exec_id=8ccd646c-fc06-4bfd-a7af-cc622cc5be81] INFO: Failed to execute 'datahub ingest'",
           '2022-06-17 03:14:45.505219 [exec_id=8ccd646c-fc06-4bfd-a7af-cc622cc5be81] INFO: Caught exception EXECUTING '
           'task_id=8ccd646c-fc06-4bfd-a7af-cc622cc5be81, name=RUN_INGEST, stacktrace=Traceback (most recent call last):\n'
           '  File "/usr/local/lib/python3.9/site-packages/acryl/executor/execution/default_executor.py", line 119, in execute_task\n'
           '    self.event_loop.run_until_complete(task_future)\n'
           '  File "/usr/local/lib/python3.9/site-packages/nest_asyncio.py", line 89, in run_until_complete\n'
           '    return f.result()\n'
           '  File "/usr/local/lib/python3.9/asyncio/futures.py", line 201, in result\n'
           '    raise self._exception\n'
           '  File "/usr/local/lib/python3.9/asyncio/tasks.py", line 256, in __step\n'
           '    result = coro.send(None)\n'
           '  File "/usr/local/lib/python3.9/site-packages/acryl/executor/execution/sub_process_ingestion_task.py", line 115, in execute\n'
           '    raise TaskError("Failed to execute \'datahub ingest\'")\n'
           "acryl.executor.execution.task.TaskError: Failed to execute 'datahub ingest'\n"]}
Execution finished with errors.

{\'tableau-metadata\': ["Connection: customSQLTablesConnection Error: [{\'message\': \'Showing partial results. The '
           'request exceeded the "\n'
           '                                   "20000 node limit. Use pagination, additional filtering, or both in the query to adjust results.\', '
           '\'extensions\': "\n'
           '                                   "{\'severity\': \'WARNING\', \'code\': \'NODE_LIMIT_EXCEEDED\', \'properties\': {\'nodeLimit\': '
           '20000}}}]"]},\n'

name: Upload image locally for testing (if not publishing)
uses: ishworkh/docker-image-artifact-upload@v1
        if: ${{ needs.setup.outputs.publish != 'true' }}
        with:
          image: ${{ steps.docker_meta.outputs.tags }}

Run ishworkh/docker-image-artifact-upload@v1
Error: RangeError [ERR_CHILD_PROCESS_STDIO_MAXBUFFER]: stdout maxBuffer length exceeded

'[2022-06-20 16:55:51,002] INFO     {datahub.cli.ingest_cli:99} - DataHub CLI version: 0.8.38\n'
           '[2022-06-20 16:55:54,017] INFO     {datahub.cli.ingest_cli:115} - Starting metadata ingestion\n'
           '[2022-06-20 16:55:54,017] INFO     {datahub.ingestion.source.sql.bigquery:367} - Populating lineage info via GCP audit logs\n'
           "[2022-06-20 16:55:54,021] ERROR    {datahub.ingestion.source.sql.bigquery:505} - lineage-gcp-logs => Error was ('Failed to load service "
           "account credentials from /tmp/tmp01jy7wi8', ValueError('Could not deserialize key data. The data may be in an incorrect format, it may "
           "be encrypted with an unsupported algorithm, or it may be an unsupported key type (e.g. EC curves with explicit parameters).', "
           "[_OpenSSLErrorWithText(code=503841036, lib=60, reason=524556, reason_text=b'error:1E08010C:DECODER routines::unsupported')]))\n"

acryl-datahub[dbt]==0.8.38.1rc1

test_name_to_assertion_map

elif kw_args.get("column_name"): <- in this case
logic=node.compiled_sql
if node.compiled_sql
else node.raw_sql,

logic

ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES

@Value

'Profiling - Unable to get column cardinality'

source_ref

source_url

source:
  type: datahub-business-glossary
  config:
    file: ./business_glossary/business_glossary.yml

sink:
  type: datahub-rest
  config:
    server: <http://localhost:8080>

Status(removed=True)

Ownerhip

Status(removed=False)

[2022-06-22 12:28:32,603] INFO     {datahub.cli.ingest_cli:99} - DataHub CLI version: 0.8.38\n'
           '1 validation error for DBTConfig\n'
           'test_results_path\n'
           '  extra fields not permitted (type=value_error.extra)\n',