great-beard-50720
06/08/2022, 11:07 AM...
def get_suite(context: ge.DataContext, suite_name:str) -> ExpectationSuite:
try:
suite = context.get_expectation_suite(expectation_suite_name=suite_name)
print(f'Loaded ExpectationSuite "{suite.expectation_suite_name}" containing {len(suite.expectations)} expectations.')
except DataContextError:
suite = context.create_expectation_suite(expectation_suite_name=suite_name)
print(f'Created ExpectationSuite "{suite.expectation_suite_name}".')
return suite
context_name = 'athena_backoffice_dev'
default_bucket = 'mosaic-backoffice'
conn_string = m_athena.get_athena_conn_string(context_name)
context_config = m_ctxt.get_context_config(context_name, conn_string, default_bucket)
context = m_ctxt.get_context(context_config)
expectation_suite_name = 'demo_suite'
expectation_suite = get_suite(context, expectation_suite_name)
batch_request = {
'datasource_name': 'athena_backoffice_dev',
'data_connector_name': 'default_inferred_data_connector_name',
'data_asset_name': 'borrowing_base.accruals',
'limit': 1000
}
validator = context.get_validator(
batch_request=BatchRequest(**batch_request),
expectation_suite_name=expectation_suite_name
)
column_names = [f'"{column_name}"' for column_name in validator.columns()]
print(f"Columns: {', '.join(column_names)}.")
h = validator.head(n_rows=5, fetch_all=False)
print(h)
When I run that, the following error is returned:
(venv) C:\Dev\great-expectations>python main.py
Loaded ExpectationSuite "demo_suite" containing 0 expectations.
Calculating Metrics: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 8.67it/s]
Columns: "trade_date", "trade_strategy", "invoice_id", "trade_id", "internal_legal_entity", "counter_party", "product", "instrument_type", "buy_sell", "native_mtm", "settle_currency", "mtm_usd", "notional_amount", "quantity", "uom", "asset_value_usd", "due_date", "days_past_due", "rating", "tier", "overrides", "enhancement_method", "is_past_due", "source_system", "is_excluded", "exclusion_reason", "hartree_id", "legal_name", "governing_law", "country", "counterparty_name", "lending_facility", "valuation_date", "run_id".
Calculating Metrics: 0%| | 0/1 [00:01<?, ?it/s]
Exceptions
{('table.head', 'batch_id=89c03fce198c1b7a63893b895140eb28', '04166707abe073177c1dd922d3584468'): {'metric_configuration': {
"metric_name": "table.head",
"metric_domain_kwargs": {
"batch_id": "89c03fce198c1b7a63893b895140eb28"
},
"metric_domain_kwargs_id": "batch_id=89c03fce198c1b7a63893b895140eb28",
"metric_value_kwargs": {
"n_rows": 5,
"fetch_all": false
},
"metric_value_kwargs_id": "04166707abe073177c1dd922d3584468",
"id": [
"table.head",
"batch_id=89c03fce198c1b7a63893b895140eb28",
"04166707abe073177c1dd922d3584468"
]
}, 'num_failures': 3, 'exception_info': {{'exception_traceback': 'Traceback (most recent call last):\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\great_expectations\\execution_engine\\execution_engine.py", line 387, in resolve_metrics\n **metric_provider_kwargs\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\great_expectations\\expectations\\metrics\\metric_provider.py", line 34, in inner_func\n return metric_fn(*args, **kwargs)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\great_expectations\\expectations\\metrics\\table_metrics\\table_head.py", line 132, in _sqlalchemy\n compile_kwargs={"literal_binds": True},\n File "<string>", line 1, in <lambda>\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\elements.py", line 481, in compile\n return self._compiler(dialect, bind=bind, **kw)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\elements.py", line 487, in _compiler\n return dialect.statement_compiler(dialect, self, **kw)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\compiler.py", line 592, in __init__\n Compiled.__init__(self, dialect, statement, **kwargs)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\compiler.py", line 322, in __init__\n self.string = self.process(self.statement, **compile_kwargs)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\compiler.py", line 352, in process\n return obj._compiler_dispatch(self, **kwargs)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\visitors.py", line 96, in _compiler_dispatch\n return meth(self, **kw)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\compiler.py", line 2202, in visit_select\n text, select, inner_columns, froms, byfrom, kwargs\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\sqlalchemy\\sql\\compiler.py", line 2320, in _compose_select_body\n text += self.limit_clause(select, **kwargs)\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\pyathena\\sqlalchemy_athena.py", line 90, in limit_clause\n if limit_clause is not None and select._simple_int_clause(limit_clause):\nAttributeError: \'Select\' object has no attribute \'_simple_int_clause\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\great_expectations\\validator\\validator.py", line 1291, in resolve_validation_graph\n runtime_configuration=runtime_configuration,\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\great_expectations\\validator\\validator.py", line 2202, in _resolve_metrics\n runtime_configuration=runtime_configuration,\n File "C:\\Dev\\great-expectations\\venv\\lib\\site-packages\\great_expectations\\execution_engine\\execution_engine.py", line 391, in resolve_metrics\n message=str(e), failed_metrics=(metric_to_resolve,)\ngreat_expectations.exceptions.exceptions.MetricResolutionError: \'Select\' object has no attribute \'_simple_int_clause\'\n', 'exception_message': "'Select' object has no attribute '_simple_int_clause'", 'raised_exception': True}}}}
occurred while resolving metrics.
Traceback (most recent call last):
File "main.py", line 81, in <module>
h = validator.head(n_rows=5, fetch_all=False)
File "C:\Dev\great-expectations\venv\lib\site-packages\great_expectations\validator\validator.py", line 2145, in head
"fetch_all": fetch_all,
File "C:\Dev\great-expectations\venv\lib\site-packages\great_expectations\validator\validator.py", line 891, in get_metric
return self.get_metrics(metrics={metric.metric_name: metric})[
File "C:\Dev\great-expectations\venv\lib\site-packages\great_expectations\validator\validator.py", line 858, in get_metrics
for metric_configuration in metrics.values()
File "C:\Dev\great-expectations\venv\lib\site-packages\great_expectations\validator\validator.py", line 858, in <dictcomp>
for metric_configuration in metrics.values()
KeyError: ('table.head', 'batch_id=89c03fce198c1b7a63893b895140eb28', '04166707abe073177c1dd922d3584468')
That is very similar to what I am seeing here: https://github.com/apache/superset/issues/20168, but I am not sure how that helps me.
Has anyone else seen this before?big-carpet-38439
06/08/2022, 2:05 PMbig-carpet-38439
06/08/2022, 2:05 PMgreat-beard-50720
06/08/2022, 2:05 PM