Hello All, I was trying to extract data from Klaviyo and ingest in Azure Databricks and the sync has failed with the syntax error. What I noticed was few columns were included in create table statement without data type which resulted in the syntax error. Here is the stack trace-
**********************************************************************
2023-01-25 17
3149 2023-01-25 12
0149 INFO i.a.w.t.s.ReplicationActivityImpl(lambda$replicate$3):209 - sync summary: io.airbyte.config.StandardSyncOutput@7be643fc[standardSyncSummary=io.airbyte.config.StandardSyncSummary@2c7ec77[status=failed,recordsSynced=41841,bytesSynced=60626678,startTime=1674647748084,endTime=1674648109925,totalStats=io.airbyte.config.SyncStats@7108bc3b[bytesEmitted=60626678,destinationStateMessagesEmitted=0,destinationWriteEndTime=1674648109924,destinationWriteStartTime=1674647748168,estimatedBytes=<null>,estimatedRecords=<null>,meanSecondsBeforeSourceStateMessageEmitted=0,maxSecondsBeforeSourceStateMessageEmitted=0,maxSecondsBetweenStateMessageEmittedandCommitted=0,meanSecondsBetweenStateMessageEmittedandCommitted=0,recordsEmitted=41841,recordsCommitted=0,replicationEndTime=1674648109925,replicationStartTime=1674647748084,sourceReadEndTime=1674648076066,sourceReadStartTime=1674647748126,sourceStateMessagesEmitted=0,additionalProperties={}],streamStats=[io.airbyte.config.StreamSyncStats@22fbe853[streamName=campaigns,streamNamespace=<null>,stats=io.airbyte.config.SyncStats@6bcba653[bytesEmitted=118350,destinationStateMessagesEmitted=<null>,destinationWriteEndTime=<null>,destinationWriteStartTime=<null>,estimatedBytes=<null>,estimatedRecords=<null>,meanSecondsBeforeSourceStateMessageEmitted=<null>,maxSecondsBeforeSourceStateMessageEmitted=<null>,maxSecondsBetweenStateMessageEmittedandCommitted=<null>,meanSecondsBetweenStateMessageEmittedandCommitted=<null>,recordsEmitted=95,recordsCommitted=<null>,replicationEndTime=<null>,replicationStartTime=<null>,sourceReadEndTime=<null>,sourceReadStartTime=<null>,sourceStateMessagesEmitted=<null>,additionalProperties={}]], io.airbyte.config.StreamSyncStats@56525b09[streamName=events,streamNamespace=<null>,stats=io.airbyte.config.SyncStats@620fdc9c[bytesEmitted=60508328,destinationStateMessagesEmitted=<null>,destinationWriteEndTime=<null>,destinationWriteStartTime=<null>,estimatedBytes=<null>,estimatedRecords=<null>,meanSecondsBeforeSourceStateMessageEmitted=<null>,maxSecondsBeforeSourceStateMessageEmitted=<null>,maxSecondsBetweenStateMessageEmittedandCommitted=<null>,meanSecondsBetweenStateMessageEmittedandCommitted=<null>,recordsEmitted=41746,recordsCommitted=<null>,replicationEndTime=<null>,replicationStartTime=<null>,sourceReadEndTime=<null>,sourceReadStartTime=<null>,sourceStateMessagesEmitted=<null>,additionalProperties={}]]]],normalizationSummary=<null>,webhookOperationSummary=<null>,state=<null>,outputCatalog=io.airbyte.protocol.models.ConfiguredAirbyteCatalog@1fcb09fe[streams=[io.airbyte.protocol.models.ConfiguredAirbyteStream@2904dd6a[stream=io.airbyte.protocol.models.AirbyteStream@6d4a5150[name=campaigns,jsonSchema={"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"lists":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"folder":{"type":["null","string"]},"object":{"type":"string"},"created":{"type":"string","format":"date-time"},"updated":{"type":"string","format":"date-time"},"list_type":{"type":"string"},"person_count":{"type":"integer"}}}},"object":{"type":"string"},"status":{"type":"string"},"created":{"type":["null","string"],"format":"date-time"},"sent_at":{"type":["null","string"],"format":"date-time"},"subject":{"type":["null","string"]},"updated":{"type":["null","string"],"format":"date-time"},"from_name":{"type":"string"},"send_time":{"type":["null","string"],"format":"date-time"},"status_id":{"type":"integer"},"from_email":{"type":"string"},"template_id":{"type":["null","string"]},"is_segmented":{"type":"boolean"},"message_type":{"type":"string"},"status_label":{"type":"string"},"campaign_type":{"type":"string"},"excluded_lists":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"folder":{"type":["null","string"]},"object":{"type":"string"},"created":{"type":"string","format":"date-time"},"updated":{"type":"string","format":"date-time"},"list_type":{"type":"string"},"person_count":{"type":"integer"}}}},"num_recipients":{"type":"integer"}}},supportedSyncModes=[full_refresh],sourceDefinedCursor=<null>,defaultCursorField=[],sourceDefinedPrimaryKey=[[id]],namespace=<null>,additionalProperties={}],syncMode=full_refresh,cursorField=[],destinationSyncMode=overwrite,primaryKey=[[id]],additionalProperties={}], io.airbyte.protocol.models.ConfiguredAirbyteStream@713d8b59[stream=io.airbyte.protocol.models.AirbyteStream@367ee879[name=events,jsonSchema={"type":"object","properties":{"id":{"type":"string"},"uuid":{"type":"string"},"object":{"type":"string"},"person":{"type":"object","properties":{"id":{"type":"string"},"$email":{"type":"string"},"object":{"type":"string"}}},"flow_id":{"type":["null","string"]},"datetime":{"type":"string"},"timestamp":{"type":"integer"},"event_name":{"type":"string"},"campaign_id":{"type":["null","string"]},"statistic_id":{"type":"string"},"flow_message_id":{"type":["null","string"]},"event_properties":{"type":"object","properties":{"items":{"type":"array","items":{"type":"object","properties":{"sku":{"type":"string"},"name":{"type":"string"},"price":{"type":"integer"},"object":{"type":"string"},"quantity":{"type":"integer"}}}},"$value":{"type":"number"}}}}},supportedSyncModes=[full_refresh, incremental],sourceDefinedCursor=true,defaultCursorField=[timestamp],sourceDefinedPrimaryKey=[[id]],namespace=<null>,additionalProperties={}],syncMode=full_refresh,cursorField=[timestamp],destinationSyncMode=overwrite,primaryKey=[[id]],additionalProperties={}]],additionalProperties={}],failures=[io.airbyte.config.FailureReason@391e8c51[failureOrigin=destination,failureType=system_error,internalMessage=java.sql.SQLException: [Databricks][DatabricksJDBCDriver](500051) ERROR processing query/statement. Error Code: 0, SQL state: org.apache.hive.service.cli.HiveSQLException: Error running query: [PARSE_SYNTAX_ERROR] org.apache.spark.sql.catalyst.parser.ParseException:
2023-01-25 17
3149 [PARSE_SYNTAX_ERROR] Syntax error at or near ',': extra input ','(line 1, pos 149)
2023-01-25 17
3149
2023-01-25 17
3149 == SQL ==
2023-01-25 17
3149 CREATE TABLE platformdbtest_sandbox._airbyte_tmp_ihi_campaigns (_airbyte_ab_id string, _airbyte_emitted_at string,
campaign_type
string,
created
,
excluded_lists
array,
from_email
string,
from_name
string,
id
string,
is_segmented
boolean,
lists
array,
message_type
string,
name
string,
num_recipients
integer,
object
string,
send_time
,
sent_at
,
status
string,
status_id
integer,
status_label
string,
subject
,
template_id
,
updated
) USING csv LOCATION 'abfss:
REDACTED_LOCAL_PART@sxsdc.dfs.core.windows.net/1674e209-94ae-4f1b-af7c-f9de6cb8fcf4/platformdbtest_sandbox/_airbyte_tmp_ihi_campaigns/' options ("header" = "true", "multiLine" = "true")
2023-01-25 17
3149 -----------------------------------------------------------------------------------------------------------------------------------------------------^^^
2023-01-25 17
3149
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.HiveThriftServerErrors$.runningQueryError(HiveThriftServerErrors.scala:48)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.$anonfun$execute$1(SparkExecuteStatementOperation.scala:585)
2023-01-25 17
3149 at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
2023-01-25 17
3149 at com.databricks.unity.UCSEphemeralState$Handle.runWith(UCSEphemeralState.scala:41)
2023-01-25 17
3149 at com.databricks.unity.HandleImpl.runWith(UCSHandle.scala:99)
2023-01-25 17
3149 at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:484)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$2(SparkExecuteStatementOperation.scala:353)
2023-01-25 17
3149 at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.ThriftLocalProperties.withLocalProperties(ThriftLocalProperties.scala:149)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.ThriftLocalProperties.withLocalProperties$(ThriftLocalProperties.scala:49)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:60)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:331)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:316)
2023-01-25 17
3149 at java.security.AccessController.doPrivileged(Native Method)
2023-01-25 17
3149 at javax.security.auth.Subject.doAs(Subject.java:422)
2023-01-25 17
3149 at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878)
2023-01-25 17
3149 at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:365)
2023-01-25 17
3149 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
2023-01-25 17
3149 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
2023-01-25 17
3149 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
2023-01-25 17
3149 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
2023-01-25 17
3149 at java.lang.Thread.run(Thread.java:750)
2023-01-25 17
3149 Caused by: org.apache.spark.sql.catalyst.parser.ParseException:
2023-01-25 17
3149 [PARSE_SYNTAX_ERROR] Syntax error at or near ',': extra input ','(line 1, pos 149)
2023-01-25 17
3149
*************************************************************************
Is there any open bug related to Klaviyo source connector or did I miss any configuration that resulted in the syntax error?