Hi Airbyte Team, i have problem
Source : Postgre
...
# ask-community-for-troubleshootingp
Peem Warayut
02/04/2022, 11:34 AMHi Airbyte Team, i have problem
Source : Postgre
Destinations : Google Cloud Storage (GCS)
1. I created 1 job sync.
2. Run job to be created
3. Check the files at Google Cloud Storage (GCS)
4. Have Missing column
5. Check data at source : Columns missing because of All NULL data
Can you suggest a way to prevent missing column?
👀 1
a
Augustin Lafanechere (Airbyte)
02/04/2022, 2:56 PMHi @Peem Warayut could you please share your sync logs and you source and destination catalogs?
Augustin Lafanechere (Airbyte)
02/04/2022, 2:56 PM Copy code
docker run -it --rm --volume airbyte_workspace:/data busybox cat /data/<job-id>/<attempt-id>/source_catalog.json
docker run -it --rm --volume airbyte_workspace:/data busybox cat /data/<job-id>/<attempt-id>/destination_catalog.jsonp
Peem Warayut
02/09/2022, 10:11 AMMissing Field : sSuffixID, sLastSaleDate, sAuthFax, sAuthPhone, MinOrder
a
Apipoj Piasak
02/09/2022, 6:19 PMHi @Augustin Lafanechere (Airbyte) I have the same issues when export data to Google Cloud Storage in jsonl format. All the field that has all records as null value will be removed.
Here are sample data:
Copy code
| COL1 | COL2 | COL3 |
----------------------
| A | NULL | 1 |
| B | NULL | 1 |
| C | NULL | 1 |Apipoj Piasak
02/09/2022, 6:38 PMsource_catalog.json
Copy code
{
"streams": [
{
"stream": {
"name": "cmssupplier",
"json_schema": {
"type": "object",
"properties": {
"poamt": {
"type": "number"
},
"rebate": {
"type": "string"
},
"sfaxno": {
"type": "string"
},
"staxid": {
"type": "string"
},
"country": {
"type": "string"
},
"prepaid": {
"type": "number"
},
"saddrno": {
"type": "string"
},
"sfaxext": {
"type": "string"
},
"sfaxoth": {
"type": "string"
},
"sstatus": {
"type": "string"
},
"altpayee": {
"type": "string"
},
"bankacct": {
"type": "string"
},
"bankname": {
"type": "string"
},
"branchid": {
"type": "string"
},
"createby": {
"type": "string"
},
"createon": {
"type": "string"
},
"currcode": {
"type": "string"
},
"isimport": {
"type": "string"
},
"leadtime": {
"type": "number"
},
"minorder": {
"type": "number"
},
"saddrsoi": {
"type": "string"
},
"sauthfax": {
"type": "string"
},
"sengname": {
"type": "string"
},
"sfaxarea": {
"type": "string"
},
"sfaxcomm": {
"type": "string"
},
"sphoneno": {
"type": "string"
},
"sreftype": {
"type": "string"
},
"stitleid": {
"type": "string"
},
"svatrate": {
"type": "number"
},
"swebsite": {
"type": "string"
},
"updateby": {
"type": "string"
},
"updateon": {
"type": "string"
},
"saddrroad": {
"type": "string"
},
"sauthcomm": {
"type": "string"
},
"sauthname": {
"type": "string"
},
"sinvaddr1": {
"type": "string"
},
"sinvaddr2": {
"type": "string"
},
"sinvaddr3": {
"type": "string"
},
"sinvaddr4": {
"type": "string"
},
"sinvaddr5": {
"type": "string"
},
"smanageid": {
"type": "string"
},
"sphoneext": {
"type": "string"
},
"sphoneno1": {
"type": "string"
},
"sphoneno2": {
"type": "string"
},
"sphoneoth": {
"type": "string"
},
"ssuffixid": {
"type": "string"
},
"lockreason": {
"type": "string"
},
"lockstatus": {
"type": "string"
},
"saddrfloor": {
"type": "string"
},
"sauthphone": {
"type": "string"
},
"semailaddr": {
"type": "string"
},
"sloadaddr1": {
"type": "string"
},
"sloadaddr2": {
"type": "string"
},
"sloadaddr3": {
"type": "string"
},
"sloadaddr4": {
"type": "string"
},
"sloadfaxno": {
"type": "string"
},
"slocalname": {
"type": "string"
},
"sphonearea": {
"type": "string"
},
"ssuppgrpid": {
"type": "string"
},
"stitlename": {
"type": "string"
},
"creditlimit": {
"type": "number"
},
"saddramphur": {
"type": "string"
},
"saddrtumbon": {
"type": "string"
},
"sphonerefno": {
"type": "string"
},
"ssuffixname": {
"type": "string"
},
"ssupplierid": {
"type": "string"
},
"bankaccttype": {
"type": "string"
},
"creditremark": {
"type": "string"
},
"rebateremark": {
"type": "string"
},
"saddrvillage": {
"type": "string"
},
"saddrzipcode": {
"type": "string"
},
"sloadzipcode": {
"type": "string"
},
"spaymentcode": {
"type": "string"
},
"spaymenttype": {
"type": "string"
},
"saddrbuilding": {
"type": "string"
},
"saddrprovince": {
"type": "string"
},
"sauthphoneext": {
"type": "string"
},
"sauthposition": {
"type": "string"
},
"slastsaledate": {
"type": "string"
},
"sloadphoneno1": {
"type": "string"
},
"sloadphoneno2": {
"type": "string"
},
"ssupplieridod": {
"type": "string"
},
"returncriteria": {
"type": "string"
},
"sauthemailaddr": {
"type": "string"
},
"sdiscountrate1": {
"type": "number"
},
"sdiscountrate2": {
"type": "number"
},
"sdiscountrate3": {
"type": "number"
},
"sfirstsaledate": {
"type": "string"
},
"ssupplieridb2s": {
"type": "string"
},
"bankaccountname": {
"type": "string"
},
"consignmentflag": {
"type": "string"
},
"sbusinesstypeid": {
"type": "string"
},
"bankguaranteeamt": {
"type": "number"
},
"deliverylocation": {
"type": "string"
},
"_airbyte_emitted_at": {
"type": "string"
},
"aptrancontrolnature": {
"type": "string"
},
"bankguaranteeremark": {
"type": "string"
},
"aptrancontrolactcode": {
"type": "string"
},
"bankguaranteeexpdate": {
"type": "string"
},
"returncriteriaremark": {
"type": "string"
},
"deliverylocationremark": {
"type": "string"
},
"_airbyte_cmssupplier_hashid": {
"type": "string"
}
}
},
"supported_sync_modes": [
"full_refresh",
"incremental"
],
"default_cursor_field": [],
"source_defined_primary_key": [],
"namespace": "officemate"
},
"sync_mode": "full_refresh",
"cursor_field": [
"updateon"
],
"destination_sync_mode": "append",
"primary_key": []
}
]
} Copy code
{
"streams": [
{
"stream": {
"name": "cmssupplier",
"json_schema": {
"type": "object",
"properties": {
"poamt": {
"type": "number"
},
"rebate": {
"type": "string"
},
"sfaxno": {
"type": "string"
},
"staxid": {
"type": "string"
},
"country": {
"type": "string"
},
"prepaid": {
"type": "number"
},
"saddrno": {
"type": "string"
},
"sfaxext": {
"type": "string"
},
"sfaxoth": {
"type": "string"
},
"sstatus": {
"type": "string"
},
"altpayee": {
"type": "string"
},
"bankacct": {
"type": "string"
},
"bankname": {
"type": "string"
},
"branchid": {
"type": "string"
},
"createby": {
"type": "string"
},
"createon": {
"type": "string"
},
"currcode": {
"type": "string"
},
"isimport": {
"type": "string"
},
"leadtime": {
"type": "number"
},
"minorder": {
"type": "number"
},
"saddrsoi": {
"type": "string"
},
"sauthfax": {
"type": "string"
},
"sengname": {
"type": "string"
},
"sfaxarea": {
"type": "string"
},
"sfaxcomm": {
"type": "string"
},
"sphoneno": {
"type": "string"
},
"sreftype": {
"type": "string"
},
"stitleid": {
"type": "string"
},
"svatrate": {
"type": "number"
},
"swebsite": {
"type": "string"
},
"updateby": {
"type": "string"
},
"updateon": {
"type": "string"
},
"saddrroad": {
"type": "string"
},
"sauthcomm": {
"type": "string"
},
"sauthname": {
"type": "string"
},
"sinvaddr1": {
"type": "string"
},
"sinvaddr2": {
"type": "string"
},
"sinvaddr3": {
"type": "string"
},
"sinvaddr4": {
"type": "string"
},
"sinvaddr5": {
"type": "string"
},
"smanageid": {
"type": "string"
},
"sphoneext": {
"type": "string"
},
"sphoneno1": {
"type": "string"
},
"sphoneno2": {
"type": "string"
},
"sphoneoth": {
"type": "string"
},
"ssuffixid": {
"type": "string"
},
"lockreason": {
"type": "string"
},
"lockstatus": {
"type": "string"
},
"saddrfloor": {
"type": "string"
},
"sauthphone": {
"type": "string"
},
"semailaddr": {
"type": "string"
},
"sloadaddr1": {
"type": "string"
},
"sloadaddr2": {
"type": "string"
},
"sloadaddr3": {
"type": "string"
},
"sloadaddr4": {
"type": "string"
},
"sloadfaxno": {
"type": "string"
},
"slocalname": {
"type": "string"
},
"sphonearea": {
"type": "string"
},
"ssuppgrpid": {
"type": "string"
},
"stitlename": {
"type": "string"
},
"creditlimit": {
"type": "number"
},
"saddramphur": {
"type": "string"
},
"saddrtumbon": {
"type": "string"
},
"sphonerefno": {
"type": "string"
},
"ssuffixname": {
"type": "string"
},
"ssupplierid": {
"type": "string"
},
"bankaccttype": {
"type": "string"
},
"creditremark": {
"type": "string"
},
"rebateremark": {
"type": "string"
},
"saddrvillage": {
"type": "string"
},
"saddrzipcode": {
"type": "string"
},
"sloadzipcode": {
"type": "string"
},
"spaymentcode": {
"type": "string"
},
"spaymenttype": {
"type": "string"
},
"saddrbuilding": {
"type": "string"
},
"saddrprovince": {
"type": "string"
},
"sauthphoneext": {
"type": "string"
},
"sauthposition": {
"type": "string"
},
"slastsaledate": {
"type": "string"
},
"sloadphoneno1": {
"type": "string"
},
"sloadphoneno2": {
"type": "string"
},
"ssupplieridod": {
"type": "string"
},
"returncriteria": {
"type": "string"
},
"sauthemailaddr": {
"type": "string"
},
"sdiscountrate1": {
"type": "number"
},
"sdiscountrate2": {
"type": "number"
},
"sdiscountrate3": {
"type": "number"
},
"sfirstsaledate": {
"type": "string"
},
"ssupplieridb2s": {
"type": "string"
},
"bankaccountname": {
"type": "string"
},
"consignmentflag": {
"type": "string"
},
"sbusinesstypeid": {
"type": "string"
},
"bankguaranteeamt": {
"type": "number"
},
"deliverylocation": {
"type": "string"
},
"_airbyte_emitted_at": {
"type": "string"
},
"aptrancontrolnature": {
"type": "string"
},
"bankguaranteeremark": {
"type": "string"
},
"aptrancontrolactcode": {
"type": "string"
},
"bankguaranteeexpdate": {
"type": "string"
},
"returncriteriaremark": {
"type": "string"
},
"deliverylocationremark": {
"type": "string"
},
"_airbyte_cmssupplier_hashid": {
"type": "string"
}
}
},
"supported_sync_modes": [
"full_refresh",
"incremental"
],
"default_cursor_field": [],
"source_defined_primary_key": []
},
"sync_mode": "full_refresh",
"cursor_field": [
"updateon"
],
"destination_sync_mode": "append",
"primary_key": []
}
]
}Apipoj Piasak
02/09/2022, 6:53 PMAnd here is the log file,
Apipoj Piasak
02/09/2022, 7:15 PMAlready submit the github issues https://github.com/airbytehq/airbyte/issues/10230