Channels
p
Peem Warayut
02/04/2022, 11:34 AMHi Airbyte Team, i have problem
Source : Postgre
Destinations : Google Cloud Storage (GCS)
1. I created 1 job sync.
2. Run job to be created
3. Check the files at Google Cloud Storage (GCS)
4. Have Missing column
5. Check data at source : Columns missing because of All NULL data
Can you suggest a way to prevent missing column?
👀 1
a
Augustin Lafanechere (Airbyte)
02/04/2022, 2:56 PMHi @Peem Warayut could you please share your sync logs and you source and destination catalogs?
Copy code
docker run -it --rm --volume airbyte_workspace:/data busybox cat /data/<job-id>/<attempt-id>/source_catalog.json
docker run -it --rm --volume airbyte_workspace:/data busybox cat /data/<job-id>/<attempt-id>/destination_catalog.jsonp
Peem Warayut
02/09/2022, 10:11 AMMissing Field : sSuffixID, sLastSaleDate, sAuthFax, sAuthPhone, MinOrder
a
Apipoj Piasak
02/09/2022, 6:19 PMHi @Augustin Lafanechere (Airbyte) I have the same issues when export data to Google Cloud Storage in jsonl format. All the field that has all records as null value will be removed.
Here are sample data:
Copy code
| COL1 | COL2 | COL3 |
----------------------
| A | NULL | 1 |
| B | NULL | 1 |
| C | NULL | 1 |source_catalog.json
Copy code
{
"streams": [
{
"stream": {
"name": "cmssupplier",
"json_schema": {
"type": "object",
"properties": {
"poamt": {
"type": "number"
},
"rebate": {
"type": "string"
},
"sfaxno": {
"type": "string"
},
"staxid": {
"type": "string"
},
"country": {
"type": "string"
},
"prepaid": {
"type": "number"
},
"saddrno": {
"type": "string"
},
"sfaxext": {
"type": "string"
},
"sfaxoth": {
"type": "string"
},
"sstatus": {
"type": "string"
},
"altpayee": {
"type": "string"
},
"bankacct": {
"type": "string"
},
"bankname": {
"type": "string"
},
"branchid": {
"type": "string"
},
"createby": {
"type": "string"
},
"createon": {
"type": "string"
},
"currcode": {
"type": "string"
},
"isimport": {
"type": "string"
},
"leadtime": {
"type": "number"
},
"minorder": {
"type": "number"
},
"saddrsoi": {
"type": "string"
},
"sauthfax": {
"type": "string"
},
"sengname": {
"type": "string"
},
"sfaxarea": {
"type": "string"
},
"sfaxcomm": {
"type": "string"
},
"sphoneno": {
"type": "string"
},
"sreftype": {
"type": "string"
},
"stitleid": {
"type": "string"
},
"svatrate": {
"type": "number"
},
"swebsite": {
"type": "string"
},
"updateby": {
"type": "string"
},
"updateon": {
"type": "string"
},
"saddrroad": {
"type": "string"
},
"sauthcomm": {
"type": "string"
},
"sauthname": {
"type": "string"
},
"sinvaddr1": {
"type": "string"
},
"sinvaddr2": {
"type": "string"
},
"sinvaddr3": {
"type": "string"
},
"sinvaddr4": {
"type": "string"
},
"sinvaddr5": {
"type": "string"
},
"smanageid": {
"type": "string"
},
"sphoneext": {
"type": "string"
},
"sphoneno1": {
"type": "string"
},
"sphoneno2": {
"type": "string"
},
"sphoneoth": {
"type": "string"
},
"ssuffixid": {
"type": "string"
},
"lockreason": {
"type": "string"
},
"lockstatus": {
"type": "string"
},
"saddrfloor": {
"type": "string"
},
"sauthphone": {
"type": "string"
},
"semailaddr": {
"type": "string"
},
"sloadaddr1": {
"type": "string"
},
"sloadaddr2": {
"type": "string"
},
"sloadaddr3": {
"type": "string"
},
"sloadaddr4": {
"type": "string"
},
"sloadfaxno": {
"type": "string"
},
"slocalname": {
"type": "string"
},
"sphonearea": {
"type": "string"
},
"ssuppgrpid": {
"type": "string"
},
"stitlename": {
"type": "string"
},
"creditlimit": {
"type": "number"
},
"saddramphur": {
"type": "string"
},
"saddrtumbon": {
"type": "string"
},
"sphonerefno": {
"type": "string"
},
"ssuffixname": {
"type": "string"
},
"ssupplierid": {
"type": "string"
},
"bankaccttype": {
"type": "string"
},
"creditremark": {
"type": "string"
},
"rebateremark": {
"type": "string"
},
"saddrvillage": {
"type": "string"
},
"saddrzipcode": {
"type": "string"
},
"sloadzipcode": {
"type": "string"
},
"spaymentcode": {
"type": "string"
},
"spaymenttype": {
"type": "string"
},
"saddrbuilding": {
"type": "string"
},
"saddrprovince": {
"type": "string"
},
"sauthphoneext": {
"type": "string"
},
"sauthposition": {
"type": "string"
},
"slastsaledate": {
"type": "string"
},
"sloadphoneno1": {
"type": "string"
},
"sloadphoneno2": {
"type": "string"
},
"ssupplieridod": {
"type": "string"
},
"returncriteria": {
"type": "string"
},
"sauthemailaddr": {
"type": "string"
},
"sdiscountrate1": {
"type": "number"
},
"sdiscountrate2": {
"type": "number"
},
"sdiscountrate3": {
"type": "number"
},
"sfirstsaledate": {
"type": "string"
},
"ssupplieridb2s": {
"type": "string"
},
"bankaccountname": {
"type": "string"
},
"consignmentflag": {
"type": "string"
},
"sbusinesstypeid": {
"type": "string"
},
"bankguaranteeamt": {
"type": "number"
},
"deliverylocation": {
"type": "string"
},
"_airbyte_emitted_at": {
"type": "string"
},
"aptrancontrolnature": {
"type": "string"
},
"bankguaranteeremark": {
"type": "string"
},
"aptrancontrolactcode": {
"type": "string"
},
"bankguaranteeexpdate": {
"type": "string"
},
"returncriteriaremark": {
"type": "string"
},
"deliverylocationremark": {
"type": "string"
},
"_airbyte_cmssupplier_hashid": {
"type": "string"
}
}
},
"supported_sync_modes": [
"full_refresh",
"incremental"
],
"default_cursor_field": [],
"source_defined_primary_key": [],
"namespace": "officemate"
},
"sync_mode": "full_refresh",
"cursor_field": [
"updateon"
],
"destination_sync_mode": "append",
"primary_key": []
}
]
} Copy code
{
"streams": [
{
"stream": {
"name": "cmssupplier",
"json_schema": {
"type": "object",
"properties": {
"poamt": {
"type": "number"
},
"rebate": {
"type": "string"
},
"sfaxno": {
"type": "string"
},
"staxid": {
"type": "string"
},
"country": {
"type": "string"
},
"prepaid": {
"type": "number"
},
"saddrno": {
"type": "string"
},
"sfaxext": {
"type": "string"
},
"sfaxoth": {
"type": "string"
},
"sstatus": {
"type": "string"
},
"altpayee": {
"type": "string"
},
"bankacct": {
"type": "string"
},
"bankname": {
"type": "string"
},
"branchid": {
"type": "string"
},
"createby": {
"type": "string"
},
"createon": {
"type": "string"
},
"currcode": {
"type": "string"
},
"isimport": {
"type": "string"
},
"leadtime": {
"type": "number"
},
"minorder": {
"type": "number"
},
"saddrsoi": {
"type": "string"
},
"sauthfax": {
"type": "string"
},
"sengname": {
"type": "string"
},
"sfaxarea": {
"type": "string"
},
"sfaxcomm": {
"type": "string"
},
"sphoneno": {
"type": "string"
},
"sreftype": {
"type": "string"
},
"stitleid": {
"type": "string"
},
"svatrate": {
"type": "number"
},
"swebsite": {
"type": "string"
},
"updateby": {
"type": "string"
},
"updateon": {
"type": "string"
},
"saddrroad": {
"type": "string"
},
"sauthcomm": {
"type": "string"
},
"sauthname": {
"type": "string"
},
"sinvaddr1": {
"type": "string"
},
"sinvaddr2": {
"type": "string"
},
"sinvaddr3": {
"type": "string"
},
"sinvaddr4": {
"type": "string"
},
"sinvaddr5": {
"type": "string"
},
"smanageid": {
"type": "string"
},
"sphoneext": {
"type": "string"
},
"sphoneno1": {
"type": "string"
},
"sphoneno2": {
"type": "string"
},
"sphoneoth": {
"type": "string"
},
"ssuffixid": {
"type": "string"
},
"lockreason": {
"type": "string"
},
"lockstatus": {
"type": "string"
},
"saddrfloor": {
"type": "string"
},
"sauthphone": {
"type": "string"
},
"semailaddr": {
"type": "string"
},
"sloadaddr1": {
"type": "string"
},
"sloadaddr2": {
"type": "string"
},
"sloadaddr3": {
"type": "string"
},
"sloadaddr4": {
"type": "string"
},
"sloadfaxno": {
"type": "string"
},
"slocalname": {
"type": "string"
},
"sphonearea": {
"type": "string"
},
"ssuppgrpid": {
"type": "string"
},
"stitlename": {
"type": "string"
},
"creditlimit": {
"type": "number"
},
"saddramphur": {
"type": "string"
},
"saddrtumbon": {
"type": "string"
},
"sphonerefno": {
"type": "string"
},
"ssuffixname": {
"type": "string"
},
"ssupplierid": {
"type": "string"
},
"bankaccttype": {
"type": "string"
},
"creditremark": {
"type": "string"
},
"rebateremark": {
"type": "string"
},
"saddrvillage": {
"type": "string"
},
"saddrzipcode": {
"type": "string"
},
"sloadzipcode": {
"type": "string"
},
"spaymentcode": {
"type": "string"
},
"spaymenttype": {
"type": "string"
},
"saddrbuilding": {
"type": "string"
},
"saddrprovince": {
"type": "string"
},
"sauthphoneext": {
"type": "string"
},
"sauthposition": {
"type": "string"
},
"slastsaledate": {
"type": "string"
},
"sloadphoneno1": {
"type": "string"
},
"sloadphoneno2": {
"type": "string"
},
"ssupplieridod": {
"type": "string"
},
"returncriteria": {
"type": "string"
},
"sauthemailaddr": {
"type": "string"
},
"sdiscountrate1": {
"type": "number"
},
"sdiscountrate2": {
"type": "number"
},
"sdiscountrate3": {
"type": "number"
},
"sfirstsaledate": {
"type": "string"
},
"ssupplieridb2s": {
"type": "string"
},
"bankaccountname": {
"type": "string"
},
"consignmentflag": {
"type": "string"
},
"sbusinesstypeid": {
"type": "string"
},
"bankguaranteeamt": {
"type": "number"
},
"deliverylocation": {
"type": "string"
},
"_airbyte_emitted_at": {
"type": "string"
},
"aptrancontrolnature": {
"type": "string"
},
"bankguaranteeremark": {
"type": "string"
},
"aptrancontrolactcode": {
"type": "string"
},
"bankguaranteeexpdate": {
"type": "string"
},
"returncriteriaremark": {
"type": "string"
},
"deliverylocationremark": {
"type": "string"
},
"_airbyte_cmssupplier_hashid": {
"type": "string"
}
}
},
"supported_sync_modes": [
"full_refresh",
"incremental"
],
"default_cursor_field": [],
"source_defined_primary_key": []
},
"sync_mode": "full_refresh",
"cursor_field": [
"updateon"
],
"destination_sync_mode": "append",
"primary_key": []
}
]
}And here is the log file,
Already submit the github issues https://github.com/airbytehq/airbyte/issues/10230