https://linen.dev logo
Join Slack
Powered by
Hi all, I created my own connector for apple searc...
# help-connector-development
s
Hi all, I created my own connector for apple search ads reports, everything works well in the main table and the data arrives correctly, but! since i have 2 objects returned in the JSON and my schema is mapping them, I have 2 extra tables (that I could ignore, but still want to makes things correctly and understand the way airbyte works) my record looks like that:
Copy code
{
  "type": "RECORD",
  "record": {
    "stream": "campaigns",
    "data": {
      "campaignId": 1043888667,
      "startTime": "2022-11-25",
      "primary_key": "2a5ab6e9b247d79699392f1a6dd87150",
      "metadata": {
        "campaignId": 1043888667,
        "campaignName": "TP_US_generic",
        "deleted": false,
        "campaignStatus": "ENABLED",
        "app_appName": "Asana Rebel: Get in Shape",
        "app_adamId": 1067860796,
        "servingStatus": "RUNNING",
        "servingStateReasons": null,
        "countriesOrRegions": [
          "US"
        ],
        "modificationTime": "2022-05-02T14:25:17.001",
        "totalBudget_amount": "100000",
        "totalBudget_currency": "EUR",
        "dailyBudget_amount": "250",
        "dailyBudget_currency": "EUR",
        "displayStatus": "RUNNING",
        "supplySources": [
          "APPSTORE_SEARCH_RESULTS"
        ],
        "adChannelType": "SEARCH",
        "orgId": 51370,
        "billingEvent": "TAPS",
        "countryOrRegion": "US"
      },
      "metrics": {
        "impressions": 25,
        "taps": 0,
        "installs": 0,
        "newDownloads": 0,
        "redownloads": 0,
        "latOnInstalls": 0,
        "latOffInstalls": 0,
        "ttr": 0,
        "avgCPA_amount": "0",
        "avgCPA_currency": "EUR",
        "avgCPT_amount": "0",
        "avgCPT_currency": "EUR",
        "avgCPM_amount": "0",
        "avgCPM_currency": "EUR",
        "localSpend_amount": "0",
        "localSpend_currency": "EUR",
        "conversionRate": 0,
        "date": "2022-11-25"
      }
    },
    "emitted_at": 1669492309596
  }
}
As you can see i have 
metadata
and 
metrics
as objects and they are mapped in the schema. In the main table: 
campaigns
there are not duplicates, but! in the 2 extra tables made for the 2 objects (
metadata
and 
metrics
) there are duplicates Could someone please give me a hint here and direct me to where there problem lies? this is my schema:
Copy code
{
  "$schema": "<http://json-schema.org/draft-04/schema#>",
  "type": "object",
  "properties": {
    "campaignId": {
      "type": "integer"
    },
    "startTime": {
      "type": "string"
    },
    "primary_key": {
      "type": "string"
    },
    "metadata": {
      "type": "object",
      "properties": {
        "campaignId": {
          "type": "integer"
        },
        "campaignName": {
          "type": "string"
        },
        "deleted": {
          "type": "boolean"
        },
        "campaignStatus": {
          "type": "string"
        },
        "app_appName": {
          "type": "string"
        },
        "app_adamId": {
          "type": "integer"
        },
        "servingStatus": {
          "type": "string"
        },
        "servingStateReasons": {
          "type": "null"
        },
        "countriesOrRegions": {
          "type": "array",
          "items": [
            {
              "type": "string"
            }
          ]
        },
        "modificationTime": {
          "type": "string"
        },
        "totalBudget_amount": {
          "type": "string"
        },
        "totalBudget_currency": {
          "type": "string"
        },
        "dailyBudget_amount": {
          "type": "string"
        },
        "dailyBudget_currency": {
          "type": "string"
        },
        "displayStatus": {
          "type": "string"
        },
        "supplySources": {
          "type": "array",
          "items": [
            {
              "type": "string"
            }
          ]
        },
        "adChannelType": {
          "type": "string"
        },
        "orgId": {
          "type": "integer"
        },
        "billingEvent": {
          "type": "string"
        },
        "countryOrRegion": {
          "type": "string"
        }
      }
    },
    "metrics": {
      "type": "object",
      "properties": {
        "impressions": {
          "type": "integer"
        },
        "taps": {
          "type": "integer"
        },
        "installs": {
          "type": "integer"
        },
        "newDownloads": {
          "type": "integer"
        },
        "redownloads": {
          "type": "integer"
        },
        "latOnInstalls": {
          "type": "integer"
        },
        "latOffInstalls": {
          "type": "integer"
        },
        "ttr": {
          "type": "number"
        },
        "avgCPA_amount": {
          "type": "string"
        },
        "avgCPA_currency": {
          "type": "string"
        },
        "avgCPT_amount": {
          "type": "string"
        },
        "avgCPT_currency": {
          "type": "string"
        },
        "avgCPM_amount": {
          "type": "string"
        },
        "avgCPM_currency": {
          "type": "string"
        },
        "localSpend_amount": {
          "type": "string"
        },
        "localSpend_currency": {
          "type": "string"
        },
        "conversionRate": {
          "type": "number"
        },
        "date": {
          "type": "string"
        }
      }
    }
  }
}
s
can you give some more detail on what you mean by duplicate? is this in the raw connector output, or when running airbyte with dedupe mode?
s
thanks for the response. it happens when running the dedupe mode
s
@Edward Gao (Airbyte) does deduplication happen for nested fields?
e
basic normalization doesn’t dedup nested fields - we don’t have a PK to dedup against (since the nested field doesn’t necessarily have the same PK as the top-level object). I didn’t find an open issue for this yet, so probably worth submitting one on github.
s
good to know @Edward Gao (Airbyte) i'll submit an issue on github
2 Views
Open in Slack
PreviousNext
Powered by