<@UGRJA9TEH> why the ingestion is very slow even w...
# troubleshooting
r
@Xiang Fu why the ingestion is very slow even when I am using 5 partitions in kafka topic but still the ingestion rate is 100 only but I want it to increase, here's my table config:
Copy code
{
  "REALTIME": {
    "tableName": "shipmentMerged_final_1_REALTIME",
    "tableType": "REALTIME",
    "segmentsConfig": {
      "replication": "2",
      "timeType": "DAYS",
      "retentionTimeUnit": "DAYS",
      "retentionTimeValue": "3",
      "timeColumnName": "s_created_at",
      "minimizeDataMovement": false
    },
    "tenants": {
      "broker": "DefaultTenant",
      "server": "DefaultTenant"
    },
    "tableIndexConfig": {
      "noDictionaryColumns": [
        "s_etd",
        "s_shipped_date",
        "a_awb_code",
        "s_order_id",
        "s_id",
        "a_id",
        "o_id",
        "a_shipment_id",
        "s_awb_assign_date",
        "s_delivered_date",
        "s_awb",
        "s_rto_initiated_date",
        "s_pickup_scheduled_date",
        "a_applied_weight_amount_double",
        "o_total_double",
        "o_created_at",
        "ts_ms_source_kafka",
        "o_net_total_double",
        "a_charge_weight_amount_double",
        "s_rto_delivered_date",
        "ar_awb_id",
        "am_picked_up_date",
        "am_ofd1",
        "s_total_double",
        "ts_ms_merged_kafka"
      ],
      "bloomFilterColumns": [
        "s_company_id"
      ],
      "invertedIndexColumns": [
        "o_customer_city",
        "o_customer_pincode",
        "o_customer_state",
        "s_company_id",
        "s_courier",
        "o_shipping_method",
        "o_payment_method",
        "s_status",
        "s_sr_courier_id"
      ],
      "sortedColumn": [
        "s_company_id"
      ],
      "varLengthDictionaryColumns": [
        "o_customer_state",
        "ar_zone",
        "s_courier",
        "o_customer_pincode",
        "o_payment_method",
        "o_shipping_method",
        "o_customer_city"
      ],
      "rangeIndexVersion": 2,
      "optimizeDictionaryForMetrics": false,
      "optimizeDictionary": false,
      "aggregateMetrics": false,
      "enableDefaultStarTree": false,
      "nullHandlingEnabled": true,
      "columnMajorSegmentBuilderEnabled": true,
      "loadMode": "MMAP",
      "enableDynamicStarTreeCreation": false,
      "optimizeDictionaryType": false,
      "noDictionarySizeRatioThreshold": 0.85,
      "autoGeneratedInvertedIndex": false,
      "createInvertedIndexDuringSegmentGeneration": true
    },
    "metadata": {
      "customConfigs": {}
    },
    "routing": {
      "instanceSelectorType": "strictReplicaGroup"
    },
    "upsertConfig": {
      "mode": "FULL",
      "consistencyMode": "NONE",
      "comparisonColumns": [
        "s_updated_at"
      ],
      "metadataTTL": 0,
      "deleteRecordColumn": "is_deleted",
      "dropOutOfOrderRecord": false,
      "enableDeletedKeysCompactionConsistency": false,
      "enablePreload": false,
      "enableSnapshot": false,
      "newSegmentTrackingTimeMs": 10000,
      "upsertViewRefreshIntervalMs": 3000,
      "deletedKeysTTL": 0,
      "hashFunction": "NONE",
      "defaultPartialUpsertStrategy": "OVERWRITE",
      "allowPartialUpsertConsumptionDuringCommit": false
    },
    "ingestionConfig": {
      "streamIngestionConfig": {
        "streamConfigMaps": [
          {
            "streamType": "kafka",
            "stream.kafka.consumer.type": "lowlevel",
            "stream.kafka.decoder.prop.format": "AVRO",
            "stream.kafka.consumer.group.id": "shipmentMerged-consumer-group",
            "stream.kafka.decoder.prop.schema.registry.rest.url": "schema-Registry-URL",
            "stream.kafka.topic.name": "pinot_d0_d2_realtime",
            "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
            "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.inputformat.avro.confluent.KafkaConfluentSchemaRegistryAvroMessageDecoder",
            "stream.kafka.broker.list": "BOOTSTRAP-SERVER",
            "stream.kafka.schema.registry.url": "SCHEMA-REGISTRY",
            "realtime.segment.flush.threshold.time": "24h",
            "realtime.segment.flush.threshold.segment.size": "150M",
            "stream.kafka.consumer.prop.auto.offset.reset": "smallest"
          }
        ],
        "columnMajorSegmentBuilderEnabled": true,
        "trackFilteredMessageOffsets": false
      },
      "transformConfigs": [
        {
          "columnName": "is_deleted",
          "transformFunction": "compareFields(op, 'd')"
        },
        {
          "columnName": "s_total_double",
          "transformFunction": "bytesToDouble(o_net_total, 10, 2)"
        },
        {
          "columnName": "o_net_total_double",
          "transformFunction": "bytesToDouble(o_net_total, 10, 2)"
        },
        {
          "columnName": "o_total_double",
          "transformFunction": "bytesToDouble(o_total, 10, 2)"
        },
        {
          "columnName": "a_applied_weight_amount_double",
          "transformFunction": "bytesToDouble(a_applied_weight_amount, 10, 2)"
        },
        {
          "columnName": "a_charge_weight_amount_double",
          "transformFunction": "bytesToDouble(a_charge_weight_amount, 10, 2)"
        }
      ],
      "continueOnError": false,
      "rowTimeValueCheck": false,
      "segmentTimeValueCheck": true
    },
    "isDimTable": false
  }
}