Rajat
05/19/2025, 7:46 AM{
"REALTIME": {
"tableName": "shipmentMerged_final_1_REALTIME",
"tableType": "REALTIME",
"segmentsConfig": {
"replication": "2",
"timeType": "DAYS",
"retentionTimeUnit": "DAYS",
"retentionTimeValue": "3",
"timeColumnName": "s_created_at",
"minimizeDataMovement": false
},
"tenants": {
"broker": "DefaultTenant",
"server": "DefaultTenant"
},
"tableIndexConfig": {
"noDictionaryColumns": [
"s_etd",
"s_shipped_date",
"a_awb_code",
"s_order_id",
"s_id",
"a_id",
"o_id",
"a_shipment_id",
"s_awb_assign_date",
"s_delivered_date",
"s_awb",
"s_rto_initiated_date",
"s_pickup_scheduled_date",
"a_applied_weight_amount_double",
"o_total_double",
"o_created_at",
"ts_ms_source_kafka",
"o_net_total_double",
"a_charge_weight_amount_double",
"s_rto_delivered_date",
"ar_awb_id",
"am_picked_up_date",
"am_ofd1",
"s_total_double",
"ts_ms_merged_kafka"
],
"bloomFilterColumns": [
"s_company_id"
],
"invertedIndexColumns": [
"o_customer_city",
"o_customer_pincode",
"o_customer_state",
"s_company_id",
"s_courier",
"o_shipping_method",
"o_payment_method",
"s_status",
"s_sr_courier_id"
],
"sortedColumn": [
"s_company_id"
],
"varLengthDictionaryColumns": [
"o_customer_state",
"ar_zone",
"s_courier",
"o_customer_pincode",
"o_payment_method",
"o_shipping_method",
"o_customer_city"
],
"rangeIndexVersion": 2,
"optimizeDictionaryForMetrics": false,
"optimizeDictionary": false,
"aggregateMetrics": false,
"enableDefaultStarTree": false,
"nullHandlingEnabled": true,
"columnMajorSegmentBuilderEnabled": true,
"loadMode": "MMAP",
"enableDynamicStarTreeCreation": false,
"optimizeDictionaryType": false,
"noDictionarySizeRatioThreshold": 0.85,
"autoGeneratedInvertedIndex": false,
"createInvertedIndexDuringSegmentGeneration": true
},
"metadata": {
"customConfigs": {}
},
"routing": {
"instanceSelectorType": "strictReplicaGroup"
},
"upsertConfig": {
"mode": "FULL",
"consistencyMode": "NONE",
"comparisonColumns": [
"s_updated_at"
],
"metadataTTL": 0,
"deleteRecordColumn": "is_deleted",
"dropOutOfOrderRecord": false,
"enableDeletedKeysCompactionConsistency": false,
"enablePreload": false,
"enableSnapshot": false,
"newSegmentTrackingTimeMs": 10000,
"upsertViewRefreshIntervalMs": 3000,
"deletedKeysTTL": 0,
"hashFunction": "NONE",
"defaultPartialUpsertStrategy": "OVERWRITE",
"allowPartialUpsertConsumptionDuringCommit": false
},
"ingestionConfig": {
"streamIngestionConfig": {
"streamConfigMaps": [
{
"streamType": "kafka",
"stream.kafka.consumer.type": "lowlevel",
"stream.kafka.decoder.prop.format": "AVRO",
"stream.kafka.consumer.group.id": "shipmentMerged-consumer-group",
"stream.kafka.decoder.prop.schema.registry.rest.url": "schema-Registry-URL",
"stream.kafka.topic.name": "pinot_d0_d2_realtime",
"stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
"stream.kafka.decoder.class.name": "org.apache.pinot.plugin.inputformat.avro.confluent.KafkaConfluentSchemaRegistryAvroMessageDecoder",
"stream.kafka.broker.list": "BOOTSTRAP-SERVER",
"stream.kafka.schema.registry.url": "SCHEMA-REGISTRY",
"realtime.segment.flush.threshold.time": "24h",
"realtime.segment.flush.threshold.segment.size": "150M",
"stream.kafka.consumer.prop.auto.offset.reset": "smallest"
}
],
"columnMajorSegmentBuilderEnabled": true,
"trackFilteredMessageOffsets": false
},
"transformConfigs": [
{
"columnName": "is_deleted",
"transformFunction": "compareFields(op, 'd')"
},
{
"columnName": "s_total_double",
"transformFunction": "bytesToDouble(o_net_total, 10, 2)"
},
{
"columnName": "o_net_total_double",
"transformFunction": "bytesToDouble(o_net_total, 10, 2)"
},
{
"columnName": "o_total_double",
"transformFunction": "bytesToDouble(o_total, 10, 2)"
},
{
"columnName": "a_applied_weight_amount_double",
"transformFunction": "bytesToDouble(a_applied_weight_amount, 10, 2)"
},
{
"columnName": "a_charge_weight_amount_double",
"transformFunction": "bytesToDouble(a_charge_weight_amount, 10, 2)"
}
],
"continueOnError": false,
"rowTimeValueCheck": false,
"segmentTimeValueCheck": true
},
"isDimTable": false
}
}