Georgi Varbanov
06/03/2025, 10:17 AMGeorgi Varbanov
06/03/2025, 10:18 AMapiVersion: v1
kind: ConfigMap
metadata:
name: pinot-controller-config
labels:
app: pinot
component: controller
data:
pinot-controller.conf: |-
controller.port=9000
pinot.set.instance.id.to.hostname=true
# Cluster name
controller.helix.cluster.name=pinot-quickstart
# ZooKeeper configuration
controller.zk.str=pinot-zookeeper:2181
# External access
controller.access.protocols.http.port=9000
# Controller periodic tasks
controller.task.scheduler.enabled=true
controller.task.frequencyPeriod=1h
# DeepStore GCS config
pinot.controller.segment.fetcher.protocols=file,http,gs
pinot.controller.segment.fetcher.gs.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
pinot.controller.storage.factory.class.gs=org.apache.pinot.plugin.filesystem.GcsPinotFS
pinot.controller.storage.factory.gs.projectId={projectId}
pinot.controller.storage.factory.gs.gcpKey=/config/service-account.json
controller.data.dir=gs://dk-pinot-data/deepstore/
# Temporary directory management
controller.local.temp.dir=/var/pinot/controller/data/tmp
controller.local.temp.cleaner.enabled=true
controller.local.temp.cleaner.initialDelay=1h
controller.local.temp.cleaner.frequencyInSeconds=3600
controller.local.temp.cleaner.cleanupOlderThanDays=1
# Segment management
controller.deleted.segments.retentionInDays=1
controller.segment.download.timeoutMs=600000
controller.statuschecker.enabled=true
controller.statuschecker.frequencyInSeconds=300
controller.statuschecker.waitForPushTimeoutMs=600000
# Tuning
controller.realtime.segment.commit.timeoutMs=120000
controller.realtime.segment.deepStoreUploadRetryEnabled=true
controller.allow.hlc.tables=false
controller.enable.split.commit=true
# Segment validation
controller.segment.validation.enabled=true
controller.segment.validation.frequencyInSeconds=3600
# Storage quotas and cleanup
controller.storage.quota.check.enabled=true
controller.storage.quota.check.frequencyInSeconds=300
controller.segment.level.validation.enabled=trueGeorgi Varbanov
06/03/2025, 10:18 AMapiVersion: v1
kind: ConfigMap
metadata:
name: pinot-server-config
labels:
app: pinot
component: server
data:
pinot-server.conf: |-
pinot.server.netty.port=8098
pinot.server.adminapi.port=8097
pinot.server.instance.dataDir=/var/pinot/server/data/index
pinot.server.instance.segmentTarDir=/var/pinot/server/data/segment
pinot.set.instance.id.to.hostname=true
pinot.server.instance.realtime.alloc.offheap=true
pinot.query.server.port=7321
pinot.query.runner.port=7732
pinot.server.instance.segment.stream.download.untar=true
pinot.server.instance.segment.stream.download.untar.rate.limit.bytes.per.sec=100000000
pinot.server.instance.table.level.max.parallel.segment.downloads=4
pinot.server.netty.native.transports.enabled=true
pinot.server.instance.enable.split.commit=true
realtime.segment.serverUploadToDeepStore=true
pinot.server.instance.segment.store.uri=gs://dk-pinot-data/deepstore
pinot.server.storage.factory.class.gs=org.apache.pinot.plugin.filesystem.GcsPinotFS
pinot.server.segment.fetcher.protocols=file,http,gs
pinot.server.segment.fetcher.gs.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcherGeorgi Varbanov
06/03/2025, 10:20 AM{
"tableName": "purchases",
"tableType": "REALTIME",
"segmentsConfig": {
"replication": "3",
"schemaName": "purchases",
"timeType": "MILLISECONDS",
"timeColumnName": "creation_date",
"segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
"minimizeDataMovement": false,
"segmentPushType": "APPEND",
"replicaGroupStrategyConfig": {
"numInstancesPerPartition": 3,
"partitionColumn": "customer_id"
},
"completionConfig": {
"completionMode": "DOWNLOAD"
},
"peerSegmentDownloadScheme": "http"
},
"tenants": {
"broker": "DefaultTenant",
"server": "DefaultTenant"
},
"tableIndexConfig": {
"noDictionaryColumns": [...
],
"aggregateMetrics": false,
"enableDefaultStarTree": false,
"invertedIndexColumns": [...
],
"streamConfigs": {
"streamType": "kafka",
"stream.kafka.consumer.type": "LOWLEVEL",
"stream.kafka.broker.list": "broker:9092",
"stream.kafka.topic.name": "pinot-bets-migration",
"stream.kafka.decoder.class.name": "org.apache.pinot.plugin.inputformat.json.JSONMessageDecoder",
"stream.kafka.consumer.prop.auto.offset.reset": "smallest",
"realtime.segment.flush.threshold.segment.rows": "2200000",
"realtime.segment.flush.threshold.time": "24h",
"realtime.segment.flush.threshold.segment.size": "0",
"realtime.segment.flush.threshold.size": "0"
},
"nullHandlingEnabled": true,
"bloomFilterColumns": [
"id",
"bets_id"
],
"rangeIndexColumns": [
"creation_date",
"bets_settlement_date",
"bets_selections_event_date"
],
"jsonIndexColumns": [],
"rangeIndexVersion": 2,
"optimizeDictionaryForMetrics": true,
"optimizeDictionary": true,
"autoGeneratedInvertedIndex": false,
"optimizeDictionaryType": true,
"noDictionarySizeRatioThreshold": 0.85,
"createInvertedIndexDuringSegmentGeneration": true,
"loadMode": "MMAP",
"enableDynamicStarTreeCreation": true,
"columnMajorSegmentBuilderEnabled": true,
"segmentPartitionConfig": {
"columnPartitionMap": {
"customer_id": {
"functionName": "Murmur",
"numPartitions": 21
}
}
}
},
"routing": {
"segmentPrunerTypes": [
"partition",
"time"
],
"instanceSelectorType": "strictReplicaGroup"
},
"upsertConfig": {
"mode": "FULL",
"comparisonColumns": [
"update_date"
],
"enableSnapshot": true,
"enablePreload": true,
"metadataTTL": 600
},
"metadata": {},
"task": {
"taskTypeConfigsMap": {
"UpsertCompactionTask": {
"schedule": "0 */5 * * * ?",
"bufferTimePeriod": "1m",
"invalidRecordsThresholdPercent": "10",
"tableMaxNumTasks": "100",
"validDocIdsType": "SNAPSHOT"
}
}
},
"ingestionConfig": {
"complexTypeConfig": {
"fieldsToUnnest": [
"Bets",
"Bets.Selections"
]
},
"transformConfigs": [...],
"continueOnError": true,
"rowTimeValueCheck": false,
"segmentTimeValueCheck": true
},
"isDimTable": false
}Georgi Varbanov
06/03/2025, 10:20 AMGeorgi Varbanov
06/03/2025, 10:27 AMGeorgi Varbanov
06/03/2025, 10:28 AMGeorgi Varbanov
06/03/2025, 10:32 AMGeorgi Varbanov
06/03/2025, 10:33 AMGeorgi Varbanov
06/03/2025, 10:48 AMGeorgi Varbanov
06/03/2025, 10:49 AMGeorgi Varbanov
06/03/2025, 11:14 AMMayank