Aaron Weiss
10/04/2022, 8:13 PMAaron Weiss
10/04/2022, 8:13 PM{
"REALTIME": {
"tableName": "immutable_events_REALTIME",
"tableType": "REALTIME",
"segmentsConfig": {
"timeType": "MILLISECONDS",
"schemaName": "immutable_events",
"retentionTimeUnit": "DAYS",
"retentionTimeValue": "4",
"replicasPerPartition": "2",
"timeColumnName": "event_timestamp",
"minimizeDataMovement": false
},
"tenants": {
"broker": "DefaultTenant",
"server": "immutable-realtime"
},
"tableIndexConfig": {
"rangeIndexVersion": 2,
"autoGeneratedInvertedIndex": false,
"createInvertedIndexDuringSegmentGeneration": false,
"sortedColumn": [
"tenant_name"
],
"loadMode": "MMAP",
"streamConfigs": {
"streamType": "kafka",
"stream.kafka.consumer.type": "lowlevel",
"stream.kafka.topic.name": "immutable-enriched-events",
"stream.kafka.decoder.class.name": "org.apache.pinot.plugin.inputformat.avro.confluent.KafkaConfluentSchemaRegistryAvroMessageDecoder",
"stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
"stream.kafka.broker.list": "ica-kafka-kafka-bootstrap:9092",
"realtime.segment.flush.threshold.rows": "10",
"realtime.segment.flush.autotune.initialRows": "100000",
"realtime.segment.flush.threshold.time": "6h",
"realtime.segment.flush.threshold.segment.size": "675M",
"stream.kafka.consumer.prop.auto.offset.reset": "smallest",
"stream.kafka.decoder.prop.schema.registry.rest.url": "<http://ica-schema-registry:8081>",
"schema.registry.url": "<http://ica-schema-registry:8081>"
},
"enableDefaultStarTree": false,
"enableDynamicStarTreeCreation": false,
"aggregateMetrics": false,
"nullHandlingEnabled": false,
"optimizeDictionaryForMetrics": false,
"noDictionarySizeRatioThreshold": 0
},
"metadata": {
"customConfigs": {}
},
"task": {
"taskTypeConfigsMap": {
"RealtimeToOfflineSegmentsTask": {
"bucketTimePeriod": "1d",
"bufferTimePeriod": "1d",
"roundBucketTimePeriod": "1d",
"mergeType": "rollup",
"user_raw_risk_score.aggregationType": "max",
"user_risk_score.aggregationType": "max",
"user_threat_score.aggregationType": "max",
"maxNumRecordsPerSegment": "5000000",
"schedule": "0 * * * * ?"
}
}
},
"routing": {
"instanceSelectorType": "strictReplicaGroup"
},
"dedupConfig": {
"dedupEnabled": true,
"hashFunction": "NONE"
},
"isDimTable": false
}
}
Aaron Weiss
10/04/2022, 8:13 PM{
"schemaName": "immutable_events",
"dimensionFieldSpecs": [
{
"name": "event_id",
"dataType": "STRING"
},
{
"name": "event_type",
"dataType": "STRING"
},
{
"name": "event_type_source",
"dataType": "STRING"
},
{
"name": "account_type",
"dataType": "STRING"
},
{
"name": "action_taken",
"dataType": "STRING"
},
{
"name": "activity_type",
"dataType": "STRING"
},
{
"name": "browser",
"dataType": "STRING"
},
{
"name": "city",
"dataType": "STRING"
},
{
"name": "country",
"dataType": "STRING"
},
{
"name": "device",
"dataType": "STRING"
},
{
"name": "device_management_status",
"dataType": "STRING"
},
{
"name": "document_type",
"dataType": "STRING"
},
{
"name": "domain",
"dataType": "STRING"
},
{
"name": "elastica_user",
"dataType": "STRING"
},
{
"name": "external_recipients",
"dataType": "STRING",
"singleValueField": false
},
{
"name": "host",
"dataType": "STRING"
},
{
"name": "internal_recipients",
"dataType": "STRING",
"singleValueField": false
},
{
"name": "object_type",
"dataType": "STRING"
},
{
"name": "policy_type",
"dataType": "STRING"
},
{
"name": "policy_violated",
"dataType": "STRING"
},
{
"name": "region",
"dataType": "STRING"
},
{
"name": "resp_code",
"dataType": "STRING"
},
{
"name": "sender",
"dataType": "STRING"
},
{
"name": "service",
"dataType": "STRING"
},
{
"name": "severity",
"dataType": "STRING"
},
{
"name": "source",
"dataType": "STRING"
},
{
"name": "status",
"dataType": "STRING"
},
{
"name": "sub_feature",
"dataType": "STRING"
},
{
"name": "user",
"dataType": "STRING"
},
{
"name": "user_name",
"dataType": "STRING"
},
{
"name": "user_agent",
"dataType": "STRING"
},
{
"name": "user_type",
"dataType": "STRING"
},
{
"name": "user_company",
"dataType": "STRING"
},
{
"name": "user_country",
"dataType": "STRING"
},
{
"name": "user_department",
"dataType": "STRING"
},
{
"name": "user_domain",
"dataType": "STRING"
},
{
"name": "user_first_name",
"dataType": "STRING"
},
{
"name": "user_is_active",
"dataType": "BOOLEAN"
},
{
"name": "user_is_admin",
"dataType": "BOOLEAN"
},
{
"name": "user_last_name",
"dataType": "STRING"
},
{
"name": "user_org_unit",
"dataType": "STRING"
},
{
"name": "user_risk_rating",
"dataType": "STRING"
},
{
"name": "user_title",
"dataType": "STRING"
},
{
"name": "tenant_name",
"dataType": "STRING"
}
],
"metricFieldSpecs": [
{
"name": "activity_count",
"dataType": "INT"
},
{
"name": "file_size",
"dataType": "INT"
},
{
"name": "req_size",
"dataType": "INT"
},
{
"name": "resp_size",
"dataType": "INT"
},
{
"name": "event_count",
"dataType": "INT",
"defaultNullValue": 1
},
{
"name": "user_risk_score",
"dataType": "INT"
},
{
"name": "user_raw_risk_score",
"dataType": "FLOAT"
},
{
"name": "user_threat_score",
"dataType": "INT"
}
],
"dateTimeFieldSpecs": [
{
"name": "event_timestamp",
"dataType": "LONG",
"format": "1:MILLISECONDS:EPOCH",
"granularity": "1:DAYS"
}
],
"primaryKeyColumns": [
"event_id"
]
}
Mayank
Aaron Weiss
10/04/2022, 8:47 PMAaron Weiss
10/04/2022, 8:49 PMHowever, you need to restart all Pinot servers so that it can rebuild the primary key to record location map with the new columns.
Aaron Weiss
10/04/2022, 8:49 PMMayank
Neha Pawar
saurabh dubey
10/05/2022, 4:45 AMAaron Weiss
10/05/2022, 12:39 PM