Hello Team, We are trying to enable the Fault-Doma...
# troubleshooting
r
Hello Team, We are trying to enable the Fault-Domain-Aware Instance Assignment on our Pinot cluster version 1.3.0 For testing, we have created 3 servers, and put them in 3 separate pools 1, 2, 3 respectively. one of ther server config is as follow:
Copy code
"pool": {
      "DefaultTenant_REALTIME": "1"
    }
  },
  "listFields": {
    "TAG_LIST": [
      "DefaultTenant_OFFLINE",
      "DefaultTenant_REALTIME"
    ]
But while creating the table we are getting below error:
Copy code
{
  "code": 500,
  "error": "Index 3 out of bounds for length 3"
}
Our Table config looks like below:
Copy code
{
  "REALTIME": {
    "tableName": "otel_spans_REALTIME",
    "tableType": "REALTIME",
    "segmentsConfig": {
      "retentionTimeUnit": "DAYS",
      "retentionTimeValue": "1",
      "segmentPushType": "APPEND",
      "timeColumnName": "startTimeUnixMilli",
      "minimizeDataMovement": false,
      "schemaName": "otel_spans",
      "replication": "3",
      "completionConfig": {
        "completionMode": "DOWNLOAD"
      }
    },
    "instanceAssignmentConfigMap": {
      "CONSUMING": {
        "partitionSelector": "FD_AWARE_INSTANCE_PARTITION_SELECTOR",
        "tagPoolConfig": {
          "tag": "DefaultTenant_REALTIME",
          "poolBased": true
        },
        "replicaGroupPartitionConfig": {
          "replicaGroupBased": true,
          "numReplicaGroups": 3
        }
      }
    },
    "tenants": {
      "broker": "DefaultTenant",
      "server": "DefaultTenant",
      "tagOverrideConfig": {}
    },
    "tableIndexConfig": {
      "rangeIndexVersion": 2,
      "loadMode": "MMAP",
      "autoGeneratedInvertedIndex": false,
      "createInvertedIndexDuringSegmentGeneration": false,
      "enableDefaultStarTree": false,
      "enableDynamicStarTreeCreation": false,
      "aggregateMetrics": false,
      "nullHandlingEnabled": false,
      "columnMajorSegmentBuilderEnabled": true,
      "optimizeDictionary": false,
      "optimizeDictionaryForMetrics": false,
      "noDictionarySizeRatioThreshold": 0.85,
      "noDictionaryColumns": [
        "traceId",
        "spanId",
        "parentSpanId",
        "resourceAttributes",
        "attributes",
        "startTimeUnixMilli",
        "endTimeUnixMilli",
        "statusMessage",
        "events"
      ],
      "invertedIndexColumns": [
        "serviceName",
        "name",
        "statusCode"
      ],
      "bloomFilterColumns": [
        "traceId"
      ],
      "onHeapDictionaryColumns": [],
      "rangeIndexColumns": [
        "duration"
      ],
      "sortedColumn": [
        "startTimeUnixMilli"
      ],
      "varLengthDictionaryColumns": []
    },
    "metadata": {},
    "quota": {},
    "routing": {
      "instanceSelectorType": "replicaGroup",
      "segmentPrunerTypes": [
        "time"
      ]
    },
    "query": {},
    "fieldConfigList": [
      {
        "name": "startTimeUnixMilli",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "endTimeUnixMilli",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "traceId",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "spanId",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "parentSpanId",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "resourceAttributes",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "events",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          }
        },
        "tierOverwrites": null
      },
      {
        "name": "attributes",
        "encodingType": "RAW",
        "indexTypes": [],
        "indexes": {
          "forward": {
            "compressionCodec": "ZSTANDARD",
            "deriveNumDocsPerChunk": false,
            "rawIndexWriterVersion": 4
          },
          "json": {
            "compressionCodec": "ZSTANDARD",
            "maxLevels": 1,
            "excludeArray": true,
            "disableCrossArrayUnnest": true,
            "includePaths": null,
            "excludePaths": null,
            "excludeFields": null,
            "indexPaths": null
          }
        },
        "tierOverwrites": null
      }
    ],
    "ingestionConfig": {
      "streamIngestionConfig": {
        "streamConfigMaps": [
          {
            "streamType": "kafka",
            "stream.kafka.topic.name": "flattened_spans",
            "stream.kafka.broker.list": "kafka:9092",
            "stream.kafka.consumer.type": "lowlevel",
            "stream.kafka.consumer.prop.auto.offset.reset": "smallest",
            "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
            "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
            "realtime.segment.flush.threshold.rows": "0",
            "realtime.segment.flush.threshold.time": "30m",
            "realtime.segment.flush.threshold.segment.size": "300M",
            "realtime.segment.serverUploadToDeepStore": "true"
          }
        ]
      },
      "continueOnError": false,
      "rowTimeValueCheck": false,
      "segmentTimeValueCheck": true
    },
    "isDimTable": false
  }
}
Does anyone have any idea? The error is not pretty clear, what is wrong with schema. One more observation is that if we only apply pool based instance assignment by removing
"partitionSelector": "FD_AWARE_INSTANCE_PARTITION_SELECTOR",
part from the config, it works properly.
@Mayank @Xiang Fu Any idea on this error?
x
can you try to tag server with pool id 0,1,2