Hi team, I want to know a few things: what will ha...
# troubleshooting
r
Hi team, I want to know a few things: what will happen if we increase the number of replicas from 2 to 3 of pinot with this values.yaml
Copy code
image:
 repository: 615177075440.dkr.ecr.ap-south-1.amazonaws.com/pinot
 tag: 1.0.1
 pullPolicy: IfNotPresent

cluster:
 name: pinot-prod

# ----------------------------------------------------------------------------
# ZOOKEEPER: 3 replicas
# ----------------------------------------------------------------------------
zookeeper:
 name: pinot-zookeeper
 replicaCount: 3
 persistence:
  enabled:   true
  storageClass: gp3   # ← GP3 EBS
  size:     10Gi
 resources:
  requests:
   cpu:  100m
   memory: 256Mi
  limits:
   cpu:  300m
   memory: 512Mi

 port: 2181

# ----------------------------------------------------------------------------
# CONTROLLER: 2 replicas, internal LB
# ----------------------------------------------------------------------------
controller:
 name: pinot-controller
 replicaCount: 2
 startCommand: "StartController"
 # podManagementPolicy: Parallel
 resources:
  requests:
   cpu:  100m
   memory: 1Gi
  limits:
   cpu:  300m
   memory: 3Gi
  
 jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=9010:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -XX:ActiveProcessorCount=2 -Xms512M -Xmx2G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-controller.log -Djute.maxbuffer=4000000"
   
 # Persist controller metadata
 persistence:
  enabled:   true
  accessMode:  ReadWriteOnce
  storageClass: gp3
  size:     50Gi
  mountPath:  /var/pinot/controller/data

 service:
  name: controller
  annotations:
   prometheus.io/scrape: "true"
   prometheus.io/path: /metrics
   prometheus.io/port: "9010"
  # labels:
  #  prometheus-monitor: "true"
  extraPorts:
   - name: controller-prom
    protocol: TCP
    containerPort: 9010
   
   
 podAnnotations:
  prometheus.io/scrape: "true"
  prometheus.io/path: /metrics
  prometheus.io/port: "9010"

 # Expose via Kubernetes Ingress on port 9000
 # ingress:
 #  v1:
 #   enabled: true
 #   ingressClassName: nginx     # or your ingress controller
 #   annotations:
 #    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
 #    nginx.ingress.kubernetes.io/rewrite-target: /
 #   hosts: [pinot-eks.sr-bi-internal.in]
 #   path: /controller
 #   tls: []
 external:
  enabled: false

# ----------------------------------------------------------------------------
# BROKER: 2 replicas, internal LB
# ----------------------------------------------------------------------------
broker:
 name: pinot-broker
 startCommand: "StartBroker"
 # podManagementPolicy: Parallel
 replicaCount: 2

 resources:
  requests:
   cpu:  200m
   memory: 1Gi
  limits:
   cpu:  500m
   memory: 3Gi

 jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=9020:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -XX:ActiveProcessorCount=2 -Xms512M -Xmx2G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-broker.log -Djute.maxbuffer=4000000"

 service:
  name: broker
  # type: LoadBalancer
  annotations:
   # service.beta.kubernetes.io/aws-load-balancer-internal: "true"
   prometheus.io/scrape: "true"
   prometheus.io/path: /metrics
   prometheus.io/port: "9020"
  # labels:
  #  prometheus-monitor: "true"
  extraPorts:
   - name: broker-prom
    protocol: TCP
    containerPort: 9020
   
 podAnnotations:
  prometheus.io/scrape: "true"
  prometheus.io/path: /metrics
  prometheus.io/port: "9020"

 # Expose via Kubernetes Ingress on port 8099
 # ingress:
 #  v1:
 #   enabled: true
 #   ingressClassName: nginx     # or your ingress controller
 #   annotations:
 #    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
 #    nginx.ingress.kubernetes.io/use-regex: "true"
 #    nginx.ingress.kubernetes.io/rewrite-target: /$2
 #   hosts: [pinot-eks.sr-bi-internal.in]
 #   path: /broker(/|$)(.*)
 #   pathType: ImplementationSpecific
 #   tls: []
 external:
  enabled: false


# ----------------------------------------------------------------------------
# PINOT SERVER: 2 replicas, each with 100 Gi gp3 PVC
# ----------------------------------------------------------------------------
server:
 name: pinot-server
 startCommand: "StartServer"
 # podManagementPolicy: Parallel
 replicaCount: 2
  
 resources:
  requests:
   cpu:  2000m  # 2 vCPU
   memory: 5Gi
  limits:
   cpu:  4000m  # 4 vCPU
   memory: 10Gi

 persistence:
  enabled: true
  accessMode: ReadWriteOnce
  size: 100G
  mountPath: /var/pinot/server/data
  storageClass: gp3
  
 jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=9030:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -Xms4G -Xmx8G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-server.log -Djute.maxbuffer=4000000"

 service:
  name: server
  # type: LoadBalancer
  annotations:
   # service.beta.kubernetes.io/aws-load-balancer-internal: "true"
   prometheus.io/scrape: "true"
   prometheus.io/path: /metrics
   prometheus.io/port: "9030"
  # labels:
  #  prometheus-monitor: "true"
  extraPorts:
   - name: server-prom
    protocol: TCP
    containerPort: 9030
   
 podAnnotations:
  prometheus.io/scrape: "true"
  prometheus.io/path: /metrics
  prometheus.io/port: "9030"


# ----------------------------------------------------------------------------
# MINION: 1 replica (for background tasks / retention, compaction, etc.)
# ----------------------------------------------------------------------------
minion:
 enabled: true      # run the minion pod
 name: pinot-minion
 startCommand: "StartMinion"
 # podManagementPolicy: Parallel
 replicaCount: 1     # scale up if you have heavy compaction/merge workloads

 resources:
  requests:
   cpu:  100m
   memory: 512Mi
  limits:
   cpu:  200m
   memory: 1Gi

 jvmOpts: "-javaagent:/opt/pinot/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar=8008:/opt/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml -XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-minion.log -Djute.maxbuffer=4000000"
I am running pinot EKS with 3 nodes but utilizing only 2 as the number of replicas are two only..... I want to use 3 nodes for better performance what will happen if I increase the number of pods to 3 and how should I do it with the existing data not getting lost? @Xiang Fu @Mayank