This message was deleted.

Hi Jay, Seems like you only have one worker slot, so regardless of how many tasks you request in the ingestion, it will only be able to run one at a time. That's why you see all of the others, in a PENDING state. You can add more resources CPU to your Middle Manager (MM) and increase worker.capacity accordingly, and/or add more MMs. For efficient processing you'll want to keep the

druid.worker.capacity = (CPUs available to the MM) - 1

.

I will be honest, I don’t know what is Middle Manager. I used druid-operator on eks and this yaml which manages

brokers, coordinators, historicals, routers

I don’t have middle manager.

# This spec only works on a single node kubernetes cluster(e.g. typical k8s cluster setup for dev using kind/minikube or single node AWS EKS cluster etc)
# as it uses local disk as "deep storage".
#
apiVersion: "<http://druid.apache.org/v1alpha1|druid.apache.org/v1alpha1>"
kind: "Druid"
metadata:
  name: tiny-cluster
spec:
  image: apache/druid:0.22.1
  # Optionally specify image for all nodes. Can be specify on nodes also
  # imagePullSecrets:
  # - name: tutu
  startScript: /druid.sh
  podLabels:
    environment: stage
    release: alpha
  podAnnotations:
    dummykey: dummyval
  readinessProbe:
    httpGet:
      path: /status/health
      port: 8088
  livenessProbe:
    httpGet:
      path: /status/health
      port: 8088
  securityContext:
    fsGroup: 0
    runAsUser: 0
    runAsGroup: 0
  services:
    - spec:
        type: ClusterIP
        clusterIP: None
  commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common"
  jvm.options: |-
    -server
    -XX:MaxDirectMemorySize=10240g
    -Duser.timezone=UTC
    -Dfile.encoding=UTF-8
    -Dlog4j.debug
    -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
    -Djava.io.tmpdir=/druid/data
    -Daws.region=us-east-1
  log4j.config: |-
    <?xml version="1.0" encoding="UTF-8" ?>
    <Configuration status="WARN">
        <Appenders>
            <Console name="Console" target="SYSTEM_OUT">
                <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
            </Console>
        </Appenders>
        <Loggers>
            <Root level="info">
                <AppenderRef ref="Console"/>
            </Root>
        </Loggers>
    </Configuration>
  common.runtime.properties: |

    # Zookeeper
    druid.zk.service.host=tiny-cluster-zk-0.tiny-cluster-zk,tiny-cluster-zk-1.tiny-cluster-zk,tiny-cluster-zk-2.tiny-cluster-zk
    druid.zk.paths.base=/druid
    druid.zk.service.compress=false
    
    # Metadata Store
    druid.metadata.storage.type=postgresql
    druid.metadata.storage.connector.connectURI=jdbc:<postgresql://xxx/druid>
    druid.metadata.storage.connector.user=postgres
    druid.metadata.storage.connector.password=druid123
    #  druid.metadata.storage.connector.password={ "type": "aws-rds-token", "user": "postgres", "host": "xxx", "port": 5432, "region": "us-east-1" }
    # Deep Storage
    druid.storage.type=s3
    druid.storage.bucket=xxx
    druid.storage.baseKey=druid
    druid.storage.disableAcl=true
    # s3
    druid.s3.accessKey=xxx
    druid.s3.secretKey=XXX
  

    # Extensions
    #
    druid.extensions.loadList=["druid-kinesis-indexing-service","druid-s3-extensions","postgresql-metadata-storage","druid-aws-rds-extensions","druid-basic-security"]

    #
    # Service discovery
    #
    druid.selectors.indexing.serviceName=druid/overlord
    druid.selectors.coordinator.serviceName=druid/coordinator


    druid.indexer.logs.type=file
    druid.indexer.logs.directory=/druid/data/indexing-logs
    druid.lookup.enableLookupSyncOnStartup=false

    # Authentication
    druid.auth.authenticatorChain=["MyBasicMetadataAuthenticator"]
    druid.auth.authenticator.MyBasicMetadataAuthenticator.type=basic
    druid.auth.authenticator.MyBasicMetadataAuthenticator.initialAdminPassword=password1
    druid.auth.authenticator.MyBasicMetadataAuthenticator.initialInternalClientPassword=password2
    druid.auth.authenticator.MyBasicMetadataAuthenticator.credentialsValidator.type=metadata
    druid.auth.authenticator.MyBasicMetadataAuthenticator.skipOnFailure=false
    druid.auth.authenticator.MyBasicMetadataAuthenticator.authorizerName=MyBasicMetadataAuthorizer
    # Escalator
    druid.escalator.type=basic
    druid.escalator.internalClientUsername=druid_system
    druid.escalator.internalClientPassword=password2
    druid.escalator.authorizerName=MyBasicMetadataAuthorizer
    # Authorizer
    druid.auth.authorizers=["MyBasicMetadataAuthorizer"]
    druid.auth.authorizer.MyBasicMetadataAuthorizer.type=basic
  volumeMounts:
    - mountPath: /druid/data
      name: data-volume
    - mountPath: /druid/deepstorage
      name: deepstorage-volume
  volumes:
    - name: data-volume
      emptyDir: {}
    - name: deepstorage-volume
      hostPath:
        path: /tmp/druid/deepstorage
        type: DirectoryOrCreate
  env:
    - name: POD_NAME
      valueFrom:
        fieldRef:
          fieldPath: metadata.name
    - name: POD_NAMESPACE
      valueFrom:
        fieldRef:
          fieldPath: metadata.namespace

  nodes:
    brokers:
      # Optionally specify for running broker as Deployment
      # kind: Deployment
      nodeType: "broker"
      # Optionally specify for broker nodes
      # imagePullSecrets:
      # - name: tutu
      druid.port: 8088
      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker"
      replicas: 3
      runtime.properties: |
        druid.service=druid/broker

        # HTTP server threads
        druid.broker.http.numConnections=5
        druid.server.http.numThreads=10

        # Processing threads and buffers
        druid.processing.buffer.sizeBytes=1
        druid.processing.numMergeBuffers=1
        druid.processing.numThreads=1
        druid.sql.enable=true
      extra.jvm.options: |-
        -Xmx512M
        -Xms512M
      hpAutoscaler:
        maxReplicas: 10
        minReplicas: 3
        scaleTargetRef:
           apiVersion: apps/v1
           kind: StatefulSet
           name: druid-tiny-cluster-brokers
        metrics:
         - type: Resource
           resource:
             name: cpu
             target:
               type: Utilization
               averageUtilization: 50

    coordinators:
      # Optionally specify for running coordinator as Deployment
      # kind: Deployment
      nodeType: "coordinator"
      druid.port: 8088
      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord"
      replicas: 3
      runtime.properties: |
        druid.service=druid/coordinator

        # HTTP server threads
        druid.coordinator.startDelay=PT30S
        druid.coordinator.period=PT30S

        # Configure this coordinator to also run as Overlord
        druid.coordinator.asOverlord.enabled=true
        druid.coordinator.asOverlord.overlordService=druid/overlord
        druid.indexer.queue.startDelay=PT30S
        druid.indexer.runner.type=local
      extra.jvm.options: |-
        -Xmx512M
        -Xms512M

    historicals:
      nodeType: "historical"
      druid.port: 8088
      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/historical"
      replicas: 3
      runtime.properties: |
        druid.service=druid/historical
        druid.server.http.numThreads=5
        druid.processing.buffer.sizeBytes=536870912
        druid.processing.numMergeBuffers=1
        druid.processing.numThreads=1
        # Segment storage
        druid.segmentCache.locations=[{\"path\":\"/druid/data/segments\",\"maxSize\":10737418240}]
        druid.server.maxSize=10737418240
      extra.jvm.options: |-
        -Xmx512M
        -Xms512M
          
    routers:
      nodeType: "router"
      druid.port: 8088
      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/router"
      replicas: 3
      runtime.properties: |
        druid.service=druid/router

        # HTTP proxy
        druid.router.http.numConnections=10
        druid.router.http.readTimeout=PT5M
        druid.router.http.numMaxThreads=10
        druid.server.http.numThreads=10

        # Service discovery
        druid.router.defaultBrokerServiceName=druid/broker
        druid.router.coordinatorServiceName=druid/coordinator

        # Management proxy to coordinator / overlord: required for unified web console.
        druid.router.managementProxy.enabled=true
      extra.jvm.options: |-
        -Xmx512M
        -Xms512M

I'm not very familiar with the Druid Operator. But you should have either Middle Managers or Indexers. Either one or the other. Those are the workers for batch ingestion and real-time streaming/query.