Hey, nobody has this issue or use S3 bucket for their outputs?

I don't think this is anything to do specifically with the output, it's around enqueuing chunks into the general ring buffer

Hi @Pat, thank you for the reply. Our entire configuration looks like this.

filters: |
    [FILTER]
        Name                kubernetes
        Alias               {{ .Release.Name }}_kubernetes_filter
        Match               kube.*
        Kube_URL            <https://kubernetes.default.svc.cluster.local:443>
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        Merge_Log           On
        # a valid json log will be available under this key
        Merge_Log_Key       log_processed
        # don't limit the API response buffer
        Buffer_Size         0
        Use_Kubelet         true

    [FILTER]
        Name           lua
        Alias          {{ .Release.Name }}_lua_filter
        Match          kube.*
        script         /fluent-bit/scripts/transform_record.lua
        time_as_table  true
        protected_mode true
        call           transform
inputs: |
    [INPUT]
        Name             tail
        Alias            {{ .Release.Name }}_tail_input
        Path             /var/log/containers/xxx-*.log, /var/log/containers/yyy-*.log, /var/log/containers/zzz-*.log
        Exclude_Path     /var/log/containers/*-fluent-bit-*_logging_fluent-bit-*.log, /var/log/containers/kube-proxy*.log
        DB               /var/fluent-bit/state/flb_container.db
        multiline.parser docker, cri
        Tag              kube.*
        threaded         on
        Mem_Buf_Limit    32MB
        Skip_Long_Lines  On
        Skip_Empty_Lines On
        Refresh_Interval 10
        Rotate_Wait      30

        # Buffering & Storage (<https://docs.fluentbit.io/manual/administration/buffering-and-storage#input-section-configuration>)
        storage.type     filesystem

    [INPUT]
        name            fluentbit_metrics
        Alias           {{ .Release.Name }}_fluentbit_metrics_input
        tag             internal_metrics
        scrape_interval 2
        scrape_on_start true

        # Buffering & Storage (<https://docs.fluentbit.io/manual/administration/buffering-and-storage#input-section-configuration>)
        storage.type                      filesystem
        storage.pause_on_chunks_overlimit on
outputs: |
    [OUTPUT]
        Name                     s3
        Alias                    {{ .Release.Name }}_s3_output
        Match                    kube.*

        # <https://docs.fluentbit.io/manual/pipeline/outputs/s3#worker-support>
        workers                  1

        region                   eu-west-1
        bucket                   my-bucket-name
        endpoint                 <http://my-bucket-name.s3-eu-west-1.amazonaws.com|my-bucket-name.s3-eu-west-1.amazonaws.com>
        s3_key_format            /xyz/eu-west-1/some-cluster-1/%Y/%m/%d/%H/$UUID.gz
        # Added canned_acl for cross account ownership
        canned_acl               bucket-owner-full-control
        upload_timeout           5m
        total_file_size          50M
        auto_retry_requests      true
        retry_limit              5
        store_dir                /var/fluent-bit/state/s3-buffer
        use_put_object           On
        compression              gzip

        # Note: Use store_dir_limit_size instead of storage.total_limit_size which can be used to other plugins,
        # because S3 has its own buffering system.
        store_dir_limit_size     512M

    [OUTPUT]
        name                     prometheus_exporter
        Alias                    {{ .Release.Name }}_prometheus_exporter_output
        match                    internal_metrics
        host                     0.0.0.0
        port                     2021

        # Buffering & Storage (<https://docs.fluentbit.io/manual/administration/buffering-and-storage#input-section-configuration>)
        storage.total_limit_size 5M
service: |
    [SERVICE]
        Daemon                    Off
        Flush                     {{ .Values.flush }}
        Log_Level                 {{ .Values.logLevel }}
        Parsers_File              parsers.conf
        Parsers_File              custom_parsers.conf
        HTTP_Server               On
        HTTP_Listen               0.0.0.0
        HTTP_Port                 {{ .Values.metricsPort }}
        Health_Check              On

        # Buffering & Storage (<https://docs.fluentbit.io/manual/administration/buffering-and-storage#service-section-configuration>)
        storage.path                        /var/fluent-bit/state/flb-storage/
        storage.sync                        normal
        storage.checksum                    off
        storage.backlog.mem_limit           5M
        storage.metrics                     on
        storage.delete_irrecoverable_chunks on

Yeah so you have threaded mode set to on for the tail input which looking at the issue seemed to be the cause