Hi everyone. I´m trying to deploy fluent-bit 4.0.4 from Helm Chart v3.1.15, and using this values.yaml:
global:
imagePullSecrets:
- helm-secret
resourcesPreset: "none"
resources:
requests:
cpu: 500m
memory: 512Mi
ephemeral-storage: 50Mi
limits:
cpu: 750m
memory: 768Mi
ephemeral-storage: 2Gi
config:
service: |
[SERVICE]
Flush 1
Log_Level info
Daemon off
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
inputs: |
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser docker
DB /var/log/flb_kube.db
Tag kube.*
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
[INPUT]
Name tail
Tag apiserver_audit.*
Path /var/log/kubernetes/*.log
Parser json
DB /var/log/flb_kube_audit.db
Mem_Buf_Limit 50MB
Skip_Long_Lines On
Refresh_Interval 10
[INPUT]
Name tail
Tag audit.*
Path /var/log/audit/audit.log
Parser auditlog
DB /var/log/flb_system_audit.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
filters: |
[FILTER]
Name kubernetes
Match kube.*
Kube_URL
https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Kube_Tag_Prefix kube.var.log.containers.
Merge_Log On
Merge_Log_Key log_processed
[FILTER]
Name record_modifier
Match *
Record tkg_cluster os-desa-global-z01-1000
Record vrli_tkgs_cluster_name os-desa-global-z01-1000
[FILTER]
Name record_modifier
Match audit.*
Record node_name ${HOSTNAME}
[FILTER]
Name modify
Match kube.*
Copy kubernetes k8s
[FILTER]
Name record_modifier
Match kube.*
Record node_name ${HOSTNAME}
[FILTER]
Name nest
Match *
Wildcard vrli_tkgs_cluster*
Operation nest
Nested_Under vrli_metadata
[FILTER]
Name nest
Match kube.*
Operation lift
Nested_Under kubernetes
[FILTER]
Name nest
Match apiserver_audit.*
Operation nest
Wildcard tkgs*
Nest_Under message
[FILTER]
Name nest
Match apiserver_audit.*
Operation nest
Wildcard vrli_metadata_vrli_tkgs*
Nest_Under message
[FILTER]
Name Lua
Match apiserver_audit.*
call filter
code function filter(tag, timestamp, record) if record["responseObject"] ~= nil and record["objectRef"] ~= nil then local flattened_record = {} flatten_json("", record, flattened_record) if flattened_record["objectRef_name"] ~= nil and flattened_record["objectRef_resource"] == "pods" then return 1, timestamp, flattened_record else return -1, timestamp, flattened_record end else return -1, timestamp, record end end function flatten_json(prefix, json, result) local tkgs_prefix = "tkgs_" local regex_patterns = { "responseObject_status_.
.reason", "responseObject_status_..message", "responseObject_status_phase" } for key, value in pairs(json) do if type(value) == "table" then flatten_json(prefix .. key .. "_", value, result) else local final_key_name = prefix .. key for _, pattern in ipairs(regex_patterns) do if string.match(final_key_name, pattern) then final_key_name = tkgs_prefix .. final_key_name break end end result[final_key_name] = tostring(value) end end end
outputs: |
[OUTPUT]
Name syslog
Match kube.*
Host 10.202.252.38
Port 514
Mode tcp
Syslog_Format rfc5424
Syslog_Hostname_key tkg_cluster
Syslog_Appname_key pod_name
Syslog_Procid_key container_name
Syslog_Message_key message
Syslog_Msgid_key namespace_name
Syslog_SD_key k8s
Syslog_SD_key labels
Syslog_SD_key annotations
Syslog_SD_key tkg
Syslog_SD_key vrli_metadata
[OUTPUT]
Name syslog
Match audit.*
Host 10.202.252.38
Port 514
Mode tcp
Syslog_Format rfc5424
Syslog_Hostname_key tkg_cluster
Syslog_Appname_key kubernetes
Syslog_Message_key message
Syslog_SD_key vrli_metadata
[OUTPUT]
Name syslog
Match apiserver_audit.*
Host 10.202.252.38
Port 514
Mode tcp
Syslog_Format rfc5424
Syslog_Hostname_key tkg_cluster
Syslog_Message_key message
Syslog_SD_key message
Syslog_SD_key vrli_metadata
Allow_longer_sd_id True
Syslog_Appname_key objectRef_name
Syslog_Procid_key objectRef_resource
Syslog_Msgid_key objectRef_namespace
Syslog_maxsize 10240
customParsers: |
[PARSER]
Name auditlog
Format regex
Regex ^(type=\w+)\s+msg=audit\((\d+\.\d+:\d+)\):\s+(.*)
Time_Key timestamp
Time_Format %s.%L
[PARSER]
Name json
Format json
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
streams: ""
plugins: ""
daemonset:
enabled: true
resourcesPreset: small
podAnnotations: {}
podLabels: {}
containerSecurityContext:
enabled: true
seLinuxOptions: {}
runAsUser: 0
runAsGroup: 0
runAsNonRoot: false
privileged: true
allowPrivilegeEscalation: true
readOnlyRootFilesystem: false
capabilities:
drop: ["ALL"]
seccompProfile:
type: "RuntimeDefault"
Fluent-bit deploys its daemonset and creates a pod on each node. I checked the pod logs and appeared to be ok, but when I go to VRLI the only thing that shows is - (see attached image)
Any ideas? is there something I'm missing?
Thanks in advance