Saral
05/22/2025, 10:47 AMinfo, warn, error, debug) from fluentbit using some filters. Can fluentbit filter help me here or is there anyother way ?Pat
05/22/2025, 11:18 AMPat
05/22/2025, 11:20 AMPat
05/22/2025, 11:23 AMSaral
05/22/2025, 11:24 AMPat
05/22/2025, 11:26 AMPat
05/22/2025, 11:26 AMPat
05/22/2025, 11:27 AMPat
05/22/2025, 11:28 AMSaral
05/22/2025, 11:29 AMPat
05/22/2025, 11:30 AMPat
05/22/2025, 11:30 AMPat
05/22/2025, 11:32 AMSaral
05/22/2025, 11:34 AMPat
05/22/2025, 11:39 AMSaral
05/22/2025, 1:11 PMSaral
06/19/2025, 5:30 AMPat
06/19/2025, 6:28 AMPat
06/19/2025, 6:29 AMSaral
06/19/2025, 8:50 AMfilters:
- grep:
exclude: "log_processed.level DEBUG|log_processed.level debug|level DEBUG|level debug"
- grep:
exclude: "(?i)\"level\": \"debug\""
log in rawstring:
{
"tag": "ccsportal.var.log.containers.structured-log-generator-6c56f8956b-7w8cf_demo_logger-ffb6731dbc72e73b02aca6cab7b929a66f04847e8856c91663a05e086d39addc.log",
"kubernetes": {
"docker_id": "ffb6731dbc72e73b02aca6cab7b929a66f04847e8856c91663a05e086d39addc",
"namespace_name": "demo",
"pod_ip": "demo",
"master_url": "demo",
"host": "ip-10-150-30-203.us-west-2.compute.internal",
"labels": {
"name": "structured-log-generator",
"app": "structured-log-generator",
"clusterName": "ccsdr02",
"pod-template-hash": "6c56f8956b",
"version": "demo"
},
"container_name": "logger",
"namespace_id": "demo",
"container_image_id": "demo",
"container_hash": "docker.io/library/python@sha256:a40cf9eba2c3ed9226afa9ace504f07ad30fe831343bb1c69f7a6707aadb7c21",
"pod_name": "structured-log-generator-6c56f8956b-7w8cf",
"container_image": "docker.io/library/python:3.9-slim",
"namespace_labels": {
"kubernetes_io/metadata_name": "demo",
"app_kubernetes_io/name": "demo",
"kubernetes_io/managed-by": "demo",
"name": "demo",
"istio-injection": "demo"
},
"pod_id": "2d7159d8-0a57-4e10-9f97-3fd047dc230f"
},
"time": "2025-06-18T11:04:31.064832Z",
"logtag": "F",
"level": "debug",
"log": "{\"time\": \"2025-06-18T11:04:31.064832Z\", \"level\": \"debug\", \"msg\": \"Generated log at level: debug\"}",
"stream": "stdout",
"@timestamp": "2025-06-18T11:04:31.064Z",
"log_processed": {
"level": "debug",
"msg": "Generated log at level: debug",
"time": "2025-06-18T11:04:31.064832Z"
},
"docker": {
"container_id": "ffb6731dbc72e73b02aca6cab7b929a66f04847e8856c91663a05e086d39addc"
},
"msg": "Generated log at level: debug",
"log_collector": "FluentBit"
}Pat
06/19/2025, 8:52 AMSaral
06/19/2025, 9:02 AM[4] ccsportal.var.log.containers.structured-log-generator-6c56f8956b-tpmt4_demo_logger-550678cc53d50e8b2549a87c0fa9dd4ca614b316452f6dd344a165b02ea2eacf.log: [[1750320617.694199000, {}], {"time"=>"2025-06-19T08:10:17.694199Z", "logtag"=>"F", "level"=>"FATAL", "log"=>"{"time": "2025-06-19T08:10:17.694199Z", "level": "fatal", "msg": "Generated log at level: fatal"}", "log_processed"=>{"time"=>"2025-06-19T08:10:17.694199Z", "level"=>"FATAL", "msg"=>"Generated log at level: fatal"}, "@timestamp"=>"2025-06-19T08:10:17.694Z", "msg"=>"Generated log at level: fatal", "docker"=>{"container_id"=>"550678cc53d50e8b2549a87c0fa9dd4ca614b316452f6dd344a165b02ea2eacf"}, "log_collector"=>"FluentBit", "stream"=>"stdout", "kubernetes"=>{"container_name"=>"logger", "namespace_name"=>"demo", "container_hash"=>"docker.io/library/python@sha256:a40cf9eba2c3ed9226afa9ace504f07ad30fe831343bb1c69f7a6707aadb7c21", "pod_name"=>"structured-log-generator-6c56f8956b-tpmt4", "labels"=>{"pod-template-hash"=>"6c56f8956b", "version"=>"demo", "clusterName"=>"ccsdr02", "name"=>"structured-log-generator", "app"=>"structured-log-generator"}, "pod_id"=>"3cdb2812-c2da-4485-8b1c-5aed8acdf7e9", "container_image"=>"docker.io/library/python:3.9-slim", "container_image_id"=>"demo", "namespace_id"=>"demo", "master_url"=>"demo", "pod_ip"=>"demo", "namespace_labels"=>{"kubernetes_io/managed-by"=>"demo", "kubernetes_io/metadata_name"=>"demo", "istio-injection"=>"demo", "name"=>"demo", "app_kubernetes_io/name"=>"demo"}, "host"=>"ip-10-150-9-1.us-west-2.compute.internal", "docker_id"=>"550678cc53d50e8b2549a87c0fa9dd4ca614b316452f6dd344a165b02ea2eacf"}}]Saral
06/19/2025, 9:02 AMSaral
06/19/2025, 9:07 AMfunction normalize_all_log_levels(tag, timestamp, record)
local function canonicalize(val)
if not val then return nil end
local lvl = tostring(val):lower()
if lvl == "error" then return "ERROR"
elseif lvl == "warn" or lvl == "warning" then return "WARN"
elseif lvl == "info" then return "INFO"
elseif lvl == "debug" then return "DEBUG"
elseif lvl == "fatal" then return "FATAL"
else return lvl:upper()
end
end
local top_keys = {"logLevel", "level"}
for _, key in ipairs(top_keys) do
if record[key] then
record[key] = canonicalize(record[key])
end
end
if record["log_processed"] then
for _, key in ipairs(top_keys) do
if record["log_processed"][key] then
record["log_processed"][key] = canonicalize(record["log_processed"][key])
end
end
end
-- Final log level
local log_level = record["logLevel"] or record["level"]
if not log_level and record["log_processed"] then
log_level = record["log_processed"]["logLevel"] or record["log_processed"]["level"]
end
-- Ensure top-level field for grep
if log_level then
record["level"] = log_level
end
if log_level == "WARN" or log_level == "ERROR" then
return -1 -- drop
end
return 2, timestamp, record
end