Hello, i am trying to use fluent-bit as a destination from Splunk Edge Processor or EP. i have defined a Splunk HEC destination. Now, i think this should work, but i am actually running fluent-bit 4.1.1 installed on ubuntu. The same machine which the Splunk Edge instance is running on. There is a specific reason, but im having problems getting fluent-bit to receive the events from Splunk. On the Splunk end i am getting:

Post "<http://192.168.141.95:9880/services/collector>": context deadline exceeded (Client.Timeout exceeded while awaiting headers)

I have tried lots of options and configurations, but my current one is cli based:

root@ip-192-168-141-95:~# /opt/fluent-bit/bin/fluent-bit -i splunk -p port=9880 -p buffer_chunk_size=1024 -p buffer_max_size=32M -p tag=splunk.logs -p net.io_timeout=300s -o stdout -p match=splunk.logs -vv

which is producing:

[2025/11/11 21:44:09.381347930] [trace] [io] connection OK
[2025/11/11 21:44:09.381397730] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.381863699] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.381894442] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382594157] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382625300] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382642132] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382657844] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382674014] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382684183] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382700140] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382710296] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382724162] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382734559] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382748716] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382759216] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382772032] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382782780] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382796156] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382805907] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382818906] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382828814] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382843934] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.382853034] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382863254] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382878776] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382888383] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382908014] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382918664] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382933485] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382943435] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382961527] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.382972431] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.382990641] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383000808] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383026942] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383042965] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383060552] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383070761] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383085467] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383097179] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383111593] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383120958] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383137668] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.383146008] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383157180] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383170359] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383179843] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:09.383193275] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.383203629] [trace] [io coro=(nil)] [net_read] ret=706
[2025/11/11 21:44:09.383216611] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.431509514] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.681537238] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.681554644] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.879452869] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:09.879531281] [trace] [io coro=(nil)] [net_read] ret=0
[2025/11/11 21:44:09.879549725] [trace] [downstream] destroy connection #48 to <tcp://192.168.141.95:46304>
[2025/11/11 21:44:09.879621135] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:09.931509675] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:10.95119333] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:10.95162342] [trace] [io coro=(nil)] [net_read] ret=0
[2025/11/11 21:44:10.95179536] [trace] [downstream] destroy connection #49 to <tcp://192.168.141.95:46314>
[2025/11/11 21:44:10.95247475] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:10.181511800] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:10.431508128] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:10.681546565] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:10.681585263] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:10.931508179] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:11.181514100] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:11.431510732] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:11.681539544] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:11.931508704] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.173087049] [trace] [io] connection OK
[2025/11/11 21:44:12.173199150] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.173810559] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173841862] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.173872772] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173883888] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.173898853] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173909280] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.173923156] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173933024] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.173946124] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173955163] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.173967800] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173977479] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.173989628] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.173999144] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.174096070] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.174110901] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.174203854] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.174395146] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.174415522] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.174426114] [trace] [io coro=(nil)] [net_read] ret=1024
[2025/11/11 21:44:12.174435379] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.174441221] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.174447781] [trace] [io coro=(nil)] [net_read] ret=314
[2025/11/11 21:44:12.174457878] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.181508649] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.181507560] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.430735078] [trace] [io coro=(nil)] [net_read] try up to 1024 bytes
[2025/11/11 21:44:12.430779926] [trace] [io coro=(nil)] [net_read] ret=0
[2025/11/11 21:44:12.430796710] [trace] [downstream] destroy connection #52 to <tcp://192.168.141.95:46322>
[2025/11/11 21:44:12.430866695] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.431506047] [trace] [sched] 0 timer coroutines destroyed
[2025/11/11 21:44:12.681535932] [trace] [sched] 0 timer coroutines destroyed

Any ideas? When i switched it to tcp, i get:

______ _                  _    ______ _ _             ___   __
|  ___| |                | |   | ___ (_) |           /   | /  |
| |_  | |_   _  ___ _ __ | |_  | |_/ /_| |_  __   __/ /| | `| |
|  _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| |  | |
| |   | | |_| |  __/ | | | |_  | |_/ / | |_   \ V /\___  |__| |_
\_|   |_|\__,_|\___|_| |_|\__| \____/|_|\__|   \_/     |_(_)___/


[2025/11/11 21:49:12.454217350] [ info] [fluent bit] version=4.1.1, commit=, pid=7654
[2025/11/11 21:49:12.454345650] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128
[2025/11/11 21:49:12.454355937] [ info] [simd    ] SSE2
[2025/11/11 21:49:12.454363428] [ info] [cmetrics] version=1.0.5
[2025/11/11 21:49:12.454371187] [ info] [ctraces ] version=0.6.6
[2025/11/11 21:49:12.454441883] [ info] [input:tcp:tcp.0] initializing
[2025/11/11 21:49:12.454450891] [ info] [input:tcp:tcp.0] storage_strategy='memory' (memory only)
[2025/11/11 21:49:12.455168829] [ info] [sp] stream processor started
[2025/11/11 21:49:12.455347140] [ info] [output:stdout:stdout.0] worker #0 started
[2025/11/11 21:49:12.455396357] [ info] [engine] Shutdown Grace Period=5, Shutdown Input Grace Period=2
"}] tcp.0: [[1762897752.520261984, {}], {"log"=>"POST /services/collector HTTP/1.1
"}] tcp.0: [[1762897752.520272277, {}], {"log"=>"Host: 192.168.141.95:9880
"}] tcp.0: [[1762897752.520273812, {}], {"log"=>"User-Agent: OpenTelemetry Collector Contrib/11f9362e
"}] tcp.0: [[1762897752.520275124, {}], {"log"=>"Content-Length: 44970
"}] tcp.0: [[1762897752.520276343, {}], {"log"=>"Authorization: Splunk my_token
"}] tcp.0: [[1762897752.520277527, {}], {"log"=>"Connection: keep-alive
"}] tcp.0: [[1762897752.520278816, {}], {"log"=>"Content-Encoding: gzip
"}] tcp.0: [[1762897752.520280153, {}], {"log"=>"Content-Type: application/json
"}] tcp.0: [[1762897752.520281350, {}], {"log"=>"__splunk_app_name: OpenTelemetry Collector Contrib
"}] tcp.0: [[1762897752.520282527, {}], {"log"=>"__splunk_app_version:
"}]] tcp.0: [[1762897752.520283955, {}], {"log"=>"Accept-Encoding: gzip
"}]] tcp.0: [[1762897752.520285037, {}], {"log"=>"Connection: close
"}]] tcp.0: [[1762897752.520286276, {}], {"log"=>"