Wei
08/20/2025, 3:40 PMCREATE FLOW intf_aggregation
SINK TO interface_sink
AS
SELECT
router,
port
(last_value(rx) - first_value(rx)) * 8/30 as rx_rate,
(last_value(tx) - first_value(tx)) * 8/30 as tx_rate,
last_value(discard) - first_value(discard) as discard_delta,
last_value(crc) - first_value(crc) as crc_delta,
date_bin('30 s'::INTERVAL, timestamp) as time_window,,
FROM interface
GROUP BY
time_window;Ruihang
08/20/2025, 11:57 PMdate_bin and group by. to your case, if tx, rx etc are not mono-increase, first/last value might not be a good chooseWei
08/21/2025, 2:13 PMWei
08/21/2025, 2:13 PMWei
08/21/2025, 6:09 PMCREATE FLOW find_approx_rate SINK TO approx_rate AS
SELECT
(max(byte) - min(byte)) / 30.0 as rate,
date_bin(INTERVAL '30 second', ts) as time_window
from
bytes_log
GROUP BY
time_window;Wei
08/21/2025, 6:11 PMWei
08/21/2025, 6:48 PMCREATE FLOW interface_rate
SINK TO interface_sink
AS
SELECT
router,
port,
(last_value(rx) - first_value(rx)) * 8 / 30 RANGE '30s' as rx,
(last_value(tx) - first_value(tx)) * 8 / 30 RANGE '30s' as tx,
last_value(discards) - first_value(discards) RANGE '30s' as discards,
last_value(crc) - first_value(crc) RANGE '30s' as crc,
date_bin('30 seconds'::INTERVAL, ts) as time_window
FROM interface
ALIGN '40s'
ORDER BY ts;
ERROR: Invalid SQL syntax: sql parser error: Can't use the RANGE keyword in Expr 30 without functionWei
08/21/2025, 6:48 PMWei
08/21/2025, 8:56 PMCREATE FLOW interface_rate
SINK TO interface_sink
AS
SELECT
router,
port,
(rx2 - rx1) * 8 / 30 as rx,
(tx2 - tx1) * 8 / 30 as tx,
discards2 - discards1 as discards,
crc2 - crc1 as crc,
time_window
FROM (
select
router,
port,
first_value(rx) range '40s' as rx1,
last_value(rx) range '40s' as rx2,
first_value(tx) RANGE '40s' as tx1,
last_value(tx) RANGE '40s' as tx2,
first_value(discards) RANGE '40s' as discards1,
last_value(discards) RANGE '40s' as discards2,
first_value(crc) RANGE '40s' as crc1,
last_value(crc) RANGE '40s' as crc2,
ts as time_window
from interface
align '30s' by (router, port)
order by ts);
it still errored out,
ERROR: Failed to encode DataFusion planWei
08/21/2025, 8:57 PMsdn=> SELECT
router,
port,
(rx2 - rx1) * 8 / 30 as rx,
(tx2 - tx1) * 8 / 30 as tx,
discards2 - discards1 as discards,
crc2 - crc1 as crc,
time_window
FROM (
select
router,
port,
first_value(rx) range '40s' as rx1,
last_value(rx) range '40s' as rx2,
first_value(tx) RANGE '40s' as tx1,
last_value(tx) RANGE '40s' as tx2,
first_value(discards) RANGE '40s' as discards1,
last_value(discards) RANGE '40s' as discards2,
first_value(crc) RANGE '40s' as crc1,
last_value(crc) RANGE '40s' as crc2,
ts as time_window
from interface
align '30s' by (router, port)
order by ts);
router | port | rx | tx | discards | crc | time_window
--------+------+-------+-------+----------+-----+----------------------------
wdc1 | nic1 | 0 | 0 | 0 | 0 | 2025-08-21 19:20:30.000000
wdc1 | nic1 | 1156 | 62 | -60 | 1 | 2025-08-21 19:21:00.000000
wdc1 | nic1 | -1657 | -1333 | 67 | -38 | 2025-08-21 19:21:30.000000
wdc1 | nic1 | 321 | 1888 | -48 | 43 | 2025-08-21 19:22:00.000000
wdc1 | nic1 | 2188 | 230 | 31 | -68 | 2025-08-21 19:22:30.000000
wdc1 | nic1 | -2119 | -2401 | -15 | 60 | 2025-08-21 19:23:00.000000
wdc1 | nic1 | -448 | 1922 | -38 | 17 | 2025-08-21 19:23:30.000000
wdc1 | nic1 | 727 | -440 | 26 | -16 | 2025-08-21 19:24:00.000000
wdc1 | nic1 | -89 | 0 | -3 | -43 | 2025-08-21 19:24:30.000000
wdc1 | nic1 | 86 | -1389 | 11 | 0 | 2025-08-21 19:25:00.000000
wdc1 | nic1 | -760 | 2345 | 26 | -23 | 2025-08-21 19:25:30.000000
wdc1 | nic1 | 2023 | -687 | 0 | 96 | 2025-08-21 19:26:00.000000
wdc1 | nic1 | -1800 | 735 | 12 | -5 | 2025-08-21 19:26:30.000000
wdc1 | nic1 | 1593 | -1249 | -34 | -36 | 2025-08-21 19:27:00.000000
wdc1 | nic1 | 464 | 155 | 19 | -2 | 2025-08-21 19:27:30.000000
wdc1 | nic1 | -1987 | 189 | 7 | -10 | 2025-08-21 19:28:00.000000
wdc1 | nic1 | -185 | -1687 | -80 | 17 | 2025-08-21 19:28:30.000000
wdc1 | nic1 | 528 | 1776 | 95 | 22 | 2025-08-21 19:29:00.000000
wdc1 | nic1 | 208 | -1635 | -24 | -14 | 2025-08-21 19:29:30.000000
wdc1 | nic1 | 1602 | 218 | -48 | -14 | 2025-08-21 19:30:00.000000
wdc1 | nic1 | 0 | 0 | 0 | 0 | 2025-08-21 19:30:30.000000
(21 rows)
sdn=>Wei
08/21/2025, 9:17 PM2025-08-21T21:16:55.581758Z ERROR meta_client::client::procedure: An error occurred in gRPC, status: status: Internal, message: "Failed to encode DataFusion plan", details: [], metadata: MetadataMap { headers: {"content-type": "application/grpc", "x-greptime-err-code": "3001", "date": "Thu, 21 Aug 2025 21:16:55 GMT", "content-length": "0"} }
2025-08-21T21:16:55.581949Z ERROR frontend::instance: Failed to execute query: CREATE FLOW interface_rate
SINK TO interface_sink
AS SELECT
router,
port,
(last_value(rx) range '40s' - first_value(rx) range '40s') * 8 / 30 as rx,
(last_value(tx) range '40s' - first_value(tx) range '40s') * 8 / 30 as tx,
(last_value(discards) range '40s' - first_value(discards) range '40s') as discard,
(last_value(crc) range '40s' - first_value(crc) range '40s') as crc,
ts as time_window
from interface
align '30s' by (router, port)
order by ts err=0: Table operation error, at src/frontend/src/instance.rs:314:22
1: Failed to execute ddl, at /greptimedb/src/operator/src/statement/ddl.rs:494:14
2: External error, at src/meta-client/src/client.rs:264:14
3: Failed to encode DataFusion plan, at src/meta-client/src/error.rs:163:1
2025-08-21T21:16:55.582051Z ERROR servers::postgres::utils: Failed to handle postgres query, code: EngineExecuteQuery, error: Failed to execute query err=0: Failed to execute query, at /greptimedb/src/servers/src/query_handler/sql.rs:88:49
1: Table operation error, at src/frontend/src/instance.rs:314:22
2: Failed to execute ddl, at /greptimedb/src/operator/src/statement/ddl.rs:494:14
3: External error, at src/meta-client/src/client.rs:264:14
4: Failed to encode DataFusion plan, at src/meta-client/src/error.rs:163:1Wei
08/21/2025, 9:18 PMWei
08/21/2025, 9:19 PMCREATE FLOW interface_rate
SINK TO interface_sink
AS
SELECT
router,
port,
(last_value(rx) range '40s' - first_value(rx) range '40s') * 8 / 30 as rx,
(last_value(tx) range '40s' - first_value(tx) range '40s') * 8 / 30 as tx,
(last_value(discards) range '40s' - first_value(discards) range '40s') as discard,
(last_value(crc) range '40s' - first_value(crc) range '40s') as crc,
ts as time_window
from interface
align '30s' by (router, port)
order by ts;Ruihang
08/21/2025, 9:23 PMWei
08/21/2025, 9:27 PMWei
08/21/2025, 9:28 PMWei
08/21/2025, 9:42 PMCREATE FLOW interface_rate
SINK TO interface_sink
AS
SELECT
router,
port,
(max(rx) - min(rx)) * 8 / 30 as rx,
(max(tx) - min(tx)) * 8 / 30 as tx,
max(discards) - min(discards) as discards,
max(crc) - min(crc) as crc,
date_bin(INTERVAL '30 second', ts) as time_window
FROM interface
group by time_window, router, port;
when I flushed flow, 0 record,
sdn=> admin flush_flow('interface_rate');
ADMIN flush_flow('interface_rate')
------------------------------------
0
(1 row)
sdn=>Wei
08/21/2025, 9:43 PM2025-08-21T21:25:31.140613Z ERROR common_meta::ddl::utils: Failed to operate datanode, peer: peer-13(10.240.64.10:6800) err=0: External error, at src/client/src/flow.rs:35:14
1: Failed to request FlowServer 10.240.64.10:6800, code: Client specified an invalid argument, at src/client/src/flow.rs:73:43
2: Failed to encode DataFusion plan, at src/client/src/error.rs:165:1
2025-08-21T21:25:31.140657Z ERROR common_procedure::local::runner: Failed to execute procedure metasrv-procedure::CreateFlow-fdafef8d-3944-4ffa-bf79-4eb3364b08e2, retry: false, clean_poisons: false err=0: Failed to execute procedure due to external error, clean poisons: false
1: Failed to operate on datanode: peer-13(10.240.64.10:6800), at src/common/meta/src/ddl/utils.rs:64:18
2: External error, at src/client/src/flow.rs:35:14
3: Failed to request FlowServer 10.240.64.10:6800, code: Client specified an invalid argument, at src/client/src/flow.rs:73:43
4: Failed to encode DataFusion plan, at src/client/src/error.rs:165:1Wei
08/21/2025, 9:44 PMnc -v 10.240.64.10 6800
Connection to 10.240.64.10 6800 port [tcp/*] succeeded!
@@^CRuihang
08/21/2025, 9:44 PMcode: Client specified an invalid argument
Wei
08/21/2025, 9:45 PMRuihang
08/21/2025, 9:46 PMis there a different way instead of using max() - min()?it depends on the semantic data type. if the purpose is to compute instant rate on gauge data, first/last should work. let me check that clause
Wei
08/21/2025, 9:47 PMRuihang
08/21/2025, 9:49 PMWei
08/21/2025, 9:54 PMCREATE FLOW interface_rate
SINK TO interface_sink
AS
SELECT
router,
port,
(last_value(rx) - first_value(rx)) * 8 / 30 as rx,
(last_value(tx) - first_value(tx)) * 8 / 30 as tx,
last_value(discards) - first_value(discards) as discards,
last_value(crc) - first_value(crc) as crc,
date_bin(INTERVAL '30 second', ts) as time_window
FROM interface
group by time_window, router, port;Wei
08/21/2025, 9:55 PMsdn=> select * from interface_sink;;
router | port | rx | tx | discards | crc | time_window | update_at
--------+------+----+----+----------+-----+-------------+-----------
(0 rows)
sdn=> admin flush_flow('interface_rate');
ADMIN flush_flow('interface_rate')
------------------------------------
0
(1 row)
sdn=> select * from interface_sink;;
router | port | rx | tx | discards | crc | time_window | update_at
--------+------+----+----+----------+-----+-------------+-----------
(0 rows)
sdn=> select * from interface_sink;
router | port | rx | tx | discards | crc | time_window | update_at
--------+------+----+----+----------+-----+-------------+-----------
(0 rows)
sdn=> admin flush_flow('interface_rate');
ERROR: Internal error: 1003
sdn=> select * from interface_sink;
router | port | rx | tx | discards | crc | time_window | update_at
--------+------+----+----+----------+-----+-------------+-----------
(0 rows)
sdn=> admin flush_flow('interface_rate');
ADMIN flush_flow('interface_rate')
------------------------------------
0
(1 row)
sdn=> select * from interface_sink;
router | port | rx | tx | discards | crc | time_window | update_at
--------+------+----+----+----------+-----+-------------+-----------
(0 rows)
sdn=>Wei
08/21/2025, 9:58 PM2025-08-21T21:52:06.516579Z ERROR client::database: Failed to do Flight get, addr: 10.240.64.6:4001, code: The request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header, at src/client/src/error.rs:165:1
2025-08-21T21:52:11.515503Z ERROR client::database: Failed to do Flight get, addr: 10.240.64.7:4001, code: The request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header, at src/client/src/error.rs:165:1
2025-08-21T21:52:11.516318Z ERROR client::database: Failed to do Flight get, addr: 10.240.64.6:4001, code: The request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header, at src/client/src/error.rs:165:1
2025-08-21T21:52:16.512786Z ERROR flow::batching_mode::task: Failed to execute query for flow=1032 with query: Dml: op=[Insert Into] table=[greptime.sdn.interface_sink]
Projection: interface.router, interface.port, CAST(last_value(interface.rx) - first_value(interface.rx) AS Int64) * Int64(8) / Int64(30) AS rx, CAST(last_value(interface.tx) - first_value(interface.tx) AS Int64) * Int64(8) / Int64(30) AS tx, last_value(interface.discards) - first_value(interface.discards) AS discards, last_value(interface.crc) - first_value(interface.crc) AS crc, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),interface.ts) AS time_window, TimestampNanosecond(1755813121510346049, Some("+00:00")) AS update_at
Aggregate: groupBy=[[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"), interface.ts), interface.router, interface.port]], aggr=[[last_value(interface.rx), first_value(interface.rx), last_value(interface.tx), first_value(interface.tx), last_value(interface.discards), first_value(interface.discards), last_value(interface.crc), first_value(interface.crc)]]
Filter: ts >= TimestampMillisecond(1755813090000, None) AND ts < TimestampMillisecond(1755813120000, None)
TableScan: interface projection=[router, port, rx, tx, discards, crc, ts] err=0: No available frontend found after timeout: 5s, context: No available frontend found that is able to process query, at src/flow/src/batching_mode/frontend_client.rs:273:10
2025-08-21T21:52:30.606084Z ERROR client::database: Failed to do Flight get, addr: 10.240.64.7:4001, code: The request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header, at src/client/src/error.rs:165:1
2025-08-21T21:52:30.606706Z ERROR client::database: Failed to do Flight get, addr: 10.240.64.6:4001, code: The request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header, at src/client/src/error.rs:165:1
2025-08-21T21:52:35.607133Z ERROR client::database: Failed to do Flight get, addr: 10.240.64.7:4001, code: The request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header, at src/client/src/error.rs:165:1Ruihang
08/21/2025, 10:02 PMThe request does not have valid authentication credentials, source: 0: Not found http or grpc authorization header
Wei
08/21/2025, 10:03 PMWei
08/21/2025, 10:03 PM[grpc]
## The address to bind the gRPC server.
bind_addr = "10.240.64.6:4001"
## The address advertised to the metasrv, and used for connections from outside the host.
## If left empty or unset, the server will automatically use the IP address of the first network interface
## on the host, with the same port number as the one specified in `grpc.bind_addr`.
server_addr = "10.240.64.6:4001"
## The number of server worker threads.
runtime_size = 8
## Compression mode for frontend side Arrow IPC service. Available options:
## - `none`: disable all compression
## - `transport`: only enable gRPC transport compression (zstd)
## - `arrow_ipc`: only enable Arrow IPC compression (lz4)
## - `all`: enable all compression.
## Default to `none`
flight_compression = "arrow_ipc"
## gRPC server TLS options, see `mysql.tls` section.
[grpc.tls]
## TLS mode.
mode = "disable"
## Certificate file path.
## @toml2docs:none-default
cert_path = ""
## Private key file path.
## @toml2docs:none-default
key_path = ""
## Watch for Certificate and key file change and auto reload.
## For now, gRPC tls config does not support auto reload.
watch = false
## MySQL server options.Wei
08/21/2025, 10:04 PMdiscord9
08/22/2025, 4:16 AMWei
08/22/2025, 3:00 PMWei
08/22/2025, 3:51 PMWei
08/22/2025, 3:52 PMCREATE FLOW interface_rate
SINK TO interface_sink
AS
SELECT
router,
port,
(last_value(rx) - first_value(rx)) * 8 / 30 as rx,
(last_value(tx) - first_value(tx)) * 8 / 30 as tx,
last_value(discards) - first_value(discards) as discards,
last_value(crc) - first_value(crc) as crc,
date_bin(INTERVAL '60 second', ts) as time_window
FROM interface
group by time_window, router, port;Wei
08/22/2025, 3:54 PMsdn=> select * from interface;
router | port | rx | tx | discards | crc | ts
--------+------+------+------+----------+-----+----------------------------
wdc1 | nic1 | 8616 | 3143 | 44 | 46 | 2025-08-22 15:36:00.000000
wdc1 | nic1 | 2549 | 2214 | 76 | 73 | 2025-08-22 15:36:30.000000
wdc1 | nic1 | 2716 | 809 | 68 | 18 | 2025-08-22 15:37:00.000000
wdc1 | nic1 | 3620 | 4180 | 52 | 91 | 2025-08-22 15:37:30.000000
wdc1 | nic1 | 633 | 8928 | 84 | 87 | 2025-08-22 15:38:01.000000
wdc1 | nic1 | 155 | 897 | 48 | 59 | 2025-08-22 15:38:31.000000
wdc1 | nic1 | 9590 | 8811 | 92 | 81 | 2025-08-22 15:39:01.000000
wdc1 | nic1 | 4165 | 8380 | 43 | 20 | 2025-08-22 15:39:31.000000
wdc1 | nic1 | 1363 | 2405 | 2 | 89 | 2025-08-22 15:40:01.000000
wdc1 | nic1 | 1956 | 9963 | 76 | 80 | 2025-08-22 15:40:31.000000
(10 rows)
sdn=>
sdn=> select * from interface_sink;
router | port | rx | tx | discards | crc | time_window | update_at
--------+------+-------+-------+----------+-----+----------------------------+----------------------------
wdc1 | nic1 | -1617 | -247 | 32 | 27 | 2025-08-22 15:36:00.000000 | 2025-08-22 15:37:04.185000
wdc1 | nic1 | 241 | 898 | -16 | 73 | 2025-08-22 15:37:00.000000 | 2025-08-22 15:38:04.209000
wdc1 | nic1 | -127 | -2141 | -36 | -28 | 2025-08-22 15:38:00.000000 | 2025-08-22 15:39:04.236000
wdc1 | nic1 | -1446 | -114 | -49 | -61 | 2025-08-22 15:39:00.000000 | 2025-08-22 15:40:04.258000
wdc1 | nic1 | 158 | 2015 | 74 | -9 | 2025-08-22 15:40:00.000000 | 2025-08-22 15:41:04.281000
(5 rows)
sdn=>Wei
08/22/2025, 3:54 PMWei
08/22/2025, 3:54 PMsdn=> SELECT
router,
port,
(last_value(rx) range '40s' - first_value(rx) range '40s') * 8 / 30 as rx,
(last_value(tx) range '40s' - first_value(tx) range '40s') * 8 / 30 as tx,
(last_value(discards) range '40s' - first_value(discards) range '40s') as discard,
(last_value(crc) range '40s' - first_value(crc) range '40s') as crc,
ts as time_window
from interface
align '30s' by (router, port)
order by ts;
router | port | rx | tx | discard | crc | time_window
--------+------+-------+-------+---------+-----+----------------------------
wdc1 | nic1 | 0 | 0 | 0 | 0 | 2025-08-22 15:35:30.000000
wdc1 | nic1 | -1617 | -247 | 32 | 27 | 2025-08-22 15:36:00.000000
wdc1 | nic1 | 44 | -374 | -8 | -55 | 2025-08-22 15:36:30.000000
wdc1 | nic1 | 241 | 898 | -16 | 73 | 2025-08-22 15:37:00.000000
wdc1 | nic1 | -796 | 1266 | 32 | -4 | 2025-08-22 15:37:30.000000
wdc1 | nic1 | -127 | -2141 | -36 | -28 | 2025-08-22 15:38:00.000000
wdc1 | nic1 | 2516 | 2110 | 44 | 22 | 2025-08-22 15:38:30.000000
wdc1 | nic1 | -1446 | -114 | -49 | -61 | 2025-08-22 15:39:00.000000
wdc1 | nic1 | -747 | -1593 | -41 | 69 | 2025-08-22 15:39:30.000000
wdc1 | nic1 | 158 | 2015 | 74 | -9 | 2025-08-22 15:40:00.000000
wdc1 | nic1 | 0 | 0 | 0 | 0 | 2025-08-22 15:40:30.000000
(11 rows)
sdn=>Wei
08/22/2025, 3:59 PMRuihang
08/22/2025, 6:30 PMRuihang
08/22/2025, 6:31 PMtime_window is the start timestamp of 30s windowRuihang
08/22/2025, 7:38 PMRuihang
08/22/2025, 8:06 PMselect ts::bigint from interface;Ruihang
08/22/2025, 8:09 PMtql eval (1755876960, 1755877231, '30s') rate(interface[40s]);Ruihang
08/22/2025, 8:10 PMtql eval (1755876960, 1755877231, '30s') rate(interface[40s]) by (router, port)Ruihang
08/22/2025, 8:11 PM