SIGSEGV

__memcpy_evex_unaligned_erms

starrocks::BinaryColumnBase<unsigned int>::filter_range

OlapChunkSource

filter_range

WHERE

SIGSEGV

memcpy

BinaryColumn

VARCHAR

STRING

JSON

unsigned int

query_id

01a0b276-0ec1-72c9-b9d8-425138bfe789

fe.audit.log

ARRAY

ARRAY

MAP

filter_range

ADMIN CHECK TABLET (tablet_id)

pipeline_max_batch_size

sql
        SET pipeline_max_batch_size = 1024;

sql
        SET enable_runtime_filter = false;

EXPLAIN ANALYZE

sql
SELECT app_code, 
       COALESCE(get_json_string(OUTPUT, '$.CBSCORE'), ...) AS CB_Score, 
       ..., 
       get_json_string(UPPER(OUTPUT), '$.CBHITTYPE') AS CBHITTYPE 
FROM dmapp.dtm_los_log_cbscore

SegmentIterator

BinaryColumnBase<unsigned int>::filter_range

SIGSEGV

memcpy

OUTPUT

BinaryColumnBase<unsigned int>

UPPER(OUTPUT)

get_json_string

OUTPUT

VARCHAR

UPPER(OUTPUT)

get_json_string(UPPER(OUTPUT), ...)

UPPER(get_json_string(OUTPUT, '$.CBHITTYPE'))

sql
        SET pipeline_max_batch_size = 1024; -- Default is usually 4096

sql
        ADMIN CHECK TABLET (tablet_id); -- Get tablet_id from BE logs if possible

UPPER(get_json_string(...))

OUTPUT

OUTPUT

OUTPUT

CBHitType

get_json_string

coalesce

memcpy

ADD COLUMN col_name data_type AS expr

VARCHAR(50)

get_json_string

COALESCE

get_json_string

dtm_los_log_cbscore

OUTPUT

sql
SHOW ALTER TABLE COLUMN;

CBHitType

COALESCE(get_json_string(...))

01a0b36a-7fa6-72e6-b55d-fdcea87835ab

SIGSEGV

std::_Sp_counted_base<...>::_M_release()

SpillProcessChannel::close

SpillProcessChannel

SpillProcessOperator

mem_limit

sql
        SET mem_limit = 16106127360; -- Increase to 15GB or higher

Out of Memory

sql
        SET enable_spill = false;

close()

filter_range

M_release

enable_spill

SHOW VARIABLES

sql
SHOW VARIABLES LIKE 'enable_spill';

sql
SHOW GLOBAL VARIABLES LIKE 'enable_spill';

sql
SHOW VARIABLES LIKE '%spill%';

spill_mode

auto

enable_spill_to_remote_storage

sql
    SET enable_spill = true;

sql
    SET GLOBAL enable_spill = true;

be.conf

spill_local_storage_dir

QuerySource=MV

default_mv_wg

default_mv_wg work group to pass this error

default_mv_wg

default_mv_wg

ALTER RESOURCE GROUP

sql
ALTER RESOURCE GROUP default_mv_wg WITH (
    "concurrency_limit" = "1",       -- Only allow 1 MV refresh at a time
    "mem_limit" = "20%",             -- Limit total group memory to 20% of BE RAM
    "big_query_mem_limit" = "10GB"   -- Kill any single refresh task that exceeds 10GB
);

concurrency_limit

mem_limit

big_query_mem_limit

fe.audit.log

SpillProcessChannel

sql
SELECT * FROM information_schema.resource_groups WHERE name = 'default_mv_wg';

sql
-- 1. Create a "low-priority" group
CREATE RESOURCE GROUP mv_heavy_task WITH ("mem_limit" = "10%", "concurrency_limit" = "1");

-- 2. Assign the specific MV to it (requires recreating or altering MV properties)
ALTER MATERIALIZED VIEW your_mv_name SET ("resource_group" = "mv_heavy_task");

default_mv_wg

null

sql
ALTER RESOURCE GROUP default_mv_wg WITH (
    "concurrency_limit" = "1",
    "mem_limit" = "20%",
    "big_query_mem_limit" = "10G",
    "spill_mem_limit_threshold" = "90%"
);

concurrency_limit

mem_limit

bui_wg

cpm_wg

big_query_mem_limit

spill_mem_limit_threshold

big_query_mem_limit

sql
SELECT * FROM information_schema.resource_groups WHERE name = 'default_mv_wg';

Memory exceed limit

big_query_mem_limit

'big_query_mem_limit' = '1073741824'

fe.audit.log

ErrorCode

mem_limit

mem_limit

'20%'

big_query_mem_limit

'1073741824'

'big_query_mem_limit' = '1073741824' is 1GB how about 10GB

big_query_mem_limit

'big_query_mem_limit' = '10737418240'

sql
ALTER RESOURCE GROUP default_mv_wg WITH (
    "big_query_mem_limit" = "10737418240"
);

mem_limit

"20%"

"10G"

big_query_mem_limit

DROP DATABASE

fe/log/fe.audit.log

DROP DATABASE

sql
    RECOVER DATABASE your_db_name;

DROP DATABASE ... FORCE

SHOW FRONTENDS;

Alive

true

LEADER

fe/log/fe.log

ReplicaWriteException

got interrupt exception... when replay journal

sql
    SET CATALOG default_catalog;
    SHOW DATABASES;

SHOW DATABASES

root

admin

meta_dir

RESTORE

RECOVER DATABASE <name>;

fe.audit.log

DROP

fe.log

meta_dir

SET

SET

CURDATE() - 7

INTERVAL

UPDATE

dmapp.dtm_los_approval

UPDATE

sql
WITH strategy AS (
    SELECT id, strategy_decision, strategy_name, strategy_version
    FROM losrep.approval
    WHERE created_date > DATE_SUB(CURDATE(), INTERVAL 7 DAY)
      AND strategy_name IS NOT NULL 
      AND strategy_name <> ''
)
UPDATE dmapp.dtm_los_approval
SET strategy_decision = strategy.strategy_decision, -- Explicitly reference the CTE
    strategy_name = strategy.strategy_name,
    strategy_version = strategy.strategy_version
FROM strategy
WHERE dmapp.dtm_los_approval.id = strategy.id;

SHOW CREATE TABLE dmapp.dtm_los_approval;

PRIMARY KEY (id)

FROM

WHERE

id

id

losrep.approval

SUBMIT TASK

SUBMIT TASK

INSERT

CREATE TABLE AS SELECT

UPDATE

INSERT

dmapp.dtm_los_approval

UPDATE

INSERT INTO ... SELECT

id

sql
SUBMIT TASK daily_approval_update
SCHEDULE EVERY(INTERVAL 1 DAY)
AS
INSERT INTO dmapp.dtm_los_approval (id, strategy_decision, strategy_name, strategy_version)
SELECT 
    id, 
    strategy_decision, 
    strategy_name, 
    strategy_version
FROM losrep.approval
WHERE created_date > DATE_SUB(CURDATE(), INTERVAL 7 DAY)
  AND strategy_name IS NOT NULL 
  AND strategy_name <> '';

INSERT

UPDATE

SCHEDULE EVERY(INTERVAL 1 DAY)

sql
    -- View the task definition
    SELECT * FROM INFORMATION_SCHEMA.tasks WHERE task_name = 'daily_approval_update';

    -- View the execution history (Success/Failure)
    SELECT * FROM INFORMATION_SCHEMA.task_runs WHERE task_name = 'daily_approval_update' ORDER BY create_time DESC;

ALTER TASK daily_approval_update RESUME;

DROP TASK daily_approval_update;

EVERY(INTERVAL ...)

SUBMIT TASK

date_trunc

CURDATE()

YYYY-MM-DD 00:00:00

WHERE created_date < date_trunc('day', now())

WHERE created_date < CURDATE()

SUBMIT TASK

START

sql
SUBMIT TASK daily_approval_update
-- Start at midnight on 2026-09-21 and repeat every 1 day
SCHEDULE START('2026-09-21 00:00:00') EVERY(INTERVAL 1 DAY)
AS
INSERT INTO dmapp.dtm_los_approval (id, strategy_decision, strategy_name, strategy_version)
SELECT 
    id, 
    strategy_decision, 
    strategy_name, 
    strategy_version
FROM losrep.approval
WHERE created_date < date_trunc('day', now()) -- Records before today's midnight
  AND strategy_name IS NOT NULL 
  AND strategy_name <> '';

START('YYYY-MM-DD HH:mm:ss')

EVERY(INTERVAL 1 DAY)

date_trunc('day', now())

00:00:00

START

INSERT

NOT MATCH

sql
    ADMIN SET FRONTEND CONFIG ("tablet_sched_max_scheduling_tablets" = "0");
    ADMIN SET FRONTEND CONFIG ("tablet_sched_max_balancing_tablets" = "0");
    ADMIN SET FRONTEND CONFIG ("disable_balance" = "true");
    ADMIN SET FRONTEND CONFIG ("disable_colocate_balance" = "true");

sql
    SET GLOBAL enable_rewrite_simple_agg_to_meta_scan = false;

fe.conf

ignore_unknown_log_id = true

ALTER SYSTEM CREATE IMAGE;

stop_be.sh --graceful

ignore_unknown_log_id = false

fe.conf

meta

score1

OUTPUT

result

get_json_string

->

get_json_string

VARCHAR

$.result.score1

NULL

sql
UPDATE dtm_log_score 
SET score1 = get_json_string(OUTPUT, '$.result.score1')
WHERE get_json_string(OUTPUT, '$.result.score1') IS NOT NULL;

sql
SELECT 
    OUTPUT,
    get_json_string(OUTPUT, '$.result.score1') AS extracted_score1
FROM dtm_log_score
WHERE json_exists(OUTPUT, '$.result.score1');

->

OUTPUT

CAST

score1

sql
UPDATE dtm_log_score 
SET score1 = CAST(OUTPUT->'result'->'score1' AS DOUBLE)
WHERE OUTPUT->'result'->'score1' IS NOT NULL;

get_json_string(json_str, path)

json_exists(json_expr, path)

1

CAST(... AS TYPE)

score1

DOUBLE

DECIMAL

result

get_json_string(OUTPUT, '$.result.score1')

OUTPUT->'result'->'score1'

score1

dtm_log_score

"0.881"

VARCHAR

DOUBLE

sql
ALTER TABLE dtm_log_score 
ADD COLUMN score1 VARCHAR(64) DEFAULT NULL;

UPDATE

score1

OUTPUT

UPDATE

sql
ALTER TABLE dtm_log_score 
ADD COLUMN score1 VARCHAR(64) AS get_json_string(OUTPUT, '$.result.score1');

score1

score1

REPLACE

NONE

sql
        ALTER TABLE dtm_log_score ADD COLUMN score1 VARCHAR(64) REPLACE DEFAULT NULL;

AFTER column_name

sql
    SHOW ALTER TABLE COLUMN;

score1

"0.881"

DECIMAL(P, S)

score1 > 0.880

DECIMAL

DECIMAL(10, 3)

9999999.999

sql
ALTER TABLE dtm_log_score 
ADD COLUMN score1 DECIMAL(10, 3) 
AS CAST(get_json_string(OUTPUT, '$.result.score1') AS DECIMAL(10, 3));

DOUBLE

DOUBLE

DOUBLE

sql
ALTER TABLE dtm_log_score 
ADD COLUMN score1 DOUBLE 
AS CAST(get_json_string(OUTPUT, '$.result.score1') AS DOUBLE);

"0.881"

DECIMAL(10, 3)

0.881000000001

CAST

NULL

score1

"error"

NULL

CAST(... AS DOUBLE)

CAST(... AS DECIMAL)

"error"

"N/A"

""

CAST

NULL

score1

"error"

NULL

score1

sql
-- If OUTPUT is: {"result": {"score1": "error"}}
-- This expression:
CAST(get_json_string(OUTPUT, '$.result.score1') AS DOUBLE) 
-- Returns: NULL

VARCHAR

sql
ALTER TABLE dtm_log_score 
ADD COLUMN score1 VARCHAR(64) 
AS get_json_string(OUTPUT, '$.result.score1');

-1

ifnull

sql
ALTER TABLE dtm_log_score 
ADD COLUMN score1 DOUBLE 
AS ifnull(CAST(get_json_string(OUTPUT, '$.result.score1') AS DOUBLE), -1.0);

"0.881"

0.881

"error"

NULL

NULL

UPDATE

UPDATE

AS (...)

UPDATE

INSERT

UPDATE

materialized column 'score' can not be specified

ALTER

UPDATE

get_json_double

sql
-- Add both columns in one go (Recommended)
ALTER TABLE dmapp.dtm_log_score 
  ADD COLUMN score DOUBLE AS get_json_double(OUTPUT, '$.result.score'),
  ADD COLUMN score1 DOUBLE AS get_json_double(OUTPUT, '$.result.score1');

ALTER

sql
    SHOW ALTER TABLE COLUMN;

FINISHED

DECIMAL(10,3)

DOUBLE

AGGREGATE KEY

DUPLICATE KEY

PRIMARY KEY

ALTER TABLE

UPDATE

dtm_log_score

SYS_RUN_DATE

ID

APP_CODE

INPUT

OUTPUT

REQUEST_TIME

RESPOND_TIME

SCORE_STAGE

TYPE

score

OUTPUT

score1

OUTPUT

SYS_RUN_DATE

ID

APP_CODE

DUPLICATE KEY

PRIMARY KEY

DUPLICATE KEY

SYS_RUN_DATE

JSON

VARCHAR(65533)

CAST(get_json_string...)

VARCHAR

JSON

ID

decimal(38, 20)

ID

BIGINT

VARCHAR

sql
CREATE TABLE `dtm_log_score` (
  `SYS_RUN_DATE` datetime NOT NULL COMMENT "",
  `ID` BIGINT NOT NULL COMMENT "", -- Changed from Decimal to Bigint
  `APP_CODE` varchar(20) NULL COMMENT "",
  `INPUT` JSON NULL COMMENT "",     -- Changed to JSON type
  `OUTPUT` JSON NULL COMMENT "",    -- Changed to JSON type
  `REQUEST_TIME` datetime NULL COMMENT "",
  `RESPOND_TIME` datetime NULL COMMENT "",
  `SCORE_STAGE` varchar(100) NULL COMMENT "",
  `TYPE` varchar(100) NULL COMMENT "",
  -- Simplified Generated Columns using get_json_double
  `score` double AS get_json_double(`OUTPUT`, '$.result.score') COMMENT "",
  `score1` double AS get_json_double(`OUTPUT`, '$.result.score1') COMMENT ""
) ENGINE=OLAP
DUPLICATE KEY(`SYS_RUN_DATE`, `ID`, `APP_CODE`) -- Logs use Duplicate Key
PARTITION BY date_trunc('day', `SYS_RUN_DATE`)   -- Crucial for performance
DISTRIBUTED BY HASH(`ID`) BUCKETS 16             -- Better distribution than APP_CODE
PROPERTIES (
    "compression" = "LZ4",
    "fast_schema_evolution" = "true",
    "replication_num" = "2"
);

WHERE SYS_RUN_DATE >= '2023-10-01'

ID

APP_CODE

APP_CODE

ID

RANDOM

CAST(get_json_string)

PRIMARY KEY

ID

BUCKETS

sql
DISTRIBUTED BY HASH(`ID`) -- Let StarRocks decide the number of buckets

ID

BUCKETS 1

BUCKETS 2

INSERT INTO ... VALUES

VALUES

VALUES

INSERT INTO ... SELECT

INSERT INTO ... SELECT

sql
INSERT INTO dwh.af_extract_mapping 
(ID, PROJECT_NAME, AF_SRC_CONN_ID, SRC_SCHEMA, SRC_TABLE, AF_DEST_CONN_ID, DEST_SCHEMA, DEST_TABLE, DEST_PRE_SQL, DEST_CHECKPOINT_SQL, SRC_EXTRACT_SQL, DEST_POST_SQL, DESCRIPTION, STATUS_FLAG, DEST_STAGING_TABLE, BATCH_SIZE, DEPENDS_ON)
SELECT 
    IFNULL(MAX(ID), 228) + 1, -- Đây là kết quả SQL động
    'DMAPP_DATA_02', 
    'DTM_PRO', 
    'QTDM', 
    'CIC_SKIPPING_RISK_INDICATORS_AVC', 
    'SR_DMAPP_PRO', 
    'qtdm', 
    'CIC_SKIPPING_RISK_INDICATORS_AVC', 
    'truncate table qtdm.CIC_SKIPPING_RISK_INDICATORS_AVC', 
    NULL, 
    'SELECT * FROM QTDM.CIC_SKIPPING_RISK_INDICATORS_AVC', 
    NULL, 
    NULL, 
    1, 
    NULL, 
    100000, 
    NULL
FROM dwh.af_extract_mapping;

SELECT

SELECT

af_extract_mapping

MAX(ID) + 1