About Data sync between Mysql and starrocks. below is my docker config.(development phase): incredible performance at normal and complex queries but one problem is there data sync between mysql and starrocks via debezium is 5s. (cause routine load for starrocks table is 5s) then it reads kafka topics. Can I expect any improvisation so I can reduce these time window. I used manual script for importing existing data from mysql to starrocks for controlled resource utilization. below is my debezium connector config. for live data sync between some of the tables . { "name": "mysql-pulse-cdc", "config": { "connector.class": "io.debezium.connector.mysql.MySqlConnector", "tasks.max": "1", "database.hostname": "192.168.0.18", "database.port": "3306", "database.user": "debezium", "database.password": "Debezium1234!", "database.server.id": "5430", "database.server.name": "pulse", "database.include.list": "pulse", "topic.prefix": "starrocks", "table.include.list": "pulse.client,pulse.client_extra_table,pulse.company,pulse.companyext,pulse.client_document,pulse.clientZipFile,pulse.client_affiliate_status,pulse.client_agreement,pulse.client_and_agreement,pulse.client_and_subscription_package,pulse.client_and_team,pulse.client_appointments,pulse.client_assignedto,pulse.client_audit,pulse.client_activities,pulse.client_billing_notes,pulse.client_billing_notes_audit,pulse.client_company_payment_processors,pulse.client_credit_course,pulse.client_custom_field_audit,pulse.client_custom_field_value,pulse.client_customer_Disposition,pulse.client_customer_Folder,pulse.client_customer_Workflow,pulse.client_customer_child_instructions,pulse.client_customer_instructions,pulse.client_customer_instructions_groups,pulse.client_customer_status,pulse.client_deleted_paid_invoice,pulse.client_dispute_result_types,pulse.client_document_name,pulse.client_enum,pulse.client_hard_deleted,pulse.client_having_latest_message,pulse.client_historic_Report_Import,pulse.client_invoice,pulse.client_invoice_new_product,pulse.client_invoice_product,pulse.client_item_audit,pulse.client_notifications,pulse.client_payment_processors,pulse.client_processing_issue,pulse.client_processing_status,pulse.client_provider_relation,pulse.client_public_status,pulse.client_public_type,pulse.client_reasons_group,pulse.client_report_order_kyes,pulse.client_report_order_supported_products,pulse.client_sales_representative,pulse.client_service_details,pulse.client_specific_invoice,pulse.client_specific_new_invoice,pulse.client_specific_new_invoice_audit,pulse.client_status_type,pulse.client_summary,pulse.clients_array_security_que_urls,pulse.affiliate,pulse.affiliate_activities,pulse.affiliate_agreement,pulse.affiliate_and_agreement,pulse.affiliate_audit,pulse.affiliatecommisionpayout,pulse.affiliate_document,pulse.affiliate_email_histroy,pulse.affiliate_management,pulse.AffiliateMonthlyPayoutScheduleProcess,pulse.affiliate_portal_setting,pulse.affiliate_status,pulse.affiliateType,pulse.credit_score,pulse.imported_and_saved_date,pulse.dispute_rounds,pulse.disputeEXtendedRounds,pulse.credit_score_temp,pulse.team,pulse.saved_letter,pulse.dispute_payment_history_table,pulse.identityIQ_dispute_and_credit_bureau,pulse.disputeforfirstimport,pulse.identityIQfirstimportdisputeandcreditbureau,pulse.dispute_with_existing_data,pulse.identityiq_dispute_with_exisiting_data,pulse.dispute,pulse.creditor,pulse.dispute_status,pulse.action_plan,pulse.actionplan_credit_resource_links,pulse.actionplan_credit_resource_links_mapping,pulse.action_plan_credt_summary_details,pulse.action_plan_mapping,pulse.action_plan_summary_category,pulse.action_plan_texts,pulse.action_plan_user_goals,pulse.fox_admin_settings,pulse.importwise_client_action_plan,pulse.scorefusion_affiliate_activity,pulse.scorefusion_affiliate_commission_management,pulse.scorefusion_company_jwt_tokens,pulse.scorefusion_company_settings,pulse.scorefusion_customer_and_low_credits_summary,pulse.scorefusion_customer_details,pulse.scorefusion_customer_enrollment_codes,pulse.scorefusion_customer_grid_summary,pulse.scorefusion_customer_identity_verification,pulse.scorefusion_customer_product_codes,pulse.scorefusion_grid_summary,pulse.scorefusion_mass_payment_activity,pulse.scorefusion_payment_gateway_setting,pulse.scorefusion_portal_setting,pulse.scorefusion_security_answer_failed_status,pulse.scorefusion_signup_step,pulse.scorefusion_subscription_packages,pulse.team_slot_plan_master,pulse.team_slot_purchase_history,pulse.batch_print_history,pulse.billing_setup,pulse.billing_card_and_bank_details,pulse.billing_setup_audit,pulse.company_custom_field,pulse.company_custom_field_schema_data,pulse.constants,pulse.credit_bureau,pulse.credit_summary,pulse.dispute_instruction,pulse.dispute_reason,pulse.email_from_domain,pulse.deleted_dispute_item_info,pulse.Constants,pulse.mobile_app,pulse.mobile_app_faq,pulse.mobile_app_invitation_sent_history,pulse.mobile_app_settings_cro,pulse.mobile_chat_attachments,pulse.mobile_client,pulse.mobile_client_feedback,pulse.mobile_client_login_logout_history,pulse.mobile_client_otp_request,pulse.mobile_team,pulse.mobile_team_login_logout_history,pulse.mobile_team_notifications,pulse.mobileapp_notifications,pulse.dispute_tab_payment_history_new,pulse.dispute_tab_payment_history_new_with_existing_data", "schema.history.internal.kafka.bootstrap.servers": "kafka:9092", "schema.history.internal.kafka.topic": "_schema_history.starrocks.pulse", "include.schema.changes": "false", "snapshot.mode": "schema_only", "snapshot.locking.mode": "none", "decimal.handling.mode": "string", "time.precision.mode": "connect", "binary.handling.mode": "base64", "bigint.unsigned.handling.mode": "long", "tombstones.on.delete": "false", "provide.transaction.metadata": "false", "database.connectionTimeZone": "UTC", "database.connection.adapter": "mysql", "database.history.skip.unparseable.ddl": "true", "errors.tolerance": "all", "errors.log.enable": "true", "errors.log.include.messages": "true", "key.converter": "org.apache.kafka.connect.json.JsonConverter", "key.converter.schemas.enable": false, "value.converter": "org.apache.kafka.connect.json.JsonConverter", "value.converter.schemas.enable": false, "transforms": "unwrap,route", "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", "transforms.unwrap.drop.tombstones": "true", "transforms.unwrap.delete.handling.mode": "rewrite", "transforms.unwrap.add.fields": "op,ts_ms", "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", "transforms.route.regex": "starrocks\\.pulse\\.(.*)", "transforms.route.replacement": "starrocks.pulse.$1", "column.exclude.list": "pulse.saved_letter.saved_letter_text" } } This time window is bearable, but looking for any possible enhancement. services: # ── ZooKeeper ───────────────────────────────────────────────────────── zookeeper: image: zookeeper:3.7.2 container_name: sr-zookeeper restart: unless-stopped ports: - "2181:2181" environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 volumes: - /opt/starrocks-cdc/data/zookeeper:/data - /opt/starrocks-cdc/data/zookeeper-logs:/datalog networks: sr_net: ipv4_address: 172.21.0.10 healthcheck: test: ["CMD", "zkServer.sh", "status"] interval: 10s timeout: 5s retries: 5 # ── Kafka ────────────────────────────────────────────────────────────── kafka: image: confluentinc/cp-kafka:7.6.0 container_name: sr-kafka restart: unless-stopped ports: - "9092:9092" environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXTPLAINTEXT,PLAINTEXT HOSTPLAINTEXT KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:9093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false" # Allow large CDC events (some MySQL rows are >1 MB after JSON envelope) KAFKA_MESSAGE_MAX_BYTES: 10485760 KAFKA_REPLICA_FETCH_MAX_BYTES: 10485760 # Retention: 7 days, no size cap (gives you replay buffer if SR goes down) KAFKA_LOG_RETENTION_MS: 604800000 KAFKA_LOG_RETENTION_HOURS: -1 KAFKA_LOG_RETENTION_BYTES: -1 KAFKA_LOG_CLEANUP_POLICY: "delete" KAFKA_LOG_SEGMENT_BYTES: 536870912 KAFKA_LOG_ROLL_MS: 86400000 KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000 KAFKA_HEAP_OPTS: "-Xms512m -Xmx1g" volumes: - /opt/starrocks-cdc/data/kafka:/var/lib/kafka/data networks: sr_net: ipv4_address: 172.21.0.11 depends_on: zookeeper: condition: service_healthy healthcheck: test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "kafka:9092"] interval: 15s timeout: 10s retries: 5 # ── Kafka Connect + Debezium ─────────────────────────────────────────── kafka-connect: image: debezium/connect:3.0.0.Final container_name: sr-debezium restart: unless-stopped ports: - "8083:8083" environment: BOOTSTRAP_SERVERS: kafka:9092 GROUP_ID: starrocks-cdc-group CONFIG_STORAGE_TOPIC: _debezium_configs OFFSET_STORAGE_TOPIC: _debezium_offsets STATUS_STORAGE_TOPIC: _debezium_statuses CONFIG_STORAGE_REPLICATION_FACTOR: 1 OFFSET_STORAGE_REPLICATION_FACTOR: 1 STATUS_STORAGE_REPLICATION_FACTOR: 1 # Advertise hostname not container IP (prevents task failures after restart) CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect # Flush offsets aggressively to limit replay window after a crash OFFSET_FLUSH_INTERVAL_MS: 5000 KAFKA_MESSAGE_MAX_BYTES: 10485760 KAFKA_REPLICA_FETCH_MAX_BYTES: 10485760 KAFKA_HEAP_OPTS: "-Xms512m -Xmx2g" # Producer side — match Kafka's max message size CONNECT_PRODUCER_MAX_REQUEST_SIZE: 10485760 CONNECT_PRODUCER_BUFFER_MEMORY: 67108864 CONNECT_PRODUCER_COMPRESSION_TYPE: lz4 volumes: - /opt/starrocks-cdc/data/debezium-offsets:/kafka/offsets networks: sr_net: ipv4_address: 172.21.0.12 depends_on: kafka: condition: service_healthy healthcheck: test: ["CMD-SHELL", "curl -sf localhost/connectors || exit 1"] interval: 15s timeout: 10s retries: 10 # ── StarRocks (FE+BE all-in-one) ────────────────────────────────────── starrocks: build: ./starrocks-image container_name: sr-starrocks hostname: 172.21.0.13 restart: unless-stopped environment: MYSQL_PWD: "starrocks" ports: - "9030:9030" # MySQL/JDBC query port - "8030:8030" # FE HTTP (for Stream Load / web) - "8040:8040" # BE HTTP volumes: - ./starrocks/fe.conf:/data/deploy/starrocks/fe/conf/fe.conf - ./starrocks/be.conf:/data/deploy/starrocks/be/conf/be.conf - /opt/starrocks-cdc/data/starrocks-meta:/data/deploy/starrocks/fe/meta - /opt/starrocks-cdc/data/starrocks-storage:/data/deploy/starrocks/be/storage # mem_limit: 10g networks: sr_net: ipv4_address: 172.21.0.13 depends_on: kafka: condition: service_healthy # ── Network ──────────────────────────────────────────────────────────── networks: sr_net: driver: bridge ipam: config: - subnet: 172.21.0.0/24 gateway: 172.21.0.1