https://linen.dev logo
Join Slack
Powered by
This message was deleted.
# general
s
This message was deleted.
p
g
@Sanyog Singh I think we have already tried that.
s
Yes @Gaurav, we tried these changes as suggested in above link but issue still persists. During our investigation, we observed that when Kafka has large volume of data (like historical data), that time we received this error. We also tried limiting the fetch size using configs 
KAFKA_MAX_REQUEST_SIZE
KAFKA_MESSAGE_MAX_BYTES
KAFKA_MAX_PARTITION_FETCH_BYTES
with no success.
It seems, it works as batch ingestion mode for Kafka ingestion. Any suggestion on how to make this work as stream ingestion.
s
What segment granularity are you using? If the Kafka data spans many time intervals a buffer is created for each segment granularity interval and when it comes time to do intermediate persist, every logical segment creates an intermediate segment file. I wonder if this is the step at which this is happening.
k
Adding on to what @Sergio Ferragut said, Please try using : https://druid.apache.org/docs/latest/development/extensions-core/kafka-supervisor-reference.html 
lateMessageRejectionPeriod
s
@Karan Kumar the requirement is to consume all data without rejecting any message.
@Sergio Ferragut this could be the cause, will investigate around segment granularity setting
s
Another thought.... the segment granularity is important, but also the order of the messages in the kafka topic will play a big role. If the message are fed to the topic ordered by time, this issue will be reduced significantly, because it will fill up buffers in order and only have a few segment granularity chunks at a time to work with.
s
@Sergio Ferragut Kafka topic is receiving bulk data as here I am trying to ingest historical data. Here is the supervisor config:
Copy code
{
  "type": "kafka",
  "spec": {
    "dataSchema": {
      "dataSource": "workflow_metadata",
      "timestampSpec": {
        "column": "timestamp",
        "format": "iso",
        "missingValue": null
      },
      "dimensionsSpec": {
        "dimensions": [
          {
            "type": "string",
            "name": "name",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "description",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "asset_catergory_id",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "asset_profile_id",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "json",
            "name": "attributes_allowed_for_update",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "json",
            "name": "attributes_exclude_for_calculation",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "carry_forward",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "dashboard_columns_view_label",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "json",
            "name": "dashboard_columns_2",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "dashboard_columns_2_view_label",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "is_allowed_dashboard_columns_2",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "is_deleted",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "is_enabled",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "is_movement_allowed_for_deactivated_instance",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "is_start_date_scheduled",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "json",
            "name": "key_attributes",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          }
        ],
        "dimensionExclusions": [
          "__time",
          "timestamp"
        ],
        "includeAllDimensions": false
      },
      "metricsSpec": [],
      "granularitySpec": {
        "type": "uniform",
        "segmentGranularity": "DAY",
        "queryGranularity": {
          "type": "none"
        },
        "rollup": false,
        "intervals": []
      },
      "transformSpec": {
        "filter": null,
        "transforms": []
      }
    },
    "ioConfig": {
      "topic": "workflow_metadata",
      "inputFormat": {
        "type": "json",
        "keepNullColumns": false,
        "assumeNewlineDelimited": false,
        "useJsonNodeReader": false
      },
      "replicas": 1,
      "taskCount": 1,
      "taskDuration": "PT1800S",
      "consumerProperties": {
        "bootstrap.servers": "xx.xxx.x.xxx:9092"
      },
      "autoScalerConfig": null,
      "pollTimeout": 100,
      "startDelay": "PT5S",
      "period": "PT30S",
      "useEarliestOffset": true,
      "completionTimeout": "PT1800S",
      "lateMessageRejectionPeriod": null,
      "earlyMessageRejectionPeriod": null,
      "lateMessageRejectionStartDateTime": null,
      "configOverrides": null,
      "idleConfig": null,
      "stream": "workflow_metadata",
      "useEarliestSequenceNumber": true
    },
    "tuningConfig": {
      "type": "kafka",
      "appendableIndexSpec": {
        "type": "onheap",
        "preserveExistingMetrics": false
      },
      "maxRowsInMemory": 500,
      "maxBytesInMemory": 0,
      "skipBytesInMemoryOverheadCheck": false,
      "maxRowsPerSegment": 5000000,
      "maxTotalRows": 1000,
      "intermediatePersistPeriod": "PT10M",
      "maxPendingPersists": 0,
      "indexSpec": {
        "bitmap": {
          "type": "roaring",
          "compressRunOnSerialization": true
        },
        "dimensionCompression": "lz4",
        "stringDictionaryEncoding": {
          "type": "utf8"
        },
        "metricCompression": "lz4",
        "longEncoding": "longs"
      },
      "indexSpecForIntermediatePersists": {
        "bitmap": {
          "type": "roaring",
          "compressRunOnSerialization": true
        },
        "dimensionCompression": "lz4",
        "stringDictionaryEncoding": {
          "type": "utf8"
        },
        "metricCompression": "lz4",
        "longEncoding": "longs"
      },
      "reportParseExceptions": false,
      "handoffConditionTimeout": 0,
      "resetOffsetAutomatically": false,
      "segmentWriteOutMediumFactory": null,
      "workerThreads": null,
      "chatThreads": null,
      "chatRetries": 8,
      "httpTimeout": "PT10S",
      "shutdownTimeout": "PT80S",
      "offsetFetchPeriod": "PT30S",
      "intermediateHandoffPeriod": "P2147483647D",
      "logParseExceptions": false,
      "maxParseExceptions": 5,
      "maxSavedParseExceptions": 0,
      "skipSequenceNumberAvailabilityCheck": false,
      "repartitionTransitionDuration": "PT120S"
    }
  },
  "context": null,
  "suspended": false
}
I tried reducing segment granularity, maxRowsInMemory, maxTotalRows, period to persist with no success.
ulimit & max_map_count set to 65535
s
How much history are you loading? Have you considered doing a batch ingestion for the history? It will be more efficient for a bulk load than trying to load all through kafka. Kafka ingestion is mainly used for incremental loading of the most recent/current events.
Your maxRowsInMemory and maxTotalRows are very low. This means that it will persist every 500 rows and then have to merge all the 500 row segments when it is time to publish. with 5 million rows per segment, the merge step will be merging 10000 segments (which is a lot, this is usually 100s at most). You also have a single task, is the data partitioned in kafka? can you use more parallelism? SQL Based Ingestion for the historic load is likely to help a lot and then continue loading the real-time data using kafka ingestion with the level of parallelism needed for the message throughput you get in real-time.
s
I have several Kafka topics where I am able to ingest historical data (upto 100k) successfully. But for couple of topics its failing. The above shared "supervisor" is trying to ingest 5k data only. I verified, its able to ingest 25 kafka messages successfully and failing with more messages. So I believe its related to message size and thus have configured "maxRowsInMemory and maxTotalRows" to a very low value to test, but no success. If the message size/number of messages, can be controlled during ingestion, that would help.
s
5k!? that should not be a problem. What is the max row size on this? Are the json fields very large? Is this 5000 rows total for the history? How many rows per DAY? If it is this small, it will make sense to use coarser granularity, perhaps MONTH. Ideally then segment granularity is such that you have millions of rows per segment granularity.
s
Yes @Sergio Ferragut, there are 50+ json fields in a row. And seems due to this heavy json parsing, it was giving error "Too many files open.." I successfully ingested data by defining data type as "string" instead of "json" for fields.
s
Could you share an example of a large row. Seems like an interesting case to add to testing. @Clint Wylie, I think you may be interested in this thread.
c
how heavily nested is the json data? json columns effectively make an internal nested column for each scalar path, if those 50+ fields are deeply nested it can result in quite a lot of columns
this is a bit of a known weak area of nested columns, particularly if there are deeply nested arrays of objects
the task logs should indicate how many nested columns are in each json column, the log message will look something like
Copy code
Column [columnName] serialized successfully with [5] nested columns.
the advantages of all of this extra processing is that at query time you can extract these nested paths using 
json_value
at the same speed as if they were standalone columns, but without having to explicitly specify flattening, but ingestion of extreme cases can be rather expensive, its an area we will keep improving over time
s
Hi @Sergio Ferragut & @Clint Wylie, sharing here a sample row:
Copy code
[{
  "_id": {
    "$oid": "63288f68ceeb94003fabcb42"
  },
  "name": "ate",
  "description": "afaf",
  "asset_category_name": "DEV_ASSET_CAT",
  "asset_profile_name": "DEV_ASSET_CAT_S2",
  "workflow_instance_label": "Shipments",
  "user_group": [
    {
      "seq": 0,
      "group_id": "bc34b3dc-baa0-4936-b7da-338cf3c02b88",
      "node_id": "447a8835-8847-412d-9ccb-771c86985eed",
      "group_name": "Initiate",
      "group_asset_attributes": [
        {
          "attribute": "Email",
          "id": "10638319-bae8-4afb-882e-c060f48c6f21",
          "options": [],
          "validations": {
            "email": true,
            "url": false,
            "character": false,
            "number": false,
            "allow_special_characters": false,
            "special_characters": "",
            "min_length": "",
            "max_length": ""
          },
          "attribute_tags": [
            {
              "attribute_tag_name": "Email",
              "attribute_tag_id": "d2685873-a74f-4fc1-9ca1-c2a335a9f1a2"
            }
          ],
          "required": null,
          "type": "textbox",
          "dependencies": [],
          "formula_array": [],
          "value": "",
          "hide_from_monitor": null,
          "hide_from_groups": null,
          "hide_from_this_group": false,
          "read_only": false,
          "is_key_column": false,
          "is_hidden": null,
          "is_quick_edit_allowed": false,
          "hide_from_tracing": false,
          "update_only": false,
          "default_value": {
            "is_allowed": false,
            "value": ""
          },
          "same_as_previous": false,
          "overridden_value": ""
        },
        {
          "attribute": "Name",
          "id": "756429c3-402b-48a8-8d12-3221bf7af571",
          "options": [],
          "validations": {
            "email": false,
            "url": false,
            "character": true,
            "number": true,
            "allow_special_characters": true,
            "special_characters": " ",
            "min_length": "1",
            "max_length": "50"
          },
          "attribute_tags": [
            {
              "attribute_tag_name": "Name",
              "attribute_tag_id": "315524b9-1cda-4567-92d6-9708b878f4f0"
            }
          ],
          "required": null,
          "type": "textbox",
          "dependencies": [],
          "formula_array": [],
          "value": "",
          "hide_from_monitor": null,
          "hide_from_groups": null,
          "hide_from_this_group": false,
          "read_only": false,
          "is_key_column": false,
          "is_hidden": null,
          "is_quick_edit_allowed": false,
          "hide_from_tracing": false,
          "update_only": false,
          "default_value": {
            "is_allowed": false,
            "value": ""
          },
          "same_as_previous": false,
          "overridden_value": ""
        },
        {
          "attribute": "DOB",
          "id": "a8705065-e3ab-4f53-a85c-a614393544f6",
          "options": [],
          "validations": {
            "allow_time_format": false,
            "date_format": "YYYY/MM/DD",
            "time_format": ""
          },
          "attribute_tags": [
            {
              "attribute_tag_name": "DOB",
              "attribute_tag_id": "bb5968e6-4a08-4414-b3d9-5f6fed113e3b"
            }
          ],
          "required": null,
          "type": "datepicker",
          "dependencies": [],
          "formula_array": [],
          "value": "",
          "hide_from_monitor": null,
          "hide_from_groups": null,
          "hide_from_this_group": false,
          "read_only": false,
          "is_key_column": false,
          "is_hidden": null,
          "is_quick_edit_allowed": false,
          "hide_from_tracing": false,
          "update_only": false,
          "default_value": {
            "is_allowed": false,
            "value": ""
          },
          "same_as_previous": false,
          "overridden_value": ""
        }
      ],
      "allow_auto_assign": false,
      "auto_send_time": "",
      "time_unit": "MINUTE",
      "hide_group_from_tracing": false,
      "allow_auto_receive": false,
      "allow_auto_dispute": false,
      "auto_dispute_condition": [],
      "delay_send": {
        "allow_delay_send": false,
        "time": 0,
        "unit": "MINUTE"
      },
      "allow_send": false,
      "allow_send_condition": [],
      "allow_receive": false,
      "allow_receive_condition": [],
      "allow_reject": false,
      "allow_reject_condition": [],
      "attribute_triggers": [],
      "attribute_validation_rules": [],
      "generic_node_mapping_id": "l88xwzcv",
      "allow_aggregation": false
    },
    {
      "seq": 1,
      "group_id": "6d24fa80-cd48-4303-9b01-3d713d13f7f9",
      "node_id": "9af497d5-0c42-4205-a215-c8e212bf0bb6",
      "group_name": "Approve",
      "group_asset_attributes": [
        {
          "attribute": "Email",
          "id": "10638319-bae8-4afb-882e-c060f48c6f21",
          "options": [],
          "validations": {
            "email": true,
            "url": false,
            "character": false,
            "number": false,
            "allow_special_characters": false,
            "special_characters": "",
            "min_length": "",
            "max_length": ""
          },
          "attribute_tags": [
            {
              "attribute_tag_name": "Email",
              "attribute_tag_id": "d2685873-a74f-4fc1-9ca1-c2a335a9f1a2"
            }
          ],
          "required": null,
          "type": "textbox",
          "dependencies": [],
          "formula_array": [],
          "value": "",
          "hide_from_monitor": null,
          "hide_from_groups": null,
          "hide_from_this_group": false,
          "read_only": false,
          "is_key_column": false,
          "is_hidden": null,
          "is_quick_edit_allowed": false,
          "hide_from_tracing": false,
          "update_only": false,
          "default_value": {
            "is_allowed": false,
            "value": ""
          },
          "same_as_previous": false,
          "overridden_value": ""
        },
        {
          "attribute": "Name",
          "id": "756429c3-402b-48a8-8d12-3221bf7af571",
          "options": [],
          "validations": {
            "email": false,
            "url": false,
            "character": true,
            "number": true,
            "allow_special_characters": true,
            "special_characters": " ",
            "min_length": "1",
            "max_length": "50"
          },
          "attribute_tags": [
            {
              "attribute_tag_name": "Name",
              "attribute_tag_id": "315524b9-1cda-4567-92d6-9708b878f4f0"
            }
          ],
          "required": null,
          "type": "textbox",
          "dependencies": [],
          "formula_array": [],
          "value": "",
          "hide_from_monitor": null,
          "hide_from_groups": null,
          "hide_from_this_group": false,
          "read_only": false,
          "is_key_column": false,
          "is_hidden": null,
          "is_quick_edit_allowed": false,
          "hide_from_tracing": false,
          "update_only": false,
          "default_value": {
            "is_allowed": false,
            "value": ""
          },
          "same_as_previous": false,
          "overridden_value": ""
        }
      ],
      "allow_auto_assign": false,
      "auto_send_time": "",
      "time_unit": "MINUTE",
      "hide_group_from_tracing": false,
      "allow_auto_receive": false,
      "allow_auto_dispute": false,
      "auto_dispute_condition": [],
      "delay_send": {
        "allow_delay_send": false,
        "time": 0,
        "unit": "MINUTE"
      },
      "allow_send": false,
      "allow_send_condition": [],
      "allow_receive": false,
      "allow_receive_condition": [],
      "allow_reject": false,
      "allow_reject_condition": [],
      "attribute_triggers": [],
      "attribute_validation_rules": [],
      "generic_node_mapping_id": "l88xz3hy"
    }
  ],
  "asset_rejection": {
    "rejection_allowed": true,
    "comment_required": false
  },
  "is_movement_allowed_for_deactivated_instance": false,
  "attributes_exclude_for_calculation": [],
  "attributes_allowed_for_update": [],
  "is_start_date_scheduled": false,
  "start_date": {
    "$date": {
      "$numberLong": "1663602536884"
    }
  },
  "end_date": {
    "$date": {
      "$numberLong": "1979145000000"
    }
  },
  "instance_refresh_conditions": "",
  "available_dashboard_columns": [
    {
      "id": "10638319-bae8-4afb-882e-c060f48c6f21",
      "attribute": "Email",
      "type": "textbox",
      "is_key_column": false,
      "is_hidden_dashboard_column": false
    },
    {
      "id": "756429c3-402b-48a8-8d12-3221bf7af571",
      "attribute": "Name",
      "type": "textbox",
      "is_key_column": false,
      "is_hidden_dashboard_column": false
    }
  ],
  "dashboard_columns": [
    {
      "column_id": "delivery_status",
      "column_header": "Delivery status",
      "column_name": "delivery_status",
      "control_type": "dropdown",
      "is_default_column": true,
      "is_required": true
    },
    {
      "column_id": "actions",
      "column_header": "Actions",
      "column_name": "actions",
      "control_type": "",
      "is_default_column": true,
      "is_required": true
    }
  ],
  "dashboard_columns_2": [],
  "is_allowed_dashboard_columns_2": false,
  "last_user_group_nodes": [
    "9af497d5-0c42-4205-a215-c8e212bf0bb6"
  ],
  "workflow_nodes": [
    {
      "is_highlighted": false,
      "node_type": "group",
      "node_id": "l88xwzcv",
      "node_name": "Initiate",
      "display_name": "Initiate",
      "short_description": "",
      "node_sub_type": "initiating_wf_node",
      "display_attributes": {
        "x": 250,
        "y": 50
      },
      "last_functional_node_execution_trigger": 4,
      "outgoing_workflow_edge": {
        "edge_id": "l88xwzcu",
        "color_code": "#3367D6",
        "target_node": "l88xz3hy"
      },
      "group_id": "bc34b3dc-baa0-4936-b7da-338cf3c02b88",
      "user_group_node_mapping_id": "447a8835-8847-412d-9ccb-771c86985eed"
    },
    {
      "is_highlighted": false,
      "node_type": "group",
      "node_id": "l88xz3hy",
      "node_name": "Approve",
      "display_name": "Approve",
      "short_description": "",
      "node_sub_type": "",
      "display_attributes": {
        "x": 500,
        "y": 50
      },
      "last_functional_node_execution_trigger": 4,
      "outgoing_workflow_edge": {
        "edge_id": "l88xz3hx",
        "color_code": "",
        "target_node": ""
      },
      "group_id": "6d24fa80-cd48-4303-9b01-3d713d13f7f9",
      "user_group_node_mapping_id": "9af497d5-0c42-4205-a215-c8e212bf0bb6"
    }
  ],
  "group_instance_visibility_settings": {
    "bc34b3dc-baa0-4936-b7da-338cf3c02b88": {
      "view_mode": 2
    },
    "6d24fa80-cd48-4303-9b01-3d713d13f7f9": {
      "view_mode": 2
    }
  },
  "key_attributes": [],
  "navigation_attribute": "tag_id",
  "dashboard_columns_view_label": "Pending for acceptance",
  "dashboard_columns_2_view_label": "Dashboard 2",
  "carry_forward": true,
  "asset_category": "5e5d8df6-a52f-48cc-8d8b-68d2ad197d63",
  "asset_profile": "53a6e631-4eed-4fec-8e4c-4d245862c884",
  "transaction_id": "a69001fc6e6d9b3ca92cb160fe836b18fd1b548227616056ff6d2c0bf5caa932",
  "created_at": {
    "$date": {
      "$numberLong": "1663602536884"
    }
  },
  "updated_at": {
    "$date": {
      "$numberLong": "1663602536884"
    }
  },
  "workflow_status": 1,
  "expressions": [],
  "status_forcefully_changed": false,
  "created_by": "d21df645-5407-4b4f-8c5d-b8670b306440",
  "is_deleted": false,
  "is_enabled": true,
  "id": "d10a808c-ecd6-402f-b300-c84f8198f843",
  "__v": 0
}]
2 Views
Open in Slack
PreviousNext
Powered by