https://openobserve.ai/ logo
Join Slack
Powered by
Hi team, we have identified two issues: 1. Queryi...
# general
m
Hi team, we have identified two issues: 1. Querying and downloading large volumes of data When running queries for data from one month prior, where the Delta Scan Size is around 469.63 GB, there is an issue when navigating beyond the first 50 results. When I try to load the next 50 records, the entire interface freezes and eventually crashes, which means I’m unable to retrieve the remaining data points for the one-month period. A similar issue occurs when trying to download a larger number of records. For example, when attempting to download 1,000 records, the download appears to time out and fails completely. It is understandable if the query or download takes some time due to the volume of data, but it should ideally continue processing rather than freezing or timing out halfway through. 2. Unnecessary data being downloaded when querying historical logs When searching for logs from a specific historical date, for example 7 August, it appears that the system is still downloading data starting from the current/latest time rather than only retrieving the relevant data for the requested date. This results in a significant amount of unnecessary data being downloaded and increases the time required to complete the query. Considering how the data is organised in S3, it seems possible to identify and retrieve only the relevant data partitions/files for the requested date range instead of downloading data from the current time backwards. Could you please look into both of these issues?
s
Could you also share your OpenObserve version, deployment setup? That’ll help us investigate this more accurately.
m
I am using v0.92.0 in cluster mode
Copy code
ZO_APP_NAME: "openobserve"
  ZO_CLUSTER_NAME: "o2"
  ZO_INSTANCE_NAME: ""
  ZO_NODE_ROLE: "all"
  ZO_NODE_ROLE_GROUP: ""
  ZO_LOCAL_MODE: "false"
  ZO_LOCAL_MODE_STORAGE: "s3"
  ZO_HTTP_PORT: "5080"
  ZO_HTTP_ADDR: ""
  ZO_HTTP_IPV6_ENABLED: "false"
  ZO_GRPC_PORT: "5081"
  ZO_GRPC_ADDR: ""
  ZO_GRPC_CONNECT_TIMEOUT: "600"
  ZO_GRPC_MAX_MESSAGE_SIZE: "16"
  ZO_GRPC_CHANNEL_CACHE_DISABLED: "false"
  ZO_GRPC_ORG_HEADER_KEY: "organization"
  ZO_GRPC_STREAM_HEADER_KEY: "stream-name"
  ZO_INTERNAL_GRPC_TOKEN: ""
  ZO_DATA_DIR: "./data/"
  ZO_DATA_DB_DIR: ""
  ZO_DATA_WAL_DIR: "/data/wal"
  ZO_DATA_IDX_DIR: "/data/idx"
  ZO_DATA_STREAM_DIR: "/data/stream"
  ZO_DATA_CACHE_DIR: "/data/cache"
  ZO_MMDB_DATA_DIR: ""
  ZO_UI_ENABLED: "true"
  ZO_UI_SQL_BASE64_ENABLED: "false"
  ZO_WEB_URL: ""
  ZO_BASE_URI: ""
  ZO_HEALTH_CHECK_ENABLED: "true"
  ZO_HEALTH_CHECK_TIMEOUT: "5"
  ZO_HEALTH_CHECK_FAILED_TIMES: "3"
  ZO_CLUSTER_COORDINATOR: "nats"
  ZO_QUEUE_STORE: "nats"
  ZO_META_STORE: "postgres"
  ZO_META_CONNECTION_POOL_MIN_SIZE: "5"
  ZO_META_CONNECTION_POOL_MAX_SIZE: "20"
  ZO_META_TRANSACTION_RETRIES: "3"
  ZO_META_TRANSACTION_LOCK_TIMEOUT: "30" # seconds - reduced from 300 to fail fast
  ZO_COLS_PER_RECORD_LIMIT: "200"
  ZO_FILE_FORMAT: "parquet"
  ZO_FEATURE_FULLTEXT_EXTRA_FIELDS: ""
  ZO_FEATURE_INDEX_EXTRA_FIELDS: ""
  ZO_FEATURE_DISTINCT_EXTRA_FIELDS: ""
  ZO_FEATURE_QUERY_QUEUE_ENABLED: "true"
  ZO_FEATURE_QUICK_MODE_FIELDS: "" # default fields for quick mode
  ZO_FEATURE_QUERY_PARTITION_STRATEGY: "file_hash"
  ZO_FEATURE_SHARED_MEMTABLE_ENABLED: "false"
  ZO_FEATURE_QUERY_STREAMING_AGGS: "true"
  ZO_UTF8_VIEW_ENABLED: "false"
  ZO_DATAFUSION_STREAMING_AGGS_CACHE_MAX_ENTRIES: "10000"
  ZO_DATAFUSION_FILE_STAT_CACHE_MAX_ENTRIES: "10000"
  ZO_RESULT_CACHE_ENABLED: "true" # Enable result cache for query results
  ZO_USE_MULTIPLE_RESULT_CACHE: "true" # Enable to use mulple result caches for query results
  ZO_RESULT_CACHE_SELECTION_STRATEGY: "overlap" # Strategy to use for result cache, default is both , possible value - both,overlap , duration
  ZO_CACHE_DELAY_SECS: "300" # Discard data of last n seconds from cached results
  ZO_DISK_RESULT_CACHE_MAX_SIZE: "0" # MB, default is 10% of local volume available space and maximum 20GB
  ZO_METRICS_CACHE_ENABLED: "true" # Enable result cache for metrics query
  ZO_METRICS_CACHE_MAX_ENTRIES: "10000"
  ZO_CONSISTENT_HASH_VNODES: "1000"
  ZO_QUERY_OPTIMIZATION_NUM_FIELDS: "0"
  ZO_SCHEMA_MAX_FIELDS_TO_ENABLE_UDS: "500"
  ZO_USER_DEFINED_SCHEMA_MAX_FIELDS: "600"
  ZO_QUICK_MODE_ENABLED: "false"
  ZO_QUICK_MODE_FORCE_ENABLED: "true"
  ZO_QUICK_MODE_NUM_FIELDS: "500"
  ZO_QUICK_MODE_STRATEGY: ""
  ZO_BLOOM_FILTER_ENABLED: "true"
  ZO_BLOOM_FILTER_DEFAULT_FIELDS: ""
  ZO_ENABLE_INVERTED_INDEX: "true"
  ZO_FEATURE_QUERY_REMOVE_FILTER_WITH_INDEX: "true"
  ZO_INVERTED_INDEX_COUNT_OPTIMIZER_ENABLED: "true"
  ZO_INVERTED_INDEX_CAMEL_CASE_TOKENIZER_DISABLED: "false"
  ZO_QUERY_ON_STREAM_SELECTION: "true" # Toggle search to be trigger based on button click event.
  ZO_DISTINCT_VALUES_INTERVAL: "10"
  ZO_DISTINCT_VALUES_HOURLY: "false"
  ZO_TRACING_SEARCH_ENABLED: "false"
  ZO_SEARCH_INSPECTOR_ENABLED: "false"
  ZO_TRACING_HEADER_KEY: "Authorization"
  ZO_TRACING_EXTRA_ENVS: "" # extra environment variables will add to tracing, multiple values should be comma separated
  OTEL_OTLP_HTTP_ENDPOINT: "<http://localhost:5080/api/default/v1/traces>"
  ZO_TELEMETRY: "true"
  ZO_TELEMETRY_HEARTBEAT: "1800"
  ZO_TELEMETRY_URL: "<https://e1.zinclabs.dev>"
  ZO_JSON_LIMIT: "209715200"
  ZO_PAYLOAD_LIMIT: "209715200"
  ZO_PARQUET_MAX_ROW_GROUP_SIZE: "0"
  ZO_MAX_FILE_SIZE_ON_DISK: "128"
  ZO_MAX_FILE_SIZE_IN_MEMORY: "128"
  ZO_MAX_FILE_RETENTION_TIME: "600"
  ZO_FILE_PUSH_INTERVAL: "10"
  ZO_FILE_PUSH_LIMIT: "10000" # over this limit will skip merging on ingester
  ZO_FILE_MOVE_FIELDS_LIMIT: "2000"
  ZO_FILE_MOVE_THREAD_NUM: "0"
  ZO_FILE_MERGE_THREAD_NUM: "0"
  ZO_MEM_DUMP_THREAD_NUM: "0"
  ZO_MEM_TABLE_MAX_SIZE: "0"
  ZO_MEM_TABLE_BUCKET_NUM: "0"
  ZO_MEM_PERSIST_INTERVAL: "2"
  ZO_ENRICHMENT_TABLE_LIMIT: "10" # size in mb
  ZO_WAL_WRITE_BUFFER_SIZE: "16384"
  ZO_WAL_WRITE_QUEUE_SIZE: "0"
  ZO_WAL_RUNTIME_WORKER_NUM: "0"
  ZO_WAL_FSYNC_DISABLED: "false"
  ZO_WAL_WRITE_QUEUE_ENABLED: "false"
  ZO_WAL_WRITE_QUEUE_FULL_REJECT: "false"
  ZO_WAL_DEDICATED_RUNTIME_ENABLED: "false"
  ZO_QUERY_THREAD_NUM: "0"
  ZO_QUERY_INDEX_THREAD_NUM: "0"
  ZO_QUERY_TIMEOUT: "3600"
  ZO_QUERY_INGESTER_TIMEOUT: "0"
  ZO_GRPC_INGEST_TIMEOUT: "600"
  ZO_ROUTE_TIMEOUT: "3600"
  ZO_QUERY_DEFAULT_LIMIT: "1000"
  ZO_ROUTE_MAX_CONNECTIONS: "1024"
  ZO_ROUTE_STRATEGY: "workload"
  ZO_HTTP_WORKER_NUM: "0"
  ZO_HTTP_WORKER_MAX_BLOCKING: "0"
  ZO_HTTP_SHUTDOWN_TIMEOUT: "5"
  ZO_HTTP_SLOW_LOG_THRESHOLD: "5"
  ZO_HTTP_ACCESS_LOG_FORMAT: ""
  ZO_INGEST_ALLOWED_UPTO: "24"
  ZO_INGEST_FLATTEN_LEVEL: "3"
  ZO_LOGS_FILE_RETENTION: "hourly"
  ZO_TRACES_FILE_RETENTION: "hourly"
  ZO_METRICS_FILE_RETENTION: "hourly"
  ZO_METRICS_QUERY_RETENTION: "hourly"
  ZO_METRICS_DEDUP_ENABLED: "true"
  ZO_METRICS_LEADER_PUSH_INTERVAL: "15"
  ZO_METRICS_LEADER_ELECTION_INTERVAL: "30"
  ZO_PROMETHEUS_HA_CLUSTER: "cluster"
  ZO_PROMETHEUS_HA_REPLICA: "__replica__"
  ZO_COMPACT_ENABLED: "true"
  ZO_COMPACT_INTERVAL: "10"
  ZO_COMPACT_SYNC_TO_DB_INTERVAL: "60"  # Reduced from 600 to sync metadata faster
  ZO_COMPACT_STRATEGY: "file_time"
  ZO_COMPACT_MAX_FILE_SIZE: "512"
  ZO_COMPACT_DATA_RETENTION_DAYS: "3650"
  ZO_COMPACT_DELETE_FILES_DELAY_MINUTES: "120"
  ZO_COMPACT_BATCH_SIZE: "0"
  ZO_COMPACT_BLOCKED_ORGS: ""
  ZO_MEMORY_CACHE_ENABLED: "true"
  ZO_MEMORY_CACHE_STRATEGY: "lru" # lru, fifo, time_lru <https://github.com/openobserve/openobserve/pull/6489>
  ZO_MEMORY_CACHE_MAX_SIZE: "2048"
  ZO_MEMORY_CACHE_SKIP_SIZE: "0"
  ZO_MEMORY_CACHE_RELEASE_SIZE: "0"
  ZO_MEMORY_CACHE_GC_SIZE: "50"
  ZO_MEMORY_CACHE_GC_INTERVAL: "0"
  ZO_MEMORY_CACHE_DATAFUSION_MEMORY_POOL: ""
  ZO_MEMORY_CACHE_DATAFUSION_MAX_SIZE: "4096"
  ZO_DISK_CACHE_ENABLED: "true"
  ZO_DISK_CACHE_STRATEGY: "time_lru" # lru, fifo, time_lru <https://github.com/openobserve/openobserve/pull/6489>
  ZO_DISK_CACHE_MAX_SIZE: "40000"
  ZO_DISK_CACHE_SKIP_SIZE: "0"
  ZO_DISK_CACHE_RELEASE_SIZE: "10000"
  ZO_DISK_CACHE_GC_SIZE: "100"
  ZO_DISK_CACHE_GC_INTERVAL: "0"
  ZO_CACHE_LATEST_FILES_ENABLED: "true"
  ZO_CACHE_LATEST_FILES_PARQUET: "true"
  ZO_CACHE_LATEST_FILES_INDEX: "true"
  ZO_CACHE_LATEST_FILES_DELETE_MERGE_FILES: "false"
  ZO_CACHE_LATEST_FILES_DOWNLOAD_FROM_NODE: "true"
  ZO_NATS_PREFIX: "o2_"
  ZO_NATS_USER: ""
  ZO_NATS_PASSWORD: ""
  ZO_NATS_REPLICAS: "3"
  ZO_NATS_QUEUE_MAX_AGE: "60" # days for queue message max age
  ZO_NATS_QUEUE_MAX_SIZE: "2048" # max size of queue message
  ZO_NATS_V211_SUPPORT: "false"
  ZO_NATS_KV_WATCH_MODULES: ""
  ZO_NATS_EVENT_STORAGE: "file" # file or memory
  ZO_NATS_CONNECT_TIMEOUT: "5"
  ZO_NATS_COMMAND_TIMEOUT: "10"
  ZO_NATS_LOCK_WAIT_TIMEOUT: "600"
  ZO_S3_PROVIDER: "s3" # Need to set this up only if minio is being for object storage. Will be set automatically if enabled minio is being used.valid values are s3, azure, minio
  ZO_S3_SERVER_URL: "<https://s3.ap-southeast-1.amazonaws.com>" # do not need to set this if enabled minio is being used. settings will be picked from minio section
  ZO_S3_REGION_NAME: "ap-southeast-1" # do not need to set this if enabled minio is being used. settings will be picked from minio section
  ZO_S3_BUCKET_NAME: "<s3>" # do not need to set this if enabled minio is being used. settings will be picked from minio section
  ZO_S3_BUCKET_PREFIX: "palo-alto"
  ZO_S3_FEATURE_FORCE_HOSTED_STYLE: "false"
  ZO_S3_FEATURE_HTTP1_ONLY: "false"
  ZO_S3_FEATURE_HTTP2_ONLY: "false"
  ZO_S3_SYNC_TO_CACHE_INTERVAL: "600"
  ZO_S3_CONNECT_TIMEOUT: "10" # in seconds
  ZO_S3_REQUEST_TIMEOUT: "3600" # in seconds
  ZO_S3_ALLOW_INVALID_CERTIFICATES: "true"
  ZO_S3_MAX_RETRIES: "10"
  ZO_S3_MAX_IDLE_PER_HOST: "0"
  ZO_S3_CONNECTION_KEEPALIVE_TIMEOUT: "20"
  ZO_USAGE_REPORTING_ENABLED: "true"
  ZO_USAGE_REPORTING_MODE: "local"
  ZO_USAGE_REPORTING_URL: "<http://localhost:5080/api/_meta/usage/_json>"
  ZO_USAGE_REPORTING_CREDS: ""
  ZO_USAGE_REPORTING_THREAD_NUM: "0"
  ZO_USAGE_ORG: "_meta"
  ZO_USAGE_BATCH_SIZE: "2000"
  ZO_USAGE_PUBLISH_INTERVAL: "60" # duration in seconds after last reporting usage will be published
  ZO_PRINT_KEY_CONFIG: "false"
  ZO_PRINT_KEY_EVENT: "false"
  ZO_PRINT_KEY_SQL: "true"
  ZO_IGNORE_FILE_RETENTION_BY_STREAM: "false"
  ZO_RUM_ENABLED: "false"
  ZO_RUM_APPLICATION_ID: ""
  ZO_RUM_SITE: ""
  ZO_RUM_SERVICE: ""
  ZO_RUM_ENV: ""
  ZO_RUM_VERSION: "0.9.1"
  ZO_RUM_ORGANIZATION_IDENTIFIER: "default"
  ZO_RUM_API_VERSION: "v1"
  ZO_RUM_INSECURE_HTTP: "false"
  ZO_COOKIE_MAX_AGE: "2592000" # 30 days
  ZO_COOKIE_SAME_SITE_LAX: "true"
  ZO_COOKIE_SECURE_ONLY: "false"
  ZO_PROF_PYROSCOPE_ENABLED: "false"
  ZO_PROF_PYROSCOPE_SERVER_URL: "<http://localhost:4040>"
  ZO_PROF_PYROSCOPE_PROJECT_NAME: "openobserve"
  ZO_ENTRY_PER_SCHEMA_VERSION_ENABLED: "false"
  ZO_QUERY_GROUP_BASE_SPEED: "768" # MB/s/core
  ZO_MINI_SEARCH_PARTITION_DURATION_SECS: "10"
  ZO_QUERY_PARTITION_BY_SECS: "86400" # In how many seconds we want to get the results per partition
  ZO_QUERY_PARTITION_MIN_SECS: "600" # seconds. This is the minimum time range for a partition
  ZO_SHOW_STREAM_DATES_DOCS_NUM: true # Show docs count and stream dates
  ZO_INGEST_BLOCKED_STREAMS: "" # // use comma to split multiple streams
  ZO_INGEST_INFER_SCHEMA_PER_REQUEST: "true"
  ZO_REPORT_SERVER_HTTP_PORT: "5082"
  ZO_CHROME_SLEEP_SECS: "50"
  ZO_CHROME_NO_SANDBOX: "true"
  ZO_SMTP_ENABLED: "false" # if true then smtp will be enabled. set SMTP creds in auth section. Set ZO_WEB_URL to the correct URL for reports to be sent
  ZO_SMTP_HOST: "<http://email-smtp.us-west-2.amazonaws.com|email-smtp.us-west-2.amazonaws.com>"
  ZO_SMTP_PORT: "465"
  ZO_SMTP_FROM_EMAIL: "<mailto:root@example.com|root@example.com>"
  ZO_SMTP_ENCRYPTION: "ssltls"
  ZO_CONCATENATED_SCHEMA_FIELD_NAME: "_all"
  ZO_SKIP_FORMAT_BULK_STREAM_NAME: "false"
  ZO_BULK_RESPONSE_INCLUDE_ERRORS_ONLY: "false"
  ZO_ALLOW_USER_DEFINED_SCHEMAS: "true"
  ZO_SKIP_FORMAT_STREAM_NAME: "false"
  ZO_MEM_TABLE_STREAMS: "" # Streams for which dedicated MemTable will be used as comma separated values
  ZO_DEFAULT_SCRAPE_INTERVAL: "15"
  ZO_MEMORY_CIRCUIT_BREAKER_ENABLED: "false"
  ZO_MEMORY_CIRCUIT_BREAKER_RATIO: "90"
  ZO_MMDB_DISABLE_DOWNLOAD: "false"
  ZO_MMDB_UPDATE_DURATION_DAYS: "30" # 30 days
  ZO_MMDB_GEOLITE_CITYDB_URL: "<https://geoip.zinclabs.dev/GeoLite2-City.mmdb>"
  ZO_MMDB_GEOLITE_CITYDB_SHA256_URL: "<https://geoip.zinclabs.dev/GeoLite2-City.sha256>"
  ZO_MMDB_GEOLITE_ASNDB_URL: "<https://geoip.zinclabs.dev/GeoLite2-ASN.mmdb>"
  ZO_MMDB_GEOLITE_ASNDB_SHA256_URL: "<https://geoip.zinclabs.dev/GeoLite2-ASN.sha256>"
  ZO_REPORT_USER_NAME: ""
  ZO_REPORT_USER_PASSWORD: ""
  ZO_REPORT_SERVER_URL: ""
  ZO_REPORT_SERVER_SKIP_TLS_VERIFY: "false"
  ZO_SELF_METRIC_CONSUMPTION_ENABLED: "false" # self-consume metrics generated by openobserve
  ZO_SELF_METRIC_CONSUMPTION_INTERVAL: "60" # self-consume metrics interval, unit seconds
  ZO_SELF_METRIC_CONSUMPTION_ACCEPTLIST: "" # only these metrics will be self-consumed, comma separated
  ZO_SWAGGER_ENABLED: "true"
  ZO_STREAMING_ENABLED: "true"
  ZO_MIN_AUTO_REFRESH_INTERVAL: "5"
  ZO_MAX_DASHBOARD_SERIES: "100"
  ZO_PROMETHEUS_ENABLED: "false"
  ZO_HISTOGRAM_ENABLED: "true" # show/hide histogram on logs page by default
  ZO_SGRAPH_ENABLED: "true"
  RUST_LOG: "info"
  RUST_BACKTRACE: "0"
@Simran(OpenObserve) let me know when there is an update
s
sure, will take a look and let u know!
p
For the issue where the UI freezes while loading the next 50 records, this is not expected behavior. Loading only 50 additional records should not cause the interface to become unresponsive, even when the overall query scans a large volume of data. To help us investigate this further, could you please provide the following information: • Approximately how many fields are present in this stream? • What is the approximate size of a single record/row? • Could you also check and share the browser memory usage while loading the next 50 records, particularly when the UI starts to freeze? Regarding the download of 1,000 records, the processing time and resource usage can vary significantly depending on the number of fields and the size of each record. Therefore, the above information will also help us understand the download timeout issue. Once we have these details, we can investigate both issues further and determine whether the problem is related to the record size, number of fields, browser memory consumption, or another bottleneck.
Open in Slack
PreviousNext
Powered by