Elon
12/07/2020, 6:16 PMMayank
Elon
12/07/2020, 6:17 PMKishore G
Jackie
12/07/2020, 6:31 PMmmap
to direct
for realtime off-heap allocation improved the GC?Kishore G
Elon
12/08/2020, 7:25 PMMatt
12/08/2020, 10:58 PMKishore G
Ken Krugler
12/11/2020, 1:24 AMKen Krugler
12/11/2020, 1:26 AMElon
12/11/2020, 6:02 AMKen Krugler
12/16/2020, 3:24 PMKen Krugler
12/29/2020, 3:17 PMKen Krugler
01/07/2021, 9:05 PMKen Krugler
01/20/2021, 3:57 PMLeon Liu
06/30/2021, 12:43 PMMayank
Subbu Subramaniam
11/10/2021, 6:31 PMAnish Nair
07/05/2022, 7:51 AMKartik Khare
09/14/2022, 3:37 PMAbhishek Gupta
09/14/2022, 5:26 PMupstream request timeout
,
when using single month data in filter then it is working. Not sure why error message isn't giving valid reason of failure.
Can you guys suggest the right and necessary table config
SELECT
DISTINCTCOUNT( CASE WHEN data_source = 'web' THEN master_id ELSE 'a' END) as activity_account_count,
DISTINCTCOUNT( CASE WHEN data_source in ('b2bn', 'b2bn_excluded') THEN master_id ELSE 'a' END) as kw_account_count,
DISTINCTCOUNT( CASE WHEN data_source = 'fpm' THEN master_id ELSE 'a' END) as fpm_account_count,
DISTINCTCOUNT( CASE WHEN data_source = 'tpm' THEN master_id ELSE 'a' END) as tpm_account_count,
DISTINCTCOUNT( CASE WHEN data_source = 'crm' THEN master_id ELSE 'a' END) as crm_account_count,
DISTINCTCOUNT( CASE WHEN data_source = 'map' THEN master_id ELSE 'a' END) as map_account_count,
DISTINCTCOUNT( master_id) as all_account_count
FROM
six_sense_dapm
WHERE
dt BETWEEN '2022-01-01' AND '2022-08-30'
AND data_source IN ('web', 'b2bn', 'b2bn_excluded', 'fpm', 'tpm', 'crm', 'map')
AND product='__all__'
table config :
{
"OFFLINE": {
"tableName": "sumologic_dapm_OFFLINE",
"tableType": "OFFLINE",
"segmentsConfig": {
"timeType": "DAYS",
"schemaName": "sumologic_dapm",
"replication": "2",
"segmentPushType": "APPEND",
"timeColumnName": "dt",
"allowNullTimeValue": false
},
"tenants": {
"broker": "DefaultTenant",
"server": "DefaultTenant"
},
"tableIndexConfig": {
"invertedIndexColumns": [
"data_source",
"product",
"source_activity_name"
],
"noDictionaryColumns": [
"external_id",
"master_id",
"secondary_id",
"source_activity_desc",
"source_activity_url",
"source_activity_referrer_url",
"source_activity_desc",
"source_activity_url_r",
"metric_value",
"source_id"
],
"rangeIndexColumns": [
"dt"
],
"optimizeDictionaryForMetrics": true,
"enableDefaultStarTree": true,
"enableDynamicStarTreeCreation": true,
"aggregateMetrics": true,
"nullHandlingEnabled": true,
"rangeIndexVersion": 2,
"autoGeneratedInvertedIndex": false,
"createInvertedIndexDuringSegmentGeneration": false,
"sortedColumn": [
"data_source",
"master_id"
],
"loadMode": "MMAP"
},
"metadata": {},
"isDimTable": false
}
}
Prakhar Pande
11/07/2022, 8:16 PMLee Wei Hern Jason
03/03/2023, 4:31 AMShreeram Goyal
03/21/2023, 4:30 PMEric Liu
05/09/2023, 11:59 PMSHA-256
algorithm for the upsert table from the performance perspective? I want to avoid collisions as much as possible.Eric Liu
08/16/2023, 3:02 AMsegmentPrunerTypes
config) for an upsert table? The partition key (say pk
) in my upsert table is the hashed value of two concatenated id fields (lets say A
and B
), and the most common pattern of queries against that table is filtering on column A
instead of the pk
.Eric Liu
08/24/2023, 3:34 PMLee Wei Hern Jason
08/25/2023, 5:20 AMSELECT start_timestamp_10mins, SUM(online_seconds)/3600.0 AS value FROM m_driver_supply_derived WHERE country_id = 6 AND city_id IN (84, 130, 20, 340, 369, 79, 360, 308, 98, 81, 230, 34, 349, 350, 370, 63, 372, 346, 231, 60, 86, 61, 218, 99, 43, 363, 222, 64, 367, 80, 15, 101, 348, 62, 361, 28, 75, 373, 356, 352, 69, 41, 362, 55, 347, 341, 374, 365, 219, 65, 275, 10, 66, 364, 40, 225, 366, 146, 35, 337, 354, 342, 344, 102, 345, 96, 132, 78, 18, 359, 36, 256, 77, 358, 343, 371, 100, 357, 368, 215, 26, 255, 44, 144, 351, 85, 353, 76) AND
business_vertical = 'BUSINESS_VERTICAL_TYPE_TRANSPORT' AND
granularity = 'GRANULARITY_BUSINESS' AND
start_timestamp_10mins >= '2023-08-15 17:00:00.0' AND start_timestamp_10mins < '2023-08-25 17:00:00.0' GROUP BY start_timestamp_10mins HAVING value >= 0 ORDER BY start_timestamp_10mins ASC LIMIT 10000000
Star Tree Index:
We are configuring it in the order of highest cardinality to lowest.
"starTreeIndexConfigs": [
{
"dimensionsSplitOrder": [
"vehicle_type_id",
"city_id",
"country_id",
"vehicle_mode",
"granularity",
"start_timestamp_10mins",
"start_timestamp_1hour",
"start_timestamp_4hour"
],
"skipStarNodeCreationForDimensions": [],
"functionColumnPairs": [
"SUM__online_seconds",
"SUM__online_count",
"SUM__in_transit_seconds"
],
"maxLeafRecords": 10000
}
],
Venkat Boina(VB)
12/01/2023, 6:05 AM