Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf56b2b8

Browse files
authored
fix: metrics can't search parquet in wal (#8896)
1 parentd360151 commitf56b2b8

File tree

15 files changed

+71
-56
lines changed

15 files changed

+71
-56
lines changed

‎src/common/meta/search.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ mod enterprise_tests {
339339
}
340340

341341
#[test]
342-
fntest_calculate_record_batches_deltas_no_cache(){
342+
fntest_calculate_record_batches_deltas_without_cache(){
343343
// Test case: No cache, entire query range should be a delta
344344
let cache_result =vec![];
345345
let query_start_time =10_000_000;

‎src/config/src/config.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1324,8 +1324,6 @@ pub struct Limit {
13241324
pubmetrics_leader_push_interval:u64,
13251325
#[env_config(name ="ZO_METRICS_LEADER_ELECTION_INTERVAL",default =30)]
13261326
pubmetrics_leader_election_interval:i64,
1327-
#[env_config(name ="ZO_METRICS_MAX_SERIES_PER_QUERY",default =30000)]
1328-
pubmetrics_max_series_per_query:usize,
13291327
#[env_config(name ="ZO_METRICS_MAX_POINTS_PER_SERIES",default =30000)]
13301328
pubmetrics_max_points_per_series:usize,
13311329
#[env_config(name ="ZO_METRICS_CACHE_MAX_ENTRIES",default =10000)]
@@ -2365,9 +2363,6 @@ fn check_common_config(cfg: &mut Config) -> Result<(), anyhow::Error> {
23652363
}
23662364

23672365
// check for metrics limit
2368-
if cfg.limit.metrics_max_series_per_query ==0{
2369-
cfg.limit.metrics_max_series_per_query =30_000;
2370-
}
23712366
if cfg.limit.metrics_max_points_per_series ==0{
23722367
cfg.limit.metrics_max_points_per_series =30_000;
23732368
}

‎src/config/src/meta/promql.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,8 @@ pub struct RequestRangeQuery {
134134
pubstep:Option<String>,
135135
/// Evaluation timeout.
136136
pubtimeout:Option<String>,
137-
///Do not use cache.
138-
pubno_cache:Option<bool>,
137+
///Use cache.
138+
pubuse_cache:Option<bool>,
139139
}
140140

141141
#[derive(Debug,Deserialize)]

‎src/handler/grpc/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ impl From<promql::MetricsQueryRequest> for cluster_rpc::MetricsQueryRequest {
6969
need_wal:false,
7070
query:Some(req_query),
7171
timeout:0,
72-
no_cache: req.no_cache.unwrap_or_default(),
72+
use_cache: req.use_cache.unwrap_or(true),
7373
}
7474
}
7575
}

‎src/handler/http/request/promql/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ async fn query(
233233
end,
234234
step:300_000_000,// 5m
235235
query_exemplars:false,
236-
no_cache:None,
236+
use_cache:None,
237237
};
238238

239239
search(&trace_id, org_id,&req, user_email, timeout).await
@@ -562,7 +562,7 @@ async fn query_range(
562562
end,
563563
step,
564564
query_exemplars,
565-
no_cache: req.no_cache,
565+
use_cache: req.use_cache,
566566
};
567567
search(&trace_id, org_id,&req, user_email, timeout).await
568568
}

‎src/proto/proto/cluster/metrics.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ message MetricsQueryRequest {
1818
boolneed_wal=4;
1919
MetricsQueryStmtquery=5;
2020
int64timeout=8;
21-
boolno_cache=9;
21+
booluse_cache=9;
2222
}
2323

2424
messageMetricsQueryStmt {

‎src/proto/src/generated/cluster.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,7 @@ pub struct MetricsQueryRequest {
496496
#[prost(int64, tag ="8")]
497497
pubtimeout:i64,
498498
#[prost(bool, tag ="9")]
499-
pubno_cache:bool,
499+
pubuse_cache:bool,
500500
}
501501
#[derive(serde::Serialize)]
502502
#[derive(Clone,PartialEq,::prost::Message)]

‎src/service/alerts/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ impl QueryConditionExt for QueryCondition {
178178
(end - start) / promql::MAX_DATA_POINTS,
179179
),
180180
query_exemplars:false,
181-
no_cache:None,
181+
use_cache:None,
182182
};
183183
let resp =match promql::search::search(&trace_id, org_id,&req,"",0).await{
184184
Ok(v) => v,

‎src/service/promql/engine.rs‎

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,7 @@ use super::{
4848
utils::{apply_label_selector, apply_matchers},
4949
};
5050
usecrate::service::promql::{
51-
DEFAULT_MAX_SERIES_PER_QUERY, aggregations, binaries, functions, micros,
52-
rewrite::remove_filter_all, value::*,
51+
aggregations, binaries, functions, micros, rewrite::remove_filter_all, value::*,
5352
};
5453

5554
pubstructEngine{
@@ -1122,7 +1121,6 @@ async fn selector_load_data_from_datafusion(
11221121
label_selector:&Option<HashSet<String>>,
11231122
query_exemplars:bool,
11241123
) ->Result<HashMap<HashLabelValue,RangeValue>>{
1125-
let cfg = config::get_config();
11261124
let table_name = selector.name.as_ref().unwrap();
11271125
letmut df_group =match ctx.table(table_name).await{
11281126
Ok(v) => v.filter(
@@ -1150,7 +1148,8 @@ async fn selector_load_data_from_datafusion(
11501148
}
11511149
}
11521150

1153-
let label_cols = df_group
1151+
// get label columns
1152+
letmut label_cols = df_group
11541153
.schema()
11551154
.fields()
11561155
.iter()
@@ -1163,16 +1162,13 @@ async fn selector_load_data_from_datafusion(
11631162
{
11641163
None
11651164
}else{
1166-
Some(col(name))
1165+
Some(name)
11671166
}
11681167
})
11691168
.collect::<Vec<_>>();
1170-
1171-
let max_series =if cfg.limit.metrics_max_series_per_query >0{
1172-
cfg.limit.metrics_max_series_per_query
1173-
}else{
1174-
DEFAULT_MAX_SERIES_PER_QUERY
1175-
};
1169+
// sort labels to have a consistent order
1170+
label_cols.sort();
1171+
let label_cols = label_cols.into_iter().map(col).collect::<Vec<_>>();
11761172

11771173
// get hash & timestamp
11781174
let start_time = std::time::Instant::now();
@@ -1182,13 +1178,11 @@ async fn selector_load_data_from_datafusion(
11821178
vec![col(HASH_LABEL)],
11831179
vec![max(col(TIMESTAMP_COL_NAME)).alias(TIMESTAMP_COL_NAME)],
11841180
)?
1185-
.sort(vec![col(HASH_LABEL).sort(true,true)])?
1186-
.limit(0,Some(max_series))?
11871181
.collect()
11881182
.await?;
11891183

11901184
let hash_field_type = schema.field_with_name(HASH_LABEL).unwrap().data_type();
1191-
let(muttimestamp_values, hash_value_set):(Vec<_>,HashSet<HashLabelValue>) =
1185+
let(timestamp_values, hash_value_set):(HashSet<i64>,HashSet<HashLabelValue>) =
11921186
if hash_field_type ==&DataType::UInt64{
11931187
sub_batch
11941188
.iter()
@@ -1232,8 +1226,6 @@ async fn selector_load_data_from_datafusion(
12321226
})
12331227
.unzip()
12341228
};
1235-
timestamp_values.sort();
1236-
timestamp_values.dedup();
12371229
let timestamp_values = timestamp_values.into_iter().map(lit).collect::<Vec<_>>();
12381230

12391231
log::info!(
@@ -1251,11 +1243,12 @@ async fn selector_load_data_from_datafusion(
12511243

12521244
letmut metrics:HashMap<HashLabelValue,RangeValue> =
12531245
HashMap::with_capacity(hash_value_set.len());
1246+
letmut labels =Vec::new();
12541247
for batchin series{
12551248
let columns = batch.columns();
12561249
let schema = batch.schema();
12571250
let fields = schema.fields();
1258-
letmutcols = fields
1251+
let cols = fields
12591252
.iter()
12601253
.zip(columns)
12611254
.filter_map(|(field, col)|{
@@ -1268,8 +1261,6 @@ async fn selector_load_data_from_datafusion(
12681261
}
12691262
})
12701263
.collect::<Vec<(_,_)>>();
1271-
cols.sort_by(|a, b| a.0.cmp(b.0));
1272-
letmut labels =Vec::with_capacity(columns.len());
12731264
if hash_field_type ==&DataType::UInt64{
12741265
let hash_values = batch
12751266
.column_by_name(HASH_LABEL)

‎src/service/promql/mod.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ pub(crate) const DEFAULT_LOOKBACK: Duration = Duration::from_secs(300); // 5m
4646
pub(crate)constMINIMAL_INTERVAL:Duration =Duration::from_secs(1);// 1s
4747
pub(crate)constMAX_DATA_POINTS:i64 =256;// Width of panel: window.innerWidth / 4
4848
pub(crate)constDEFAULT_MAX_POINTS_PER_SERIES:usize =30000;// Maximum number of points per series
49-
constDEFAULT_MAX_SERIES_PER_QUERY:usize =30000;// Maximum number of series in a single query
5049
constDEFAULT_STEP:Duration =Duration::from_secs(15);// default step in seconds
5150
constMIN_TIMESERIES_POINTS_FOR_TIME_ROUNDING:i64 =10;// Adjust this value as needed
5251

@@ -70,7 +69,7 @@ pub struct MetricsQueryRequest {
7069
pubend:i64,
7170
pubstep:i64,
7271
pubquery_exemplars:bool,
73-
pubno_cache:Option<bool>,
72+
pubuse_cache:Option<bool>,
7473
}
7574

7675
#[derive(Debug,Serialize)]

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp