Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit7c78b26

Browse files
Enable tenant ID selection for multi-tenant queries. (grafana#5821)
Closesgrafana#5820Co-authored-by: JordanRushing <rushing.jordan@gmail.com>
1 parent9d0fd3a commit7c78b26

File tree

2 files changed

+204
-10
lines changed

2 files changed

+204
-10
lines changed

‎pkg/querier/multi_tenant_querier.go‎

Lines changed: 95 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,24 @@ func (q *MultiTenantQuerier) SelectLogs(ctx context.Context, params logql.Select
4242
returnq.Querier.SelectLogs(ctx,params)
4343
}
4444

45-
iters:=make([]iter.EntryIterator,len(tenantIDs))
46-
fori,id:=rangetenantIDs {
45+
selector,err:=params.LogSelector()
46+
iferr!=nil {
47+
returnnil,err
48+
}
49+
matchedTenants,filteredMatchers:=filterValuesByMatchers(defaultTenantLabel,tenantIDs,selector.Matchers()...)
50+
params.Selector=replaceMatchers(selector,filteredMatchers).String()
51+
52+
iters:=make([]iter.EntryIterator,len(matchedTenants))
53+
i:=0
54+
forid:=rangematchedTenants {
4755
singleContext:=user.InjectOrgID(ctx,id)
4856
iter,err:=q.Querier.SelectLogs(singleContext,params)
4957
iferr!=nil {
5058
returnnil,err
5159
}
5260

5361
iters[i]=NewTenantEntryIterator(iter,id)
62+
i++
5463
}
5564
returniter.NewSortEntryIterator(iters,params.Direction),nil
5665
}
@@ -65,15 +74,23 @@ func (q *MultiTenantQuerier) SelectSamples(ctx context.Context, params logql.Sel
6574
returnq.Querier.SelectSamples(ctx,params)
6675
}
6776

68-
iters:=make([]iter.SampleIterator,len(tenantIDs))
69-
fori,id:=rangetenantIDs {
77+
matchedTenants,updatedSelector,err:=removeTenantSelector(params,tenantIDs)
78+
iferr!=nil {
79+
returnnil,err
80+
}
81+
params.Selector=updatedSelector.String()
82+
83+
iters:=make([]iter.SampleIterator,len(matchedTenants))
84+
i:=0
85+
forid:=rangematchedTenants {
7086
singleContext:=user.InjectOrgID(ctx,id)
7187
iter,err:=q.Querier.SelectSamples(singleContext,params)
7288
iferr!=nil {
7389
returnnil,err
7490
}
7591

7692
iters[i]=NewTenantSampleIterator(iter,id)
93+
i++
7794
}
7895
returniter.NewSortSampleIterator(iters),nil
7996
}
@@ -136,6 +153,80 @@ func (q *MultiTenantQuerier) Series(ctx context.Context, req *logproto.SeriesReq
136153
returnlogproto.MergeSeriesResponses(responses)
137154
}
138155

156+
// removeTenantSelector filters the given tenant IDs based on any tenant ID filter the in passed selector.
157+
funcremoveTenantSelector(params logql.SelectSampleParams,tenantIDs []string) (map[string]struct{}, syntax.Expr,error) {
158+
expr,err:=params.Expr()
159+
iferr!=nil {
160+
returnnil,nil,err
161+
}
162+
matchedTenants,filteredMatchers:=filterValuesByMatchers(defaultTenantLabel,tenantIDs,expr.Selector().Matchers()...)
163+
updatedExpr:=replaceMatchers(expr,filteredMatchers)
164+
returnmatchedTenants,updatedExpr,nil
165+
}
166+
167+
// replaceMatchers traverses the passed expression and replaces all matchers.
168+
funcreplaceMatchers(expr syntax.Expr,matchers []*labels.Matcher) syntax.Expr {
169+
expr,_=syntax.Clone(expr)
170+
expr.Walk(func(einterface{}) {
171+
switchconcrete:=e.(type) {
172+
case*syntax.MatchersExpr:
173+
concrete.Mts=matchers
174+
}
175+
})
176+
returnexpr
177+
}
178+
179+
// See https://github.com/grafana/mimir/blob/114ab88b50638a2047e2ca2a60640f6ca6fe8c17/pkg/querier/tenantfederation/tenant_federation.go#L29-L69
180+
// filterValuesByMatchers applies matchers to inputed `idLabelName` and
181+
// `ids`. A set of matched IDs is returned and also all label matchers not
182+
// targeting the `idLabelName` label.
183+
//
184+
// In case a label matcher is set on a label conflicting with `idLabelName`, we
185+
// need to rename this labelMatcher's name to its original name. This is used
186+
// to as part of Select in the mergeQueryable, to ensure only relevant queries
187+
// are considered and the forwarded matchers do not contain matchers on the
188+
// `idLabelName`.
189+
funcfilterValuesByMatchers(idLabelNamestring,ids []string,matchers...*labels.Matcher) (matchedIDsmap[string]struct{},unrelatedMatchers []*labels.Matcher) {
190+
// this contains the matchers which are not related to idLabelName
191+
unrelatedMatchers=make([]*labels.Matcher,0,len(matchers))
192+
193+
// build map of values to consider for the matchers
194+
matchedIDs=sliceToSet(ids)
195+
196+
for_,m:=rangematchers {
197+
switchm.Name {
198+
// matcher has idLabelName to target a specific tenant(s)
199+
caseidLabelName:
200+
forvalue:=rangematchedIDs {
201+
if!m.Matches(value) {
202+
delete(matchedIDs,value)
203+
}
204+
}
205+
206+
// check if has the retained label name
207+
caseretainExistingPrefix+idLabelName:
208+
// rewrite label to the original name, by copying matcher and
209+
// replacing the label name
210+
rewrittenM:=*m
211+
rewrittenM.Name=idLabelName
212+
unrelatedMatchers=append(unrelatedMatchers,&rewrittenM)
213+
214+
default:
215+
unrelatedMatchers=append(unrelatedMatchers,m)
216+
}
217+
}
218+
219+
returnmatchedIDs,unrelatedMatchers
220+
}
221+
222+
funcsliceToSet(values []string)map[string]struct{} {
223+
out:=make(map[string]struct{},len(values))
224+
for_,v:=rangevalues {
225+
out[v]=struct{}{}
226+
}
227+
returnout
228+
}
229+
139230
typerelabelstruct {
140231
tenantIDstring
141232
cachemap[string]labels.Labels

‎pkg/querier/multi_tenant_querier_test.go‎

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@ import (
44
"context"
55
"fmt"
66
"strconv"
7+
"strings"
78
"testing"
89
"time"
10+
"unicode"
911

1012
"github.com/go-kit/log"
1113
"github.com/prometheus/prometheus/model/labels"
@@ -27,12 +29,14 @@ func TestMultiTenantQuerier_SelectLogs(t *testing.T) {
2729
for_,tc:=range []struct {
2830
descstring
2931
orgIDstring
32+
selectorstring
3033
expLabels []string
3134
expLines []string
3235
}{
3336
{
3437
"two tenants",
3538
"1|2",
39+
`{type="test"}`,
3640
[]string{
3741
`{__tenant_id__="1", type="test"}`,
3842
`{__tenant_id__="1", type="test"}`,
@@ -41,9 +45,30 @@ func TestMultiTenantQuerier_SelectLogs(t *testing.T) {
4145
},
4246
[]string{"line 1","line 2","line 1","line 2"},
4347
},
48+
{
49+
"two tenants with selector",
50+
"1|2",
51+
`{type="test", __tenant_id__="1"}`,
52+
[]string{
53+
`{__tenant_id__="1", type="test"}`,
54+
`{__tenant_id__="1", type="test"}`,
55+
},
56+
[]string{"line 1","line 2","line 1","line 2"},
57+
},
58+
{
59+
"two tenants with selector and pipeline filter",
60+
"1|2",
61+
`{type="test", __tenant_id__!="2"} | logfmt | some_lable="foobar"`,
62+
[]string{
63+
`{__tenant_id__="1", type="test"}`,
64+
`{__tenant_id__="1", type="test"}`,
65+
},
66+
[]string{"line 1","line 2","line 1","line 2"},
67+
},
4468
{
4569
"one tenant",
4670
"1",
71+
`{type="test"}`,
4772
[]string{
4873
`{type="test"}`,
4974
`{type="test"}`,
@@ -59,7 +84,7 @@ func TestMultiTenantQuerier_SelectLogs(t *testing.T) {
5984

6085
ctx:=user.InjectOrgID(context.Background(),tc.orgID)
6186
params:= logql.SelectLogParams{QueryRequest:&logproto.QueryRequest{
62-
Selector:`{type="test"}`,
87+
Selector:tc.selector,
6388
Direction:logproto.BACKWARD,
6489
Limit:0,
6590
Shards:nil,
@@ -86,11 +111,13 @@ func TestMultiTenantQuerier_SelectSamples(t *testing.T) {
86111
for_,tc:=range []struct {
87112
descstring
88113
orgIDstring
114+
selectorstring
89115
expLabels []string
90116
}{
91117
{
92118
"two tenants",
93119
"1|2",
120+
`count_over_time({foo="bar"}[1m]) > 10`,
94121
[]string{
95122
`{__tenant_id__="1", app="foo"}`,
96123
`{__tenant_id__="2", app="foo"}`,
@@ -102,9 +129,21 @@ func TestMultiTenantQuerier_SelectSamples(t *testing.T) {
102129
`{__tenant_id__="1", app="bar"}`,
103130
},
104131
},
132+
{
133+
"two tenants with selector",
134+
"1|2",
135+
`count_over_time({foo="bar", __tenant_id__="1"}[1m]) > 10`,
136+
[]string{
137+
`{__tenant_id__="1", app="foo"}`,
138+
`{__tenant_id__="1", app="bar"}`,
139+
`{__tenant_id__="1", app="foo"}`,
140+
`{__tenant_id__="1", app="bar"}`,
141+
},
142+
},
105143
{
106144
"one tenant",
107145
"1",
146+
`count_over_time({foo="bar"}[1m]) > 10`,
108147
[]string{
109148
`{app="foo"}`,
110149
`{app="bar"}`,
@@ -120,16 +159,42 @@ func TestMultiTenantQuerier_SelectSamples(t *testing.T) {
120159
multiTenantQuerier:=NewMultiTenantQuerier(querier,log.NewNopLogger())
121160

122161
ctx:=user.InjectOrgID(context.Background(),tc.orgID)
123-
params:= logql.SelectSampleParams{}
162+
params:= logql.SelectSampleParams{SampleQueryRequest:&logproto.SampleQueryRequest{
163+
Selector:tc.selector,
164+
}}
124165
iter,err:=multiTenantQuerier.SelectSamples(ctx,params)
125166
require.NoError(t,err)
126167

127-
entriesCount:=0
168+
received:=make([]string,0,len(tc.expLabels))
128169
foriter.Next() {
129-
require.Equalf(t,tc.expLabels[entriesCount],iter.Labels(),"Entry %d",entriesCount)
130-
entriesCount++
170+
received=append(received,iter.Labels())
131171
}
132-
require.Equalf(t,len(tc.expLabels),entriesCount,"Expected %d entries but got %d",len(tc.expLabels),entriesCount)
172+
require.ElementsMatch(t,tc.expLabels,received)
173+
})
174+
}
175+
}
176+
177+
funcTestMultiTenantQuerier_TenantFilter(t*testing.T) {
178+
for_,tc:=range []struct {
179+
selectorstring
180+
expectedstring
181+
}{
182+
{
183+
`count_over_time({foo="bar", __tenant_id__="1"}[1m]) > 10`,
184+
`(count_over_time({foo="bar"}[1m]) > 10)`,
185+
},
186+
{
187+
`topk(2, count_over_time({app="foo", __tenant_id__="1"}[3m]))`,
188+
`topk(2, count_over_time({app="foo"}[3m]))`,
189+
},
190+
} {
191+
t.Run(tc.selector,func(t*testing.T) {
192+
params:= logql.SelectSampleParams{SampleQueryRequest:&logproto.SampleQueryRequest{
193+
Selector:tc.selector,
194+
}}
195+
_,updatedSelector,err:=removeTenantSelector(params, []string{})
196+
require.NoError(t,err)
197+
require.Equal(t,removeWhiteSpace(tc.expected),removeWhiteSpace(updatedSelector.String()))
133198
})
134199
}
135200
}
@@ -355,3 +420,41 @@ func mockSeriesResponse() *logproto.SeriesResponse {
355420
},
356421
}
357422
}
423+
424+
funcremoveWhiteSpace(sstring)string {
425+
returnstrings.Map(func(rrune)rune {
426+
ifr==' '||unicode.IsSpace(r) {
427+
return-1
428+
}
429+
returnr
430+
},s)
431+
}
432+
433+
funcTestSliceToSet(t*testing.T) {
434+
for_,tc:=range []struct {
435+
descstring
436+
slice []string
437+
expectedmap[string]struct{}
438+
}{
439+
{
440+
desc:"empty slice",
441+
slice: []string{},
442+
expected:map[string]struct{}{},
443+
},
444+
{
445+
desc:"single element",
446+
slice: []string{"a"},
447+
expected:map[string]struct{}{"a": {}},
448+
},
449+
{
450+
desc:"multiple elements",
451+
slice: []string{"a","b","c"},
452+
expected:map[string]struct{}{"a": {},"b": {},"c": {}},
453+
},
454+
} {
455+
t.Run(tc.desc,func(t*testing.T) {
456+
actual:=sliceToSet(tc.slice)
457+
require.Equal(t,tc.expected,actual)
458+
})
459+
}
460+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp