Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit1cf80f3

Browse files
committed
function value
1 parent4b54f66 commit1cf80f3

File tree

15 files changed

+245
-128
lines changed

15 files changed

+245
-128
lines changed
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
fromtypingimportUnion
22

33
fromapp.translator.core.models.fieldimportAlias,Field,FieldValue,Keyword
4+
fromapp.translator.core.models.function_valueimportFunctionValue
45
fromapp.translator.core.models.identifierimportIdentifier
56

6-
TOKEN_TYPE=Union[FieldValue,Keyword,Identifier,Field,Alias]
7+
QUERY_TOKEN_TYPE=Union[FieldValue,FunctionValue,Keyword,Identifier,Field,Alias]
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
fromtypingimportOptional,Union
2+
3+
fromapp.translator.core.custom_types.tokensimportSTR_SEARCH_OPERATORS
4+
fromapp.translator.core.models.functions.baseimportFunction
5+
fromapp.translator.core.models.identifierimportIdentifier
6+
fromapp.translator.core.str_value_managerimportStrValue
7+
8+
9+
classFunctionValue:
10+
def__init__(self,function:Function,operator:Identifier,value:Union[int,str,StrValue,list,tuple]):
11+
self.function=function
12+
self.operator=operator
13+
self.values= []
14+
self.__add_value(value)
15+
16+
@property
17+
defvalue(self)->Union[int,str,StrValue,list[Union[int,str,StrValue]]]:
18+
ifisinstance(self.values,list)andlen(self.values)==1:
19+
returnself.values[0]
20+
returnself.values
21+
22+
@value.setter
23+
defvalue(self,new_value:Union[int,str,StrValue,list[Union[int,str,StrValue]]])->None:
24+
self.values= []
25+
self.__add_value(new_value)
26+
27+
def__add_value(self,value:Optional[Union[int,str,StrValue,list,tuple]])->None:
28+
ifvalueandisinstance(value, (list,tuple)):
29+
forvinvalue:
30+
self.__add_value(v)
31+
elif (
32+
value
33+
andisinstance(value,str)
34+
andvalue.isnumeric()
35+
andself.operator.token_typenotinSTR_SEARCH_OPERATORS
36+
):
37+
self.values.append(int(value))
38+
elifvalueisnotNoneandisinstance(value, (int,str)):
39+
self.values.append(value)

‎uncoder-core/app/translator/core/models/query_container.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
fromdatetimeimportdatetime
44
fromtypingimportOptional
55

6-
fromapp.translator.core.constimportTOKEN_TYPE
6+
fromapp.translator.core.constimportQUERY_TOKEN_TYPE
77
fromapp.translator.core.custom_types.meta_infoimportSeverityType
88
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME
99
fromapp.translator.core.models.fieldimportField
@@ -65,6 +65,6 @@ class RawQueryDictContainer:
6565

6666
@dataclass
6767
classTokenizedQueryContainer:
68-
tokens:list[TOKEN_TYPE]
68+
tokens:list[QUERY_TOKEN_TYPE]
6969
meta_info:MetaInfoContainer
7070
functions:ParsedFunctions=field(default_factory=ParsedFunctions)

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@
1818

1919
importre
2020
fromabcimportABC,abstractmethod
21-
fromtypingimportUnion
21+
fromtypingimportOptional,Union
2222

23-
fromapp.translator.core.constimportTOKEN_TYPE
23+
fromapp.translator.core.constimportQUERY_TOKEN_TYPE
2424
fromapp.translator.core.exceptions.parserimportTokenizerGeneralException
2525
fromapp.translator.core.functionsimportPlatformFunctions
2626
fromapp.translator.core.mappingimportBasePlatformMappings,SourceMapping
27-
fromapp.translator.core.models.fieldimportField,FieldValue,Keyword
28-
fromapp.translator.core.models.functions.baseimportParsedFunctions
29-
fromapp.translator.core.models.identifierimportIdentifier
27+
fromapp.translator.core.models.fieldimportField,FieldValue
28+
fromapp.translator.core.models.function_valueimportFunctionValue
29+
fromapp.translator.core.models.functions.baseimportFunction
3030
fromapp.translator.core.models.platform_detailsimportPlatformDetails
3131
fromapp.translator.core.models.query_containerimportRawQueryContainer,TokenizedQueryContainer
3232
fromapp.translator.core.tokenizerimportQueryTokenizer
@@ -55,24 +55,30 @@ class PlatformQueryParser(QueryParser, ABC):
5555
tokenizer:QueryTokenizer=None
5656
platform_functions:PlatformFunctions=None
5757

58-
defget_fields_tokens(self,tokens:list[Union[FieldValue,Keyword,Identifier]])->list[Field]:
59-
return [token.fieldfortokeninself.tokenizer.filter_tokens(tokens,FieldValue)]
60-
61-
defget_tokens_and_source_mappings(
62-
self,query:str,log_sources:dict[str,Union[str,list[str]]]
63-
)->tuple[list[TOKEN_TYPE],list[SourceMapping]]:
58+
defget_query_tokens(self,query:str)->list[QUERY_TOKEN_TYPE]:
6459
ifnotquery:
6560
raiseTokenizerGeneralException("Can't translate empty query. Please provide more details")
66-
tokens=self.tokenizer.tokenize(query=query)
67-
field_tokens=self.get_fields_tokens(tokens=tokens)
61+
returnself.tokenizer.tokenize(query=query)
62+
63+
defget_field_tokens(
64+
self,query_tokens:list[QUERY_TOKEN_TYPE],functions:Optional[list[Function]]=None
65+
)->list[Field]:
66+
field_tokens= []
67+
fortokeninquery_tokens:
68+
ifisinstance(token,FieldValue):
69+
field_tokens.append(token.field)
70+
elifisinstance(token,FunctionValue):
71+
field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function]))
72+
73+
iffunctions:
74+
field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args(functions))
75+
76+
returnfield_tokens
77+
78+
defget_source_mappings(
79+
self,field_tokens:list[Field],log_sources:dict[str,Union[str,list[str]]]
80+
)->list[SourceMapping]:
6881
field_names= [field.source_nameforfieldinfield_tokens]
6982
source_mappings=self.mappings.get_suitable_source_mappings(field_names=field_names,**log_sources)
7083
self.tokenizer.set_field_tokens_generic_names_map(field_tokens,source_mappings,self.mappings.default_mapping)
71-
72-
returntokens,source_mappings
73-
74-
defset_functions_fields_generic_names(
75-
self,functions:ParsedFunctions,source_mappings:list[SourceMapping]
76-
)->None:
77-
field_tokens=self.tokenizer.get_field_tokens_from_func_args(args=functions.functions)
78-
self.tokenizer.set_field_tokens_generic_names_map(field_tokens,source_mappings,self.mappings.default_mapping)
84+
returnsource_mappings

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
fromtypingimportClassVar,Optional,Union
2323

2424
fromapp.translator.constimportDEFAULT_VALUE_TYPE
25-
fromapp.translator.core.constimportTOKEN_TYPE
25+
fromapp.translator.core.constimportQUERY_TOKEN_TYPE
2626
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var,wrap_query_with_meta_info_ctx_var
2727
fromapp.translator.core.custom_types.tokensimportLogicalOperatorType,OperatorType
2828
fromapp.translator.core.custom_types.valuesimportValueType
@@ -32,6 +32,7 @@
3232
fromapp.translator.core.functionsimportPlatformFunctions
3333
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
3434
fromapp.translator.core.models.fieldimportField,FieldField,FieldValue,Keyword,PredefinedField
35+
fromapp.translator.core.models.function_valueimportFunctionValue
3536
fromapp.translator.core.models.functions.baseimportFunction,RenderedFunctions
3637
fromapp.translator.core.models.identifierimportIdentifier
3738
fromapp.translator.core.models.platform_detailsimportPlatformDetails
@@ -258,7 +259,9 @@ def map_predefined_field(self, predefined_field: PredefinedField) -> str:
258259

259260
returnmapped_predefined_field_name
260261

261-
defapply_token(self,token:Union[FieldValue,Keyword,Identifier],source_mapping:SourceMapping)->str:
262+
defapply_token(# noqa: PLR0911
263+
self,token:Union[FieldValue,Function,Keyword,Identifier],source_mapping:SourceMapping
264+
)->str:
262265
ifisinstance(token,FieldValue):
263266
iftoken.alias:
264267
mapped_fields= [token.alias.name]
@@ -286,6 +289,12 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
286289
]
287290
)
288291
returnself.group_token%joinediflen(cross_paired_fields)>1elsejoined
292+
ifisinstance(token,FunctionValue):
293+
func_render=self.platform_functions.manager.get_in_query_render(token.function.name)
294+
rendered_func=func_render.render(token.function,source_mapping)
295+
returnself.field_value_render.apply_field_value(
296+
field=rendered_func,operator=token.operator,value=token.value
297+
)
289298
ifisinstance(token,Function):
290299
func_render=self.platform_functions.manager.get_in_query_render(token.name)
291300
returnfunc_render.render(token,source_mapping)
@@ -296,7 +305,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
296305

297306
returntoken.token_type
298307

299-
defgenerate_query(self,tokens:list[TOKEN_TYPE],source_mapping:SourceMapping)->str:
308+
defgenerate_query(self,tokens:list[QUERY_TOKEN_TYPE],source_mapping:SourceMapping)->str:
300309
result_values= []
301310
unmapped_fields=set()
302311
fortokenintokens:
@@ -412,37 +421,45 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
412421
defined_raw_log_fields.append(prefix)
413422
return"\n".join(defined_raw_log_fields)
414423

424+
def_generate_from_tokenized_query_container_by_source_mapping(
425+
self,query_container:TokenizedQueryContainer,source_mapping:SourceMapping
426+
)->str:
427+
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
428+
prefix=self.generate_prefix(source_mapping.log_source_signature,rendered_functions.rendered_prefix)
429+
430+
ifsource_mapping.raw_log_fields:
431+
defined_raw_log_fields=self.generate_raw_log_fields(
432+
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
433+
)
434+
prefix+=f"\n{defined_raw_log_fields}"
435+
query=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
436+
not_supported_functions=query_container.functions.not_supported+rendered_functions.not_supported
437+
returnself.finalize_query(
438+
prefix=prefix,
439+
query=query,
440+
functions=rendered_functions.rendered,
441+
not_supported_functions=not_supported_functions,
442+
meta_info=query_container.meta_info,
443+
source_mapping=source_mapping,
444+
)
445+
415446
defgenerate_from_tokenized_query_container(self,query_container:TokenizedQueryContainer)->str:
416447
queries_map= {}
417448
errors= []
418449
source_mappings=self._get_source_mappings(query_container.meta_info.source_mapping_ids)
419450

420451
forsource_mappinginsource_mappings:
421-
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
422-
prefix=self.generate_prefix(source_mapping.log_source_signature,rendered_functions.rendered_prefix)
423452
try:
424-
ifsource_mapping.raw_log_fields:
425-
defined_raw_log_fields=self.generate_raw_log_fields(
426-
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
427-
)
428-
prefix+=f"\n{defined_raw_log_fields}"
429-
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
430-
exceptStrictPlatformExceptionaserr:
431-
errors.append(err)
432-
continue
433-
else:
434-
not_supported_functions=query_container.functions.not_supported+rendered_functions.not_supported
435-
finalized_query=self.finalize_query(
436-
prefix=prefix,
437-
query=result,
438-
functions=rendered_functions.rendered,
439-
not_supported_functions=not_supported_functions,
440-
meta_info=query_container.meta_info,
441-
source_mapping=source_mapping,
453+
finalized_query=self._generate_from_tokenized_query_container_by_source_mapping(
454+
query_container,source_mapping
442455
)
443456
ifreturn_only_first_query_ctx_var.get()isTrue:
444457
returnfinalized_query
445458
queries_map[source_mapping.source_id]=finalized_query
459+
exceptStrictPlatformExceptionaserr:
460+
errors.append(err)
461+
continue
462+
446463
ifnotqueries_mapanderrors:
447464
raiseerrors[0]
448465
returnself.finalize(queries_map)

‎uncoder-core/app/translator/core/tokenizer.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,20 @@
2020
fromabcimportABC,abstractmethod
2121
fromtypingimportAny,ClassVar,Optional,Union
2222

23-
fromapp.translator.core.constimportTOKEN_TYPE
23+
fromapp.translator.core.constimportQUERY_TOKEN_TYPE
2424
fromapp.translator.core.custom_types.tokensimportGroupType,LogicalOperatorType,OperatorType
2525
fromapp.translator.core.custom_types.valuesimportValueType
2626
fromapp.translator.core.escape_managerimportEscapeManager
27+
fromapp.translator.core.exceptions.functionsimportNotSupportedFunctionException
2728
fromapp.translator.core.exceptions.parserimport (
2829
QueryParenthesesException,
2930
TokenizerGeneralException,
3031
UnsupportedOperatorException,
3132
)
33+
fromapp.translator.core.functionsimportPlatformFunctions
3234
fromapp.translator.core.mappingimportSourceMapping
3335
fromapp.translator.core.models.fieldimportField,FieldField,FieldValue,Keyword
36+
fromapp.translator.core.models.function_valueimportFunctionValue
3437
fromapp.translator.core.models.functions.baseimportFunction
3538
fromapp.translator.core.models.functions.evalimportEvalArg
3639
fromapp.translator.core.models.functions.group_byimportGroupByFunction
@@ -64,6 +67,7 @@ class QueryTokenizer(BaseTokenizer):
6467

6568
# do not modify, use subclasses to define this attribute
6669
field_pattern:str=None
70+
function_pattern:str=None
6771
_value_pattern:str=None
6872
value_pattern:str=None
6973
multi_value_pattern:str=None
@@ -73,6 +77,7 @@ class QueryTokenizer(BaseTokenizer):
7377
wildcard_symbol=None
7478
escape_manager:EscapeManager=None
7579
str_value_manager:StrValueManager=None
80+
platform_functions:PlatformFunctions=None
7681

7782
def__init_subclass__(cls,**kwargs):
7883
cls._validate_re_patterns()
@@ -268,9 +273,16 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s+"
268273

269274
returnFalse
270275

276+
defsearch_function_value(self,query:str)->tuple[FunctionValue,str]:# noqa: ARG002
277+
raiseNotSupportedFunctionException
278+
279+
@staticmethod
280+
def_check_function_value_match(query:str)->bool:# noqa: ARG004
281+
returnFalse
282+
271283
def_get_next_token(
272284
self,query:str
273-
)->tuple[Union[FieldValue,Keyword,Identifier,list[Union[FieldValue,Identifier]]],str]:
285+
)->tuple[Union[FieldValue,FunctionValue,Keyword,Identifier,list[Union[FieldValue,Identifier]]],str]:
274286
query=query.strip("\n").strip(" ").strip("\n")
275287
ifquery.startswith(GroupType.L_PAREN):
276288
returnIdentifier(token_type=GroupType.L_PAREN),query[1:]
@@ -280,6 +292,8 @@ def _get_next_token(
280292
logical_operator=logical_operator_search.group("logical_operator")
281293
pos=logical_operator_search.end()
282294
returnIdentifier(token_type=logical_operator.lower()),query[pos:]
295+
ifself.platform_functionsandself._check_function_value_match(query):
296+
returnself.search_function_value(query)
283297
ifself._check_field_value_match(query):
284298
returnself.search_field_value(query)
285299
ifself.keyword_patternandre.match(self.keyword_pattern,query):
@@ -288,7 +302,7 @@ def _get_next_token(
288302
raiseTokenizerGeneralException("Unsupported query entry")
289303

290304
@staticmethod
291-
def_validate_parentheses(tokens:list[TOKEN_TYPE])->None:
305+
def_validate_parentheses(tokens:list[QUERY_TOKEN_TYPE])->None:
292306
parentheses= []
293307
fortokenintokens:
294308
ifisinstance(token,Identifier)andtoken.token_typein (GroupType.L_PAREN,GroupType.R_PAREN):
@@ -320,8 +334,9 @@ def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]:
320334

321335
@staticmethod
322336
deffilter_tokens(
323-
tokens:list[TOKEN_TYPE],token_type:Union[type[FieldValue],type[Field],type[Keyword],type[Identifier]]
324-
)->list[TOKEN_TYPE]:
337+
tokens:list[QUERY_TOKEN_TYPE],
338+
token_type:Union[type[FieldValue],type[Field],type[Keyword],type[Identifier]],
339+
)->list[QUERY_TOKEN_TYPE]:
325340
return [tokenfortokenintokensifisinstance(token,token_type)]
326341

327342
defget_field_tokens_from_func_args(# noqa: PLR0912
@@ -339,6 +354,8 @@ def get_field_tokens_from_func_args( # noqa: PLR0912
339354
elifisinstance(arg,FieldValue):
340355
ifarg.field:
341356
result.append(arg.field)
357+
elifisinstance(arg,FunctionValue):
358+
result.extend(self.get_field_tokens_from_func_args(args=[arg.function]))
342359
elifisinstance(arg,GroupByFunction):
343360
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
344361
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))

‎uncoder-core/app/translator/platforms/base/aql/parsers/aql.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@
2727
fromapp.translator.platforms.base.aql.functionsimportAQLFunctions,aql_functions
2828
fromapp.translator.platforms.base.aql.log_source_mapimportLOG_SOURCE_FUNCTIONS_MAP
2929
fromapp.translator.platforms.base.aql.mappingimportAQLMappings,aql_mappings
30-
fromapp.translator.platforms.base.aql.tokenizerimportAQLTokenizer,aql_tokenizer
30+
fromapp.translator.platforms.base.aql.tokenizerimportAQLTokenizer
3131
fromapp.translator.tools.utilsimportget_match_group
3232

3333

3434
classAQLQueryParser(PlatformQueryParser):
35-
tokenizer:AQLTokenizer=aql_tokenizer
35+
tokenizer:AQLTokenizer=AQLTokenizer(aql_functions)
3636
mappings:AQLMappings=aql_mappings
3737
platform_functions:AQLFunctions=aql_functions
3838

@@ -116,10 +116,10 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[
116116

117117
defparse(self,raw_query_container:RawQueryContainer)->TokenizedQueryContainer:
118118
query,log_sources,functions=self._parse_query(raw_query_container.query)
119-
tokens,source_mappings=self.get_tokens_and_source_mappings(query,log_sources)
120-
fields_tokens=self.get_fields_tokens(tokens=tokens)
121-
self.set_functions_fields_generic_names(functions=functions,source_mappings=source_mappings)
119+
query_tokens=self.get_query_tokens(query)
120+
field_tokens=self.get_field_tokens(query_tokens,functions.functions)
121+
source_mappings=self.get_source_mappings(field_tokens,log_sources)
122122
meta_info=raw_query_container.meta_info
123-
meta_info.query_fields=fields_tokens
123+
meta_info.query_fields=field_tokens
124124
meta_info.source_mapping_ids= [source_mapping.source_idforsource_mappinginsource_mappings]
125-
returnTokenizedQueryContainer(tokens=tokens,meta_info=meta_info,functions=functions)
125+
returnTokenizedQueryContainer(tokens=query_tokens,meta_info=meta_info,functions=functions)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp