Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb94266a

Browse files
committed
FieldField class, renders improvements
1 parent7606f21 commitb94266a

File tree

37 files changed

+221
-187
lines changed

37 files changed

+221
-187
lines changed
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
fromtypingimportUnion
2+
3+
fromapp.translator.core.models.fieldimportAlias,Field,FieldValue,Keyword
4+
fromapp.translator.core.models.identifierimportIdentifier
5+
6+
TOKEN_TYPE=Union[FieldValue,Keyword,Identifier,Field,Alias]
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
fromcontextvarsimportContextVar
2+
fromtypingimportOptional
23

34
return_only_first_query_ctx_var:ContextVar[bool]=ContextVar("return_only_first_query_ctx_var",default=False)
45
"""Set to True to return only first query if rendered multiple options"""
6+
7+
wrap_query_with_meta_info_ctx_var:ContextVar[bool]=ContextVar("wrap_query_with_meta_info_ctx_var",default=True)
8+
"""Set to False not to wrap query with meta info commentary"""
9+
10+
preset_log_source_str_ctx_var:ContextVar[Optional[str]]=ContextVar("preset_log_source_str_ctx_var",default=None)

‎uncoder-core/app/translator/core/models/field.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
3737
self.__generic_names_map=generic_names_map
3838

3939

40+
classFieldField:
41+
def__init__(
42+
self,
43+
source_name_left:str,
44+
operator:Identifier,
45+
source_name_right:str,
46+
is_alias_left:bool=False,
47+
is_alias_right:bool=False,
48+
):
49+
self.field_left=Field(source_name=source_name_left)
50+
self.alias_left=Alias(name=source_name_left)ifis_alias_leftelseNone
51+
self.operator=operator
52+
self.field_right=Field(source_name=source_name_right)
53+
self.alias_right=Alias(name=source_name_right)ifis_alias_rightelseNone
54+
55+
4056
classFieldValue:
4157
def__init__(
4258
self,

‎uncoder-core/app/translator/core/models/query_container.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
fromdatetimeimportdatetime
44
fromtypingimportOptional
55

6+
fromapp.translator.core.constimportTOKEN_TYPE
67
fromapp.translator.core.custom_types.meta_infoimportSeverityType
78
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME
89
fromapp.translator.core.models.fieldimportField
910
fromapp.translator.core.models.functions.baseimportParsedFunctions
10-
fromapp.translator.core.tokenizerimportTOKEN_TYPE
1111

1212

1313
classMetaInfoContainer:

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
fromabcimportABC,abstractmethod
2121
fromtypingimportUnion
2222

23+
fromapp.translator.core.constimportTOKEN_TYPE
2324
fromapp.translator.core.exceptions.parserimportTokenizerGeneralException
2425
fromapp.translator.core.functionsimportPlatformFunctions
2526
fromapp.translator.core.mappingimportBasePlatformMappings,SourceMapping
@@ -28,7 +29,7 @@
2829
fromapp.translator.core.models.identifierimportIdentifier
2930
fromapp.translator.core.models.platform_detailsimportPlatformDetails
3031
fromapp.translator.core.models.query_containerimportRawQueryContainer,TokenizedQueryContainer
31-
fromapp.translator.core.tokenizerimportTOKEN_TYPE,QueryTokenizer
32+
fromapp.translator.core.tokenizerimportQueryTokenizer
3233

3334

3435
classQueryParser(ABC):

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 57 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -16,36 +16,36 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19-
19+
importitertools
2020
fromabcimportABC,abstractmethod
2121
fromcollections.abcimportCallable
2222
fromtypingimportClassVar,Optional,Union
2323

2424
fromapp.translator.constimportDEFAULT_VALUE_TYPE
25-
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var
25+
fromapp.translator.core.constimportTOKEN_TYPE
26+
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var,wrap_query_with_meta_info_ctx_var
2627
fromapp.translator.core.custom_types.tokensimportLogicalOperatorType,OperatorType
2728
fromapp.translator.core.custom_types.valuesimportValueType
2829
fromapp.translator.core.escape_managerimportEscapeManager
2930
fromapp.translator.core.exceptions.coreimportNotImplementedException,StrictPlatformException
3031
fromapp.translator.core.exceptions.parserimportUnsupportedOperatorException
3132
fromapp.translator.core.functionsimportPlatformFunctions
3233
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
33-
fromapp.translator.core.models.fieldimportField,FieldValue,Keyword
34+
fromapp.translator.core.models.fieldimportField,FieldField,FieldValue,Keyword
3435
fromapp.translator.core.models.functions.baseimportFunction,RenderedFunctions
3536
fromapp.translator.core.models.identifierimportIdentifier
3637
fromapp.translator.core.models.platform_detailsimportPlatformDetails
3738
fromapp.translator.core.models.query_containerimportMetaInfoContainer,RawQueryContainer,TokenizedQueryContainer
3839
fromapp.translator.core.str_value_managerimportStrValue,StrValueManager
39-
fromapp.translator.core.tokenizerimportTOKEN_TYPE
4040

4141

42-
classBaseQueryFieldValue(ABC):
42+
classBaseFieldValueRender(ABC):
4343
details:PlatformDetails=None
4444
escape_manager:EscapeManager=None
4545
str_value_manager:StrValueManager=None
4646

4747
def__init__(self,or_token:str):
48-
self.field_value:dict[str,Callable[[str,DEFAULT_VALUE_TYPE],str]]= {
48+
self.modifiers_map:dict[str,Callable[[str,DEFAULT_VALUE_TYPE],str]]= {
4949
OperatorType.EQ:self.equal_modifier,
5050
OperatorType.NOT_EQ:self.not_equal_modifier,
5151
OperatorType.LT:self.less_modifier,
@@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value)
155155
returnself.escape_manager.escape(value,value_type)
156156

157157
defapply_field_value(self,field:str,operator:Identifier,value:DEFAULT_VALUE_TYPE)->str:
158-
ifmodifier_function:=self.field_value.get(operator.token_type):
158+
ifmodifier_function:=self.modifiers_map.get(operator.token_type):
159159
returnmodifier_function(field,value)
160160
raiseUnsupportedOperatorException(operator.token_type)
161161

162162

163+
classBaseFieldFieldRender(ABC):
164+
operators_map:ClassVar[dict[str,str]]= {}
165+
166+
defapply_field_field(self,field_left:str,operator:Identifier,field_right:str)->str:
167+
ifmapped_operator:=self.operators_map.get(operator.token_type):
168+
returnf"{field_left}{mapped_operator}{field_right}"
169+
raiseUnsupportedOperatorException(operator.token_type)
170+
171+
163172
classQueryRender(ABC):
164173
comment_symbol:str=None
165174
details:PlatformDetails=None
@@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str:
180189
not_supported_functions_str="\n".join(line_template+func.lstrip()forfuncinnot_supported_functions)
181190
return"\n\n"+self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}")
182191

192+
defwrap_with_not_supported_functions(self,query:str,not_supported_functions:Optional[list]=None)->str:
193+
ifnot_supported_functionsandwrap_query_with_meta_info_ctx_var.get():
194+
rendered_not_supported=self.render_not_supported_functions(not_supported_functions)
195+
returnquery+rendered_not_supported
196+
197+
returnquery
198+
183199
defwrap_with_comment(self,value:str)->str:
184200
returnf"{self.comment_symbol}{value}"
185201

@@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender):
199215
group_token="(%s)"
200216
query_parts_delimiter=" "
201217

202-
field_value_map=BaseQueryFieldValue(or_token=or_token)
218+
field_field_render=BaseFieldFieldRender()
219+
field_value_render=BaseFieldValueRender(or_token=or_token)
203220

204221
raw_log_field_pattern_map:ClassVar[dict[str,str]]=None
205222

206223
def__init__(self):
207224
super().__init__()
208-
self.operator_map= {
225+
self.logical_operators_map= {
209226
LogicalOperatorType.AND:f"{self.and_token} ",
210227
LogicalOperatorType.OR:f"{self.or_token} ",
211228
LogicalOperatorType.NOT:f"{self.not_token} ",
@@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
233250

234251
defapply_token(self,token:Union[FieldValue,Keyword,Identifier],source_mapping:SourceMapping)->str:
235252
ifisinstance(token,FieldValue):
236-
iftoken.alias:
237-
field_name=token.alias.name
238-
else:
239-
mapped_fields=self.map_field(token.field,source_mapping)
240-
iflen(mapped_fields)>1:
241-
returnself.group_token%self.operator_map[LogicalOperatorType.OR].join(
242-
[
243-
self.field_value_map.apply_field_value(
244-
field=field,operator=token.operator,value=token.value
245-
)
246-
forfieldinmapped_fields
247-
]
248-
)
249-
250-
field_name=mapped_fields[0]
251-
252-
returnself.field_value_map.apply_field_value(field=field_name,operator=token.operator,value=token.value)
253-
253+
mapped_fields= [token.alias.name]iftoken.aliaselseself.map_field(token.field,source_mapping)
254+
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
255+
[
256+
self.field_value_render.apply_field_value(field=field,operator=token.operator,value=token.value)
257+
forfieldinmapped_fields
258+
]
259+
)
260+
returnself.group_token%joinediflen(mapped_fields)>1elsejoined
261+
ifisinstance(token,FieldField):
262+
alias_left,field_left=token.alias_left,token.field_left
263+
mapped_fields_left= [alias_left.name]ifalias_leftelseself.map_field(field_left,source_mapping)
264+
alias_right,field_right=token.alias_right,token.field_right
265+
mapped_fields_right= [alias_right.name]ifalias_rightelseself.map_field(field_right,source_mapping)
266+
cross_paired_fields=list(itertools.product(mapped_fields_left,mapped_fields_right))
267+
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
268+
[
269+
self.field_field_render.apply_field_field(pair[0],token.operator,pair[1])
270+
forpairincross_paired_fields
271+
]
272+
)
273+
returnself.group_token%joinediflen(cross_paired_fields)>1elsejoined
254274
ifisinstance(token,Function):
255275
func_render=self.platform_functions.manager.get_in_query_render(token.name)
256276
returnfunc_render.render(token,source_mapping)
257277
ifisinstance(token,Keyword):
258-
returnself.field_value_map.apply_field_value(field="",operator=token.operator,value=token.value)
278+
returnself.field_value_render.apply_field_value(field="",operator=token.operator,value=token.value)
259279
iftoken.token_typeinLogicalOperatorType:
260-
returnself.operator_map.get(token.token_type)
280+
returnself.logical_operators_map.get(token.token_type)
261281

262282
returntoken.token_type
263283

@@ -267,8 +287,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping
267287
result_values.append(self.apply_token(token=token,source_mapping=source_mapping))
268288
return"".join(result_values)
269289

270-
defwrap_query_with_meta_info(self,meta_info:MetaInfoContainer,query:str)->str:
271-
ifmeta_infoand (meta_info.idormeta_info.title):
290+
defwrap_with_meta_info(self,query:str,meta_info:MetaInfoContainer)->str:
291+
ifwrap_query_with_meta_info_ctx_var.get()andmeta_infoand (meta_info.idormeta_info.title):
272292
meta_info_dict= {
273293
"name: ":meta_info.title,
274294
"uuid: ":meta_info.id,
@@ -301,11 +321,8 @@ def finalize_query(
301321
**kwargs,# noqa: ARG002
302322
)->str:
303323
query=self._join_query_parts(prefix,query,functions)
304-
query=self.wrap_query_with_meta_info(meta_info=meta_info,query=query)
305-
ifnot_supported_functions:
306-
rendered_not_supported=self.render_not_supported_functions(not_supported_functions)
307-
returnquery+rendered_not_supported
308-
returnquery
324+
query=self.wrap_with_meta_info(query,meta_info)
325+
returnself.wrap_with_not_supported_functions(query,not_supported_functions)
309326

310327
@staticmethod
311328
defunique_queries(queries_map:dict[str,str])->dict[str,dict[str]]:
@@ -336,7 +353,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp
336353

337354
returnsource_mappings
338355

339-
def_generate_from_raw_query_container(self,query_container:RawQueryContainer)->str:
356+
defgenerate_from_raw_query_container(self,query_container:RawQueryContainer)->str:
340357
returnself.finalize_query(
341358
prefix="",query=query_container.query,functions="",meta_info=query_container.meta_info
342359
)
@@ -374,7 +391,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
374391
defined_raw_log_fields.append(prefix)
375392
return"\n".join(defined_raw_log_fields)
376393

377-
def_generate_from_tokenized_query_container(self,query_container:TokenizedQueryContainer)->str:
394+
defgenerate_from_tokenized_query_container(self,query_container:TokenizedQueryContainer)->str:
378395
queries_map= {}
379396
errors= []
380397
source_mappings=self._get_source_mappings(query_container.meta_info.source_mapping_ids)
@@ -411,6 +428,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
411428

412429
defgenerate(self,query_container:Union[RawQueryContainer,TokenizedQueryContainer])->str:
413430
ifisinstance(query_container,RawQueryContainer):
414-
returnself._generate_from_raw_query_container(query_container)
431+
returnself.generate_from_raw_query_container(query_container)
415432

416-
returnself._generate_from_tokenized_query_container(query_container)
433+
returnself.generate_from_tokenized_query_container(query_container)

‎uncoder-core/app/translator/core/tokenizer.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
fromabcimportABC,abstractmethod
2121
fromtypingimportAny,ClassVar,Optional,Union
2222

23+
fromapp.translator.core.constimportTOKEN_TYPE
2324
fromapp.translator.core.custom_types.tokensimportGroupType,LogicalOperatorType,OperatorType
2425
fromapp.translator.core.custom_types.valuesimportValueType
2526
fromapp.translator.core.escape_managerimportEscapeManager
@@ -29,18 +30,18 @@
2930
UnsupportedOperatorException,
3031
)
3132
fromapp.translator.core.mappingimportSourceMapping
32-
fromapp.translator.core.models.fieldimportField,FieldValue,Keyword
33+
fromapp.translator.core.models.fieldimportField,FieldField,FieldValue,Keyword
3334
fromapp.translator.core.models.functions.baseimportFunction
3435
fromapp.translator.core.models.functions.evalimportEvalArg
3536
fromapp.translator.core.models.functions.group_byimportGroupByFunction
37+
fromapp.translator.core.models.functions.joinimportJoinFunction
3638
fromapp.translator.core.models.functions.renameimportRenameArg
3739
fromapp.translator.core.models.functions.sortimportSortArg
40+
fromapp.translator.core.models.functions.unionimportUnionFunction
3841
fromapp.translator.core.models.identifierimportIdentifier
3942
fromapp.translator.core.str_value_managerimportStrValue,StrValueManager
4043
fromapp.translator.tools.utilsimportget_match_group
4144

42-
TOKEN_TYPE=Union[FieldValue,Keyword,Identifier,Field]
43-
4445

4546
classBaseTokenizer(ABC):
4647
@abstractmethod
@@ -323,20 +324,31 @@ def filter_tokens(
323324
)->list[TOKEN_TYPE]:
324325
return [tokenfortokenintokensifisinstance(token,token_type)]
325326

326-
defget_field_tokens_from_func_args(
327+
defget_field_tokens_from_func_args(# noqa: PLR0912
327328
self,args:list[Union[Field,FieldValue,Keyword,Identifier,Function,SortArg]]
328329
)->list[Field]:
329330
result= []
330331
forarginargs:
331332
ifisinstance(arg,Field):
332333
result.append(arg)
334+
elifisinstance(arg,FieldField):
335+
ifnotarg.alias_leftorarg.alias_left.name!=arg.field_left.source_name:
336+
result.append(arg.field_left)
337+
ifnotarg.alias_rightorarg.alias_right.name!=arg.field_right.source_name:
338+
result.append(arg.field_right)
333339
elifisinstance(arg,FieldValue):
334340
ifnotarg.aliasorarg.alias.name!=arg.field.source_name:
335341
result.append(arg.field)
336342
elifisinstance(arg,GroupByFunction):
337343
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
338344
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))
339345
result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_]))
346+
elifisinstance(arg, (JoinFunction,UnionFunction)):
347+
result.extend(self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.tokens))
348+
result.extend(
349+
self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.functions.functions)
350+
)
351+
result.extend(self.get_field_tokens_from_func_args(args=arg.condition))
340352
elifisinstance(arg,Function):
341353
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
342354
elifisinstance(arg,SortArg)andisinstance(arg.field,Field):

‎uncoder-core/app/translator/platforms/athena/renders/athena.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@
2121
fromapp.translator.managersimportrender_manager
2222
fromapp.translator.platforms.athena.constimportathena_details
2323
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_mappings
24-
fromapp.translator.platforms.base.sql.renders.sqlimportSqlFieldValue,SqlQueryRender
24+
fromapp.translator.platforms.base.sql.renders.sqlimportSqlFieldValueRender,SqlQueryRender
2525

2626

27-
classAthenaFieldValue(SqlFieldValue):
27+
classAthenaFieldValueRender(SqlFieldValueRender):
2828
details:PlatformDetails=athena_details
2929

3030

@@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender):
3535

3636
or_token="OR"
3737

38-
field_value_map=AthenaFieldValue(or_token=or_token)
38+
field_value_render=AthenaFieldValueRender(or_token=or_token)
3939
comment_symbol="--"
4040
is_single_line_comment=True
4141

‎uncoder-core/app/translator/platforms/base/aql/renders/aql.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121

2222
fromapp.translator.constimportDEFAULT_VALUE_TYPE
2323
fromapp.translator.core.custom_types.valuesimportValueType
24-
fromapp.translator.core.renderimportBaseQueryFieldValue,PlatformQueryRender
24+
fromapp.translator.core.renderimportBaseFieldValueRender,PlatformQueryRender
2525
fromapp.translator.core.str_value_managerimportStrValue
2626
fromapp.translator.platforms.base.aql.mappingimportAQLLogSourceSignature,AQLMappings,aql_mappings
2727
fromapp.translator.platforms.base.aql.str_value_managerimportaql_str_value_manager
2828

2929

30-
classAQLFieldValue(BaseQueryFieldValue):
30+
classAQLFieldValueRender(BaseFieldValueRender):
3131
str_value_manager=aql_str_value_manager
3232

3333
@staticmethod
@@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender):
127127
and_token="AND"
128128
not_token="NOT"
129129

130-
field_value_map=AQLFieldValue(or_token=or_token)
131-
132130
defgenerate_prefix(self,log_source_signature:AQLLogSourceSignature,functions_prefix:str="")->str:# noqa: ARG002
133131
table=str(log_source_signature)
134132
extra_condition=log_source_signature.extra_condition

‎uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121

2222
fromapp.translator.constimportDEFAULT_VALUE_TYPE
2323
fromapp.translator.core.custom_types.valuesimportValueType
24-
fromapp.translator.core.renderimportBaseQueryFieldValue,PlatformQueryRender
24+
fromapp.translator.core.renderimportBaseFieldValueRender,PlatformQueryRender
2525
fromapp.translator.core.str_value_managerimportStrValue
2626
fromapp.translator.platforms.base.lucene.mappingimportLuceneLogSourceSignature
2727
fromapp.translator.platforms.base.lucene.str_value_managerimportlucene_str_value_manager
2828

2929

30-
classLuceneFieldValue(BaseQueryFieldValue):
30+
classLuceneFieldValueRender(BaseFieldValueRender):
3131
str_value_manager=lucene_str_value_manager
3232

3333
@staticmethod

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp