Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd10cc06

Browse files
authored
Merge pull request#173 from UncoderIO/gis-8193
mapping flow changes, render unmapped fields to comment
2 parentsd9b83b2 +7d4cc57 commitd10cc06

File tree

71 files changed

+324
-251
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+324
-251
lines changed

‎uncoder-core/app/translator/core/exceptions/core.py‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,14 @@ class BasePlatformException(BaseException):
1010

1111

1212
classStrictPlatformException(BasePlatformException):
13-
field_name:str=None
14-
15-
def__init__(
16-
self,platform_name:str,field_name:str,mapping:Optional[str]=None,detected_fields:Optional[list]=None
17-
):
13+
def__init__(self,platform_name:str,fields:list[str],mapping:Optional[str]=None):
1814
message= (
1915
f"Platform{platform_name} has strict mapping. "
20-
f"Source fields:{', '.join(detected_fields)ifdetected_fieldselsefield_name} has no mapping."
16+
f"Source fields:{', '.join(fields)} have no mapping."
2117
f" Mapping file:{mapping}."
2218
ifmapping
2319
else""
2420
)
25-
self.field_name=field_name
2621
super().__init__(message)
2722

2823

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,16 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe
9494
defrender(self,function:Function,source_mapping:SourceMapping)->str:
9595
raiseNotImplementedError
9696

97-
@staticmethod
98-
defmap_field(field:Union[Alias,Field],source_mapping:SourceMapping)->str:
97+
defmap_field(self,field:Union[Alias,Field],source_mapping:SourceMapping)->str:
9998
ifisinstance(field,Alias):
10099
returnfield.name
101100

102-
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
103-
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
104-
ifisinstance(mapped_field,list):
105-
mapped_field=mapped_field[0]
101+
ifisinstance(field,Field):
102+
mappings=self.manager.platform_functions.platform_query_render.mappings
103+
mapped_fields=mappings.map_field(field,source_mapping)
104+
returnmapped_fields[0]
106105

107-
returnmapped_fieldifmapped_fieldelsefield.source_name
106+
raiseNotSupportedFunctionException
108107

109108

110109
classPlatformFunctionsManager:

‎uncoder-core/app/translator/core/mapping.py‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from __future__importannotations
22

33
fromabcimportABC,abstractmethod
4-
fromtypingimportOptional,TypeVar
4+
fromtypingimportTYPE_CHECKING,Optional,TypeVar
55

6+
fromapp.translator.core.exceptions.coreimportStrictPlatformException
7+
fromapp.translator.core.models.platform_detailsimportPlatformDetails
68
fromapp.translator.mappings.utils.load_from_filesimportLoaderFileMappings
79

10+
ifTYPE_CHECKING:
11+
fromapp.translator.core.models.query_tokens.fieldimportField
12+
13+
814
DEFAULT_MAPPING_NAME="default"
915

1016

@@ -85,12 +91,16 @@ def __init__(
8591

8692

8793
classBasePlatformMappings:
94+
details:PlatformDetails=None
95+
96+
is_strict_mapping:bool=False
8897
skip_load_default_mappings:bool=True
8998
extend_default_mapping_with_all_fields:bool=False
9099

91-
def__init__(self,platform_dir:str):
100+
def__init__(self,platform_dir:str,platform_details:PlatformDetails):
92101
self._loader=LoaderFileMappings()
93102
self._platform_dir=platform_dir
103+
self.details=platform_details
94104
self._source_mappings=self.prepare_mapping()
95105

96106
defupdate_default_source_mapping(self,default_mapping:SourceMapping,fields_mapping:FieldsMapping)->None:
@@ -148,6 +158,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
148158
defdefault_mapping(self)->SourceMapping:
149159
returnself._source_mappings[DEFAULT_MAPPING_NAME]
150160

161+
defcheck_fields_mapping_existence(self,field_tokens:list[Field],source_mapping:SourceMapping)->list[str]:
162+
unmapped= []
163+
forfieldinfield_tokens:
164+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
165+
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
166+
ifnotmapped_fieldandfield.source_namenotinunmapped:
167+
unmapped.append(field.source_name)
168+
169+
ifself.is_strict_mappingandunmapped:
170+
raiseStrictPlatformException(
171+
platform_name=self.details.name,fields=unmapped,mapping=source_mapping.source_id
172+
)
173+
174+
returnunmapped
175+
176+
@staticmethod
177+
defmap_field(field:Field,source_mapping:SourceMapping)->list[str]:
178+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
179+
# field can be mapped to corresponding platform field name or list of platform field names
180+
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
181+
182+
ifisinstance(mapped_field,str):
183+
mapped_field= [mapped_field]
184+
185+
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
186+
151187

152188
classBaseCommonPlatformMappings(ABC,BasePlatformMappings):
153189
defprepare_mapping(self)->dict[str,SourceMapping]:

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ class QueryRender(ABC):
184184
details:PlatformDetails=None
185185
is_single_line_comment:bool=False
186186
unsupported_functions_text="Unsupported functions were excluded from the result query:"
187+
unmapped_fields_text="Unmapped fields: "
187188

188189
platform_functions:PlatformFunctions=None
189190

@@ -206,6 +207,11 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:
206207

207208
returnquery
208209

210+
defwrap_with_unmapped_fields(self,query:str,fields:Optional[list[str]])->str:
211+
iffields:
212+
returnquery+"\n\n"+self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}")
213+
returnquery
214+
209215
defwrap_with_comment(self,value:str)->str:
210216
returnf"{self.comment_symbol}{value}"
211217

@@ -216,7 +222,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
216222

217223
classPlatformQueryRender(QueryRender):
218224
mappings:BasePlatformMappings=None
219-
is_strict_mapping:bool=False
220225

221226
or_token="or"
222227
and_token="and"
@@ -247,22 +252,10 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu
247252
defgenerate_functions(self,functions:list[Function],source_mapping:SourceMapping)->RenderedFunctions:
248253
returnself.platform_functions.render(functions,source_mapping)
249254

250-
defmap_field(self,field:Field,source_mapping:SourceMapping)->list[str]:
251-
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
252-
# field can be mapped to corresponding platform field name or list of platform field names
253-
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
254-
ifnotmapped_fieldandself.is_strict_mapping:
255-
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
256-
257-
ifisinstance(mapped_field,str):
258-
mapped_field= [mapped_field]
259-
260-
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
261-
262255
defmap_predefined_field(self,predefined_field:PredefinedField)->str:
263256
ifnot (mapped_predefined_field_name:=self.predefined_fields_map.get(predefined_field.name)):
264-
ifself.is_strict_mapping:
265-
raiseStrictPlatformException(field_name=predefined_field.name,platform_name=self.details.name)
257+
ifself.mappings.is_strict_mapping:
258+
raiseStrictPlatformException(platform_name=self.details.name,fields=[predefined_field.name])
266259

267260
returnpredefined_field.name
268261

@@ -275,7 +268,7 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
275268
eliftoken.predefined_field:
276269
mapped_fields= [self.map_predefined_field(token.predefined_field)]
277270
else:
278-
mapped_fields=self.map_field(token.field,source_mapping)
271+
mapped_fields=self.mappings.map_field(token.field,source_mapping)
279272
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
280273
[
281274
self.field_value_render.apply_field_value(field=field,operator=token.operator,value=token.value)
@@ -285,9 +278,13 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
285278
returnself.group_token%joinediflen(mapped_fields)>1elsejoined
286279
ifisinstance(token,FieldField):
287280
alias_left,field_left=token.alias_left,token.field_left
288-
mapped_fields_left= [alias_left.name]ifalias_leftelseself.map_field(field_left,source_mapping)
281+
mapped_fields_left= (
282+
[alias_left.name]ifalias_leftelseself.mappings.map_field(field_left,source_mapping)
283+
)
289284
alias_right,field_right=token.alias_right,token.field_right
290-
mapped_fields_right= [alias_right.name]ifalias_rightelseself.map_field(field_right,source_mapping)
285+
mapped_fields_right= (
286+
[alias_right.name]ifalias_rightelseself.mappings.map_field(field_right,source_mapping)
287+
)
291288
cross_paired_fields=list(itertools.product(mapped_fields_left,mapped_fields_right))
292289
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
293290
[
@@ -311,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
311308

312309
defgenerate_query(self,tokens:list[QUERY_TOKEN_TYPE],source_mapping:SourceMapping)->str:
313310
result_values= []
314-
unmapped_fields=set()
315311
fortokenintokens:
316-
try:
317-
result_values.append(self.apply_token(token=token,source_mapping=source_mapping))
318-
exceptStrictPlatformExceptionaserr:
319-
unmapped_fields.add(err.field_name)
320-
ifunmapped_fields:
321-
raiseStrictPlatformException(self.details.name,"",source_mapping.source_id,sorted(unmapped_fields))
312+
result_values.append(self.apply_token(token=token,source_mapping=source_mapping))
313+
322314
return"".join(result_values)
323315

324316
defwrap_with_meta_info(self,query:str,meta_info:Optional[MetaInfoContainer])->str:
@@ -351,11 +343,13 @@ def finalize_query(
351343
meta_info:Optional[MetaInfoContainer]=None,
352344
source_mapping:Optional[SourceMapping]=None,# noqa: ARG002
353345
not_supported_functions:Optional[list]=None,
346+
unmapped_fields:Optional[list[str]]=None,
354347
*args,# noqa: ARG002
355348
**kwargs,# noqa: ARG002
356349
)->str:
357350
query=self._join_query_parts(prefix,query,functions)
358351
query=self.wrap_with_meta_info(query,meta_info)
352+
query=self.wrap_with_unmapped_fields(query,unmapped_fields)
359353
returnself.wrap_with_not_supported_functions(query,not_supported_functions)
360354

361355
@staticmethod
@@ -417,8 +411,10 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
417411
mapped_field=source_mapping.fields_mapping.get_platform_field_name(
418412
generic_field_name=generic_field_name
419413
)
420-
ifnotmapped_fieldandself.is_strict_mapping:
421-
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
414+
ifnotmapped_fieldandself.mappings.is_strict_mapping:
415+
raiseStrictPlatformException(
416+
platform_name=self.details.name,fields=[field.source_name],mapping=source_mapping.source_id
417+
)
422418
ifprefix_list:=self.process_raw_log_field_prefix(field=mapped_field,source_mapping=source_mapping):
423419
forprefixinprefix_list:
424420
ifprefixnotindefined_raw_log_fields:
@@ -428,6 +424,9 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
428424
def_generate_from_tokenized_query_container_by_source_mapping(
429425
self,query_container:TokenizedQueryContainer,source_mapping:SourceMapping
430426
)->str:
427+
unmapped_fields=self.mappings.check_fields_mapping_existence(
428+
query_container.meta_info.query_fields,source_mapping
429+
)
431430
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
432431
prefix=self.generate_prefix(source_mapping.log_source_signature,rendered_functions.rendered_prefix)
433432

@@ -443,6 +442,7 @@ def _generate_from_tokenized_query_container_by_source_mapping(
443442
query=query,
444443
functions=rendered_functions.rendered,
445444
not_supported_functions=not_supported_functions,
445+
unmapped_fields=unmapped_fields,
446446
meta_info=query_container.meta_info,
447447
source_mapping=source_mapping,
448448
)

‎uncoder-core/app/translator/platforms/athena/const.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99
"alt_platform_name":"OCSF",
1010
}
1111

12-
athena_details=PlatformDetails(**ATHENA_QUERY_DETAILS)
12+
athena_query_details=PlatformDetails(**ATHENA_QUERY_DETAILS)

‎uncoder-core/app/translator/platforms/athena/mapping.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
fromtypingimportOptional
22

33
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
4+
fromapp.translator.platforms.athena.constimportathena_query_details
45

56

67
classAthenaLogSourceSignature(LogSourceSignature):
@@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s
4041
returnsuitable_source_mappings
4142

4243

43-
athena_mappings=AthenaMappings(platform_dir="athena")
44+
athena_query_mappings=AthenaMappings(platform_dir="athena",platform_details=athena_query_details)

‎uncoder-core/app/translator/platforms/athena/parsers/athena.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818

1919
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2020
fromapp.translator.managersimportparser_manager
21-
fromapp.translator.platforms.athena.constimportathena_details
22-
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_mappings
21+
fromapp.translator.platforms.athena.constimportathena_query_details
22+
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_query_mappings
2323
fromapp.translator.platforms.base.sql.parsers.sqlimportSqlQueryParser
2424

2525

2626
@parser_manager.register_supported_by_roota
2727
classAthenaQueryParser(SqlQueryParser):
28-
details:PlatformDetails=athena_details
29-
mappings:AthenaMappings=athena_mappings
28+
details:PlatformDetails=athena_query_details
29+
mappings:AthenaMappings=athena_query_mappings
3030
query_delimiter_pattern=r"\sFROM\s\S*\sWHERE\s"
3131
table_pattern=r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"

‎uncoder-core/app/translator/platforms/athena/renders/athena.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,19 @@
1919

2020
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2121
fromapp.translator.managersimportrender_manager
22-
fromapp.translator.platforms.athena.constimportathena_details
23-
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_mappings
22+
fromapp.translator.platforms.athena.constimportathena_query_details
23+
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_query_mappings
2424
fromapp.translator.platforms.base.sql.renders.sqlimportSqlFieldValueRender,SqlQueryRender
2525

2626

2727
classAthenaFieldValueRender(SqlFieldValueRender):
28-
details:PlatformDetails=athena_details
28+
details:PlatformDetails=athena_query_details
2929

3030

3131
@render_manager.register
3232
classAthenaQueryRender(SqlQueryRender):
33-
details:PlatformDetails=athena_details
34-
mappings:AthenaMappings=athena_mappings
33+
details:PlatformDetails=athena_query_details
34+
mappings:AthenaMappings=athena_query_mappings
3535

3636
or_token="OR"
3737

‎uncoder-core/app/translator/platforms/athena/renders/athena_cti.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@
2020
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2121
fromapp.translator.core.render_ctiimportRenderCTI
2222
fromapp.translator.managersimportrender_cti_manager
23-
fromapp.translator.platforms.athena.constimportathena_details
23+
fromapp.translator.platforms.athena.constimportathena_query_details
2424
fromapp.translator.platforms.athena.mappings.athena_ctiimportDEFAULT_ATHENA_MAPPING
2525

2626

2727
@render_cti_manager.register
2828
classAthenaCTI(RenderCTI):
29-
details:PlatformDetails=athena_details
29+
details:PlatformDetails=athena_query_details
3030

3131
field_value_template:str="{key} = '{value}'"
3232
or_operator:str=" OR "

‎uncoder-core/app/translator/platforms/base/aql/mapping.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,3 @@ def get_suitable_source_mappings(
9090
suitable_source_mappings= [self._source_mappings[DEFAULT_MAPPING_NAME]]
9191

9292
returnsuitable_source_mappings
93-
94-
95-
aql_mappings=AQLMappings(platform_dir="qradar")

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp