Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit14ec9a0

Browse files
committed
add unmapped fields to comment
1 parenta4e8cda commit14ec9a0

File tree

69 files changed

+317
-194
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+317
-194
lines changed

‎uncoder-core/app/translator/core/mapping.py‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from __future__importannotations
22

33
fromabcimportABC,abstractmethod
4-
fromtypingimportOptional,TypeVar
4+
fromtypingimportTYPE_CHECKING,Optional,TypeVar
55

6+
fromapp.translator.core.exceptions.coreimportStrictPlatformException
7+
fromapp.translator.core.models.platform_detailsimportPlatformDetails
68
fromapp.translator.mappings.utils.load_from_filesimportLoaderFileMappings
79

10+
ifTYPE_CHECKING:
11+
fromapp.translator.core.models.query_tokens.fieldimportField
12+
13+
814
DEFAULT_MAPPING_NAME="default"
915

1016

@@ -85,12 +91,16 @@ def __init__(
8591

8692

8793
classBasePlatformMappings:
94+
details:PlatformDetails=None
95+
96+
is_strict_mapping:bool=False
8897
skip_load_default_mappings:bool=True
8998
extend_default_mapping_with_all_fields:bool=False
9099

91-
def__init__(self,platform_dir:str):
100+
def__init__(self,platform_dir:str,platform_details:PlatformDetails):
92101
self._loader=LoaderFileMappings()
93102
self._platform_dir=platform_dir
103+
self.details=platform_details
94104
self._source_mappings=self.prepare_mapping()
95105

96106
defupdate_default_source_mapping(self,default_mapping:SourceMapping,fields_mapping:FieldsMapping)->None:
@@ -148,6 +158,29 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
148158
defdefault_mapping(self)->SourceMapping:
149159
returnself._source_mappings[DEFAULT_MAPPING_NAME]
150160

161+
defcheck_fields_mapping_existence(self,field_tokens:list[Field],source_mapping:SourceMapping)->list[Field]:
162+
not_mapped= []
163+
forfieldinfield_tokens:
164+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
165+
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
166+
ifnotmapped_field:
167+
ifself.is_strict_mapping:
168+
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
169+
not_mapped.append(field)
170+
171+
returnnot_mapped
172+
173+
@staticmethod
174+
defmap_field(field:Field,source_mapping:SourceMapping)->list[str]:
175+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
176+
# field can be mapped to corresponding platform field name or list of platform field names
177+
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
178+
179+
ifisinstance(mapped_field,str):
180+
mapped_field= [mapped_field]
181+
182+
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
183+
151184

152185
classBaseCommonPlatformMappings(ABC,BasePlatformMappings):
153186
defprepare_mapping(self)->dict[str,SourceMapping]:

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ class QueryRender(ABC):
184184
details:PlatformDetails=None
185185
is_single_line_comment:bool=False
186186
unsupported_functions_text="Unsupported functions were excluded from the result query:"
187+
unmapped_fields_text="Unmapped fields: "
187188

188189
platform_functions:PlatformFunctions=None
189190

@@ -206,6 +207,12 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:
206207

207208
returnquery
208209

210+
defwrap_with_unmapped_fields(self,query:str,fields:Optional[list[Field]])->str:
211+
iffields:
212+
joined=", ".join(field.source_nameforfieldinfields)
213+
returnquery+"\n\n"+self.wrap_with_comment(f"{self.unmapped_fields_text}{joined}")
214+
returnquery
215+
209216
defwrap_with_comment(self,value:str)->str:
210217
returnf"{self.comment_symbol}{value}"
211218

@@ -216,7 +223,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
216223

217224
classPlatformQueryRender(QueryRender):
218225
mappings:BasePlatformMappings=None
219-
is_strict_mapping:bool=False
220226

221227
or_token="or"
222228
and_token="and"
@@ -247,21 +253,9 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu
247253
defgenerate_functions(self,functions:list[Function],source_mapping:SourceMapping)->RenderedFunctions:
248254
returnself.platform_functions.render(functions,source_mapping)
249255

250-
defmap_field(self,field:Field,source_mapping:SourceMapping)->list[str]:
251-
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
252-
# field can be mapped to corresponding platform field name or list of platform field names
253-
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
254-
ifnotmapped_fieldandself.is_strict_mapping:
255-
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
256-
257-
ifisinstance(mapped_field,str):
258-
mapped_field= [mapped_field]
259-
260-
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
261-
262256
defmap_predefined_field(self,predefined_field:PredefinedField)->str:
263257
ifnot (mapped_predefined_field_name:=self.predefined_fields_map.get(predefined_field.name)):
264-
ifself.is_strict_mapping:
258+
ifself.mappings.is_strict_mapping:
265259
raiseStrictPlatformException(field_name=predefined_field.name,platform_name=self.details.name)
266260

267261
returnpredefined_field.name
@@ -275,7 +269,7 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
275269
eliftoken.predefined_field:
276270
mapped_fields= [self.map_predefined_field(token.predefined_field)]
277271
else:
278-
mapped_fields=self.map_field(token.field,source_mapping)
272+
mapped_fields=self.mappings.map_field(token.field,source_mapping)
279273
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
280274
[
281275
self.field_value_render.apply_field_value(field=field,operator=token.operator,value=token.value)
@@ -285,9 +279,13 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
285279
returnself.group_token%joinediflen(mapped_fields)>1elsejoined
286280
ifisinstance(token,FieldField):
287281
alias_left,field_left=token.alias_left,token.field_left
288-
mapped_fields_left= [alias_left.name]ifalias_leftelseself.map_field(field_left,source_mapping)
282+
mapped_fields_left= (
283+
[alias_left.name]ifalias_leftelseself.mappings.map_field(field_left,source_mapping)
284+
)
289285
alias_right,field_right=token.alias_right,token.field_right
290-
mapped_fields_right= [alias_right.name]ifalias_rightelseself.map_field(field_right,source_mapping)
286+
mapped_fields_right= (
287+
[alias_right.name]ifalias_rightelseself.mappings.map_field(field_right,source_mapping)
288+
)
291289
cross_paired_fields=list(itertools.product(mapped_fields_left,mapped_fields_right))
292290
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
293291
[
@@ -351,11 +349,13 @@ def finalize_query(
351349
meta_info:Optional[MetaInfoContainer]=None,
352350
source_mapping:Optional[SourceMapping]=None,# noqa: ARG002
353351
not_supported_functions:Optional[list]=None,
352+
unmapped_fields:Optional[list[Field]]=None,
354353
*args,# noqa: ARG002
355354
**kwargs,# noqa: ARG002
356355
)->str:
357356
query=self._join_query_parts(prefix,query,functions)
358357
query=self.wrap_with_meta_info(query,meta_info)
358+
query=self.wrap_with_unmapped_fields(query,unmapped_fields)
359359
returnself.wrap_with_not_supported_functions(query,not_supported_functions)
360360

361361
@staticmethod
@@ -417,7 +417,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
417417
mapped_field=source_mapping.fields_mapping.get_platform_field_name(
418418
generic_field_name=generic_field_name
419419
)
420-
ifnotmapped_fieldandself.is_strict_mapping:
420+
ifnotmapped_fieldandself.mappings.is_strict_mapping:
421421
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
422422
ifprefix_list:=self.process_raw_log_field_prefix(field=mapped_field,source_mapping=source_mapping):
423423
forprefixinprefix_list:
@@ -428,6 +428,9 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
428428
def_generate_from_tokenized_query_container_by_source_mapping(
429429
self,query_container:TokenizedQueryContainer,source_mapping:SourceMapping
430430
)->str:
431+
unmapped_fields=self.mappings.check_fields_mapping_existence(
432+
query_container.meta_info.query_fields,source_mapping
433+
)
431434
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
432435
prefix=self.generate_prefix(source_mapping.log_source_signature,rendered_functions.rendered_prefix)
433436

@@ -443,6 +446,7 @@ def _generate_from_tokenized_query_container_by_source_mapping(
443446
query=query,
444447
functions=rendered_functions.rendered,
445448
not_supported_functions=not_supported_functions,
449+
unmapped_fields=unmapped_fields,
446450
meta_info=query_container.meta_info,
447451
source_mapping=source_mapping,
448452
)

‎uncoder-core/app/translator/platforms/athena/const.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99
"alt_platform_name":"OCSF",
1010
}
1111

12-
athena_details=PlatformDetails(**ATHENA_QUERY_DETAILS)
12+
athena_query_details=PlatformDetails(**ATHENA_QUERY_DETAILS)

‎uncoder-core/app/translator/platforms/athena/mapping.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
fromtypingimportOptional
22

33
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
4+
fromapp.translator.platforms.athena.constimportathena_query_details
45

56

67
classAthenaLogSourceSignature(LogSourceSignature):
@@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s
4041
returnsuitable_source_mappings
4142

4243

43-
athena_mappings=AthenaMappings(platform_dir="athena")
44+
athena_query_mappings=AthenaMappings(platform_dir="athena",platform_details=athena_query_details)

‎uncoder-core/app/translator/platforms/athena/parsers/athena.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818

1919
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2020
fromapp.translator.managersimportparser_manager
21-
fromapp.translator.platforms.athena.constimportathena_details
22-
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_mappings
21+
fromapp.translator.platforms.athena.constimportathena_query_details
22+
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_query_mappings
2323
fromapp.translator.platforms.base.sql.parsers.sqlimportSqlQueryParser
2424

2525

2626
@parser_manager.register_supported_by_roota
2727
classAthenaQueryParser(SqlQueryParser):
28-
details:PlatformDetails=athena_details
29-
mappings:AthenaMappings=athena_mappings
28+
details:PlatformDetails=athena_query_details
29+
mappings:AthenaMappings=athena_query_mappings
3030
query_delimiter_pattern=r"\sFROM\s\S*\sWHERE\s"
3131
table_pattern=r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"

‎uncoder-core/app/translator/platforms/athena/renders/athena.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,19 @@
1919

2020
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2121
fromapp.translator.managersimportrender_manager
22-
fromapp.translator.platforms.athena.constimportathena_details
23-
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_mappings
22+
fromapp.translator.platforms.athena.constimportathena_query_details
23+
fromapp.translator.platforms.athena.mappingimportAthenaMappings,athena_query_mappings
2424
fromapp.translator.platforms.base.sql.renders.sqlimportSqlFieldValueRender,SqlQueryRender
2525

2626

2727
classAthenaFieldValueRender(SqlFieldValueRender):
28-
details:PlatformDetails=athena_details
28+
details:PlatformDetails=athena_query_details
2929

3030

3131
@render_manager.register
3232
classAthenaQueryRender(SqlQueryRender):
33-
details:PlatformDetails=athena_details
34-
mappings:AthenaMappings=athena_mappings
33+
details:PlatformDetails=athena_query_details
34+
mappings:AthenaMappings=athena_query_mappings
3535

3636
or_token="OR"
3737

‎uncoder-core/app/translator/platforms/athena/renders/athena_cti.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@
2020
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2121
fromapp.translator.core.render_ctiimportRenderCTI
2222
fromapp.translator.managersimportrender_cti_manager
23-
fromapp.translator.platforms.athena.constimportathena_details
23+
fromapp.translator.platforms.athena.constimportathena_query_details
2424
fromapp.translator.platforms.athena.mappings.athena_ctiimportDEFAULT_ATHENA_MAPPING
2525

2626

2727
@render_cti_manager.register
2828
classAthenaCTI(RenderCTI):
29-
details:PlatformDetails=athena_details
29+
details:PlatformDetails=athena_query_details
3030

3131
field_value_template:str="{key} = '{value}'"
3232
or_operator:str=" OR "

‎uncoder-core/app/translator/platforms/base/aql/mapping.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,3 @@ def get_suitable_source_mappings(
9090
suitable_source_mappings= [self._source_mappings[DEFAULT_MAPPING_NAME]]
9191

9292
returnsuitable_source_mappings
93-
94-
95-
aql_mappings=AQLMappings(platform_dir="qradar")

‎uncoder-core/app/translator/platforms/base/aql/parsers/aql.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,12 @@
2626
fromapp.translator.platforms.base.aql.constimportNUM_VALUE_PATTERN,SINGLE_QUOTES_VALUE_PATTERN,TABLE_GROUP_PATTERN
2727
fromapp.translator.platforms.base.aql.functionsimportAQLFunctions,aql_functions
2828
fromapp.translator.platforms.base.aql.log_source_mapimportLOG_SOURCE_FUNCTIONS_MAP
29-
fromapp.translator.platforms.base.aql.mappingimportAQLMappings,aql_mappings
3029
fromapp.translator.platforms.base.aql.tokenizerimportAQLTokenizer
3130
fromapp.translator.tools.utilsimportget_match_group
3231

3332

3433
classAQLQueryParser(PlatformQueryParser):
3534
tokenizer:AQLTokenizer=AQLTokenizer(aql_functions)
36-
mappings:AQLMappings=aql_mappings
3735
platform_functions:AQLFunctions=aql_functions
3836

3937
log_source_functions= ("LOGSOURCENAME","LOGSOURCEGROUPNAME")

‎uncoder-core/app/translator/platforms/base/aql/renders/aql.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
fromapp.translator.core.custom_types.valuesimportValueType
2424
fromapp.translator.core.renderimportBaseFieldValueRender,PlatformQueryRender
2525
fromapp.translator.core.str_value_managerimportStrValue
26-
fromapp.translator.platforms.base.aql.mappingimportAQLLogSourceSignature,AQLMappings,aql_mappings
26+
fromapp.translator.platforms.base.aql.mappingimportAQLLogSourceSignature
2727
fromapp.translator.platforms.base.aql.str_value_managerimportaql_str_value_manager
2828

2929

@@ -121,8 +121,6 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
121121

122122

123123
classAQLQueryRender(PlatformQueryRender):
124-
mappings:AQLMappings=aql_mappings
125-
126124
or_token="OR"
127125
and_token="AND"
128126
not_token="NOT"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp