Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit4b54f66

Browse files
authored
Merge pull request#166 from UncoderIO/gis-8036
predefined field processing
2 parents50a0bbf +a4ab1a3 commit4b54f66

File tree

17 files changed

+124
-39
lines changed

17 files changed

+124
-39
lines changed

‎uncoder-core/app/translator/core/custom_types/functions.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ class FunctionType(CustomEnum):
1515
latest="latest"
1616

1717
divide="divide"
18+
multiply="multiply"
1819

1920
lower="lower"
2021
split="split"
@@ -28,6 +29,7 @@ class FunctionType(CustomEnum):
2829
bin="bin"
2930
eval="eval"
3031
fields="fields"
32+
iploc="iploc"
3133
join="join"
3234
rename="rename"
3335
search="search"
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
fromapp.translator.tools.custom_enumimportCustomEnum
2+
3+
4+
classIPLocationType(CustomEnum):
5+
asn="ip_loc_asn"
6+
asn_org="ip_loc_asn_org"
7+
city="ip_loc_city"
8+
continent="ip_loc_continent"
9+
country="ip_loc_country"
10+
lat_lon="ip_loc_lat_lon"
11+
region="ip_loc_region"
12+
timezone="ip_loc_timezone"

‎uncoder-core/app/translator/core/models/field.py‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
3737
self.__generic_names_map=generic_names_map
3838

3939

40+
classPredefinedField:
41+
def__init__(self,name:str):
42+
self.name=name
43+
44+
4045
classFieldField:
4146
def__init__(
4247
self,
@@ -46,10 +51,10 @@ def __init__(
4651
is_alias_left:bool=False,
4752
is_alias_right:bool=False,
4853
):
49-
self.field_left=Field(source_name=source_name_left)
54+
self.field_left=Field(source_name=source_name_left)ifnotis_alias_leftelseNone
5055
self.alias_left=Alias(name=source_name_left)ifis_alias_leftelseNone
5156
self.operator=operator
52-
self.field_right=Field(source_name=source_name_right)
57+
self.field_right=Field(source_name=source_name_right)ifnotis_alias_rightelseNone
5358
self.alias_right=Alias(name=source_name_right)ifis_alias_rightelseNone
5459

5560

@@ -60,11 +65,14 @@ def __init__(
6065
operator:Identifier,
6166
value:Union[int,str,StrValue,list,tuple],
6267
is_alias:bool=False,
68+
is_predefined_field:bool=False,
6369
):
64-
self.field=Field(source_name=source_name)
65-
self.alias=None
66-
ifis_alias:
67-
self.alias=Alias(name=source_name)
70+
# mapped by platform fields mapping
71+
self.field=Field(source_name=source_name)ifnot (is_aliasoris_predefined_field)elseNone
72+
# not mapped
73+
self.alias=Alias(name=source_name)ifis_aliaselseNone
74+
# mapped by platform predefined fields mapping
75+
self.predefined_field=PredefinedField(name=source_name)ifis_predefined_fieldelseNone
6876

6977
self.operator=operator
7078
self.values= []
@@ -96,10 +104,13 @@ def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -
96104
self.values.append(value)
97105

98106
def__repr__(self):
99-
ifself.field:
100-
returnf"{self.field.source_name}{self.operator.token_type}{self.values}"
107+
ifself.alias:
108+
returnf"{self.alias.name}{self.operator.token_type}{self.values}"
109+
110+
ifself.predefined_field:
111+
returnf"{self.predefined_field.name}{self.operator.token_type}{self.values}"
101112

102-
returnf"{self.alias.name}{self.operator.token_type}{self.values}"
113+
returnf"{self.field.source_name}{self.operator.token_type}{self.values}"
103114

104115

105116
classKeyword:

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
fromapp.translator.core.exceptions.parserimportUnsupportedOperatorException
3232
fromapp.translator.core.functionsimportPlatformFunctions
3333
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
34-
fromapp.translator.core.models.fieldimportField,FieldField,FieldValue,Keyword
34+
fromapp.translator.core.models.fieldimportField,FieldField,FieldValue,Keyword,PredefinedField
3535
fromapp.translator.core.models.functions.baseimportFunction,RenderedFunctions
3636
fromapp.translator.core.models.identifierimportIdentifier
3737
fromapp.translator.core.models.platform_detailsimportPlatformDetails
@@ -218,7 +218,8 @@ class PlatformQueryRender(QueryRender):
218218
field_field_render=BaseFieldFieldRender()
219219
field_value_render=BaseFieldValueRender(or_token=or_token)
220220

221-
raw_log_field_pattern_map:ClassVar[dict[str,str]]=None
221+
predefined_fields_map:ClassVar[dict[str,str]]= {}
222+
raw_log_field_patterns_map:ClassVar[dict[str,str]]= {}
222223

223224
def__init__(self):
224225
super().__init__()
@@ -248,9 +249,23 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
248249

249250
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
250251

252+
defmap_predefined_field(self,predefined_field:PredefinedField)->str:
253+
ifnot (mapped_predefined_field_name:=self.predefined_fields_map.get(predefined_field.name)):
254+
ifself.is_strict_mapping:
255+
raiseStrictPlatformException(field_name=predefined_field.name,platform_name=self.details.name)
256+
257+
returnpredefined_field.name
258+
259+
returnmapped_predefined_field_name
260+
251261
defapply_token(self,token:Union[FieldValue,Keyword,Identifier],source_mapping:SourceMapping)->str:
252262
ifisinstance(token,FieldValue):
253-
mapped_fields= [token.alias.name]iftoken.aliaselseself.map_field(token.field,source_mapping)
263+
iftoken.alias:
264+
mapped_fields= [token.alias.name]
265+
eliftoken.predefined_field:
266+
mapped_fields= [self.map_predefined_field(token.predefined_field)]
267+
else:
268+
mapped_fields=self.map_field(token.field,source_mapping)
254269
joined=self.logical_operators_map[LogicalOperatorType.OR].join(
255270
[
256271
self.field_value_render.apply_field_value(field=field,operator=token.operator,value=token.value)
@@ -365,7 +380,7 @@ def generate_from_raw_query_container(self, query_container: RawQueryContainer)
365380
)
366381

367382
defprocess_raw_log_field(self,field:str,field_type:str)->Optional[str]:
368-
ifraw_log_field_pattern:=self.raw_log_field_pattern_map.get(field_type):
383+
ifraw_log_field_pattern:=self.raw_log_field_patterns_map.get(field_type):
369384
returnraw_log_field_pattern.format(field=field)
370385

371386
defprocess_raw_log_field_prefix(self,field:str,source_mapping:SourceMapping)->Optional[list]:
@@ -379,7 +394,7 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping
379394
return [self.process_raw_log_field(field=field,field_type=raw_log_field_type)]
380395

381396
defgenerate_raw_log_fields(self,fields:list[Field],source_mapping:SourceMapping)->str:
382-
ifself.raw_log_field_pattern_mapisNone:
397+
ifnotself.raw_log_field_patterns_map:
383398
return""
384399
defined_raw_log_fields= []
385400
forfieldinfields:

‎uncoder-core/app/translator/core/tokenizer.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -332,12 +332,12 @@ def get_field_tokens_from_func_args( # noqa: PLR0912
332332
ifisinstance(arg,Field):
333333
result.append(arg)
334334
elifisinstance(arg,FieldField):
335-
ifnotarg.alias_leftorarg.alias_left.name!=arg.field_left.source_name:
335+
ifarg.field_left:
336336
result.append(arg.field_left)
337-
ifnotarg.alias_rightorarg.alias_right.name!=arg.field_right.source_name:
337+
ifarg.field_right:
338338
result.append(arg.field_right)
339339
elifisinstance(arg,FieldValue):
340-
ifnotarg.aliasorarg.alias.name!=arg.field.source_name:
340+
ifarg.field:
341341
result.append(arg.field)
342342
elifisinstance(arg,GroupByFunction):
343343
result.extend(self.get_field_tokens_from_func_args(args=arg.args))

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ field_mapping:
1111
dns_query_name:xdm.network.dns.dns_question.name
1212
QueryName:xdm.network.dns.dns_question.name
1313
query:xdm.network.dns.dns_question.name
14-
dns-record-type:xdm.network.dns.dns_question.type
14+
dns-record-type:xdm.network.dns.dns_question.type

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/linux_process_creation.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ field_mapping:
2626
ParentProduct:actor_process_signature_product
2727
ParentCompany:actor_process_signature_vendor
2828
md5:action_process_image_md5
29-
sha256:action_process_image_sha256
29+
sha256:action_process_image_sha256
30+
EventID:action_evtlog_event_id

‎uncoder-core/app/translator/mappings/platforms/qradar/default.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ field_mapping:
5959
-dst-packets
6060
src-bytes:src-bytes
6161
dst-bytes:dst-bytes
62-
ExternalSeverity:External Severity
62+
ExternalSeverity:
63+
-External Severity
64+
-Observeit Severity
6365
SourceMAC:
6466
-SourceMAC
6567
-MAC
@@ -73,6 +75,6 @@ field_mapping:
7375
SourceUserName:SourceUserName
7476
url_category:XForceCategoryByURL
7577
EventSeverity:EventSeverity
76-
Source:
78+
Source:
7779
-Source
78-
-source
80+
-source

‎uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@ default_log_source:
1111
field_mapping:
1212
src-ip:
1313
-sourceip
14+
-sourceIP
15+
-SourceIP
1416
-SrcHost
1517
-LocalHost
1618
-Source
1719
-NetworkView
20+
-HostName
1821
src-port:
1922
-sourceport
2023
-SrcPort

‎uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@ default_log_source:
1111
category:8110
1212

1313
field_mapping:
14-
CommandLine:Command
14+
CommandLine:
15+
-Command
16+
-ASACommand
1517
Image:Process Path
1618
ParentCommandLine:Parent Command
1719
ParentImage:Parent Process Path
1820
User:username
19-
LogonId:Logon ID
21+
LogonId:Logon ID
22+
EventID:ASASyslogCode

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp