Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd69da7f

Browse files
committed
Merge branch 'main' into gis-improve_qradar_palo_alto_mapping
# Conflicts:#uncoder-core/app/translator/mappings/platforms/qradar/default.yml#uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml#uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml#uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml
2 parentsfc38d67 +bf008fe commitd69da7f

File tree

229 files changed

+6655
-1653
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

229 files changed

+6655
-1653
lines changed

‎uncoder-core/app/translator/const.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99

1010
CTI_IOCS_PER_QUERY_LIMIT=25
1111

12-
DEFAULT_VALUE_TYPE=Union[int,str,StrValue,list[Union[int,str,StrValue]]]
12+
DEFAULT_VALUE_TYPE=Union[bool,int,str,StrValue,list[Union[int,str,StrValue]]]
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
fromtypingimportUnion
2+
3+
fromapp.translator.core.models.query_tokens.fieldimportAlias,Field
4+
fromapp.translator.core.models.query_tokens.field_fieldimportFieldField
5+
fromapp.translator.core.models.query_tokens.field_valueimportFieldValue
6+
fromapp.translator.core.models.query_tokens.function_valueimportFunctionValue
7+
fromapp.translator.core.models.query_tokens.identifierimportIdentifier
8+
fromapp.translator.core.models.query_tokens.keywordimportKeyword
9+
10+
QUERY_TOKEN_TYPE=Union[FieldField,FieldValue,FunctionValue,Keyword,Identifier,Field,Alias]
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
fromcontextvarsimportContextVar
2+
fromtypingimportOptional
23

34
return_only_first_query_ctx_var:ContextVar[bool]=ContextVar("return_only_first_query_ctx_var",default=False)
45
"""Set to True to return only first query if rendered multiple options"""
6+
7+
wrap_query_with_meta_info_ctx_var:ContextVar[bool]=ContextVar("wrap_query_with_meta_info_ctx_var",default=True)
8+
"""Set to False not to wrap query with meta info commentary"""
9+
10+
preset_log_source_str_ctx_var:ContextVar[Optional[str]]=ContextVar("preset_log_source_str_ctx_var",default=None)

‎uncoder-core/app/translator/core/custom_types/functions.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,21 @@ class FunctionType(CustomEnum):
1515
latest="latest"
1616

1717
divide="divide"
18+
multiply="multiply"
1819

1920
lower="lower"
2021
split="split"
2122
upper="upper"
2223

2324
array_length="array_length"
24-
compare="compare"
2525
extract_time="extract_time"
2626
ipv4_is_in_range="ipv4_is_in_range"
2727

2828
bin="bin"
2929
eval="eval"
3030
fields="fields"
31+
iploc="iploc"
32+
join="join"
3133
rename="rename"
3234
search="search"
3335
sort_limit="sort_limit"
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
fromapp.translator.tools.custom_enumimportCustomEnum
2+
3+
4+
classIPLocationType(CustomEnum):
5+
asn="ip_loc_asn"
6+
asn_org="ip_loc_asn_org"
7+
city="ip_loc_city"
8+
continent="ip_loc_continent"
9+
country="ip_loc_country"
10+
lat_lon="ip_loc_lat_lon"
11+
region="ip_loc_region"
12+
timezone="ip_loc_timezone"
13+
14+
15+
classTimeType(CustomEnum):
16+
timestamp="timestamp"

‎uncoder-core/app/translator/core/exceptions/core.py‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,25 @@
11
fromtypingimportOptional
22

33

4-
classNotImplementedException(BaseException):
5-
...
6-
7-
84
classBasePlatformException(BaseException):
95
...
106

117

128
classStrictPlatformException(BasePlatformException):
13-
field_name:str=None
14-
15-
def__init__(
16-
self,platform_name:str,field_name:str,mapping:Optional[str]=None,detected_fields:Optional[list]=None
17-
):
9+
def__init__(self,platform_name:str,fields:list[str],mapping:Optional[str]=None):
1810
message= (
1911
f"Platform{platform_name} has strict mapping. "
20-
f"Source fields:{', '.join(detected_fields)ifdetected_fieldselsefield_name} has no mapping."
12+
f"Source fields:{', '.join(fields)} have no mapping."
2113
f" Mapping file:{mapping}."
2214
ifmapping
2315
else""
2416
)
25-
self.field_name=field_name
17+
super().__init__(message)
18+
19+
20+
classUnsupportedMappingsException(BasePlatformException):
21+
def__init__(self,platform_name:str,mappings:list[str]):
22+
message=f"Platform{platform_name} does not support these mappings:{mappings}."
2623
super().__init__(message)
2724

2825

@@ -93,5 +90,9 @@ class InvalidJSONStructure(InvalidRuleStructure):
9390
rule_type:str="JSON"
9491

9592

93+
classInvalidTOMLStructure(InvalidRuleStructure):
94+
rule_type:str="TOML"
95+
96+
9697
classInvalidXMLStructure(InvalidRuleStructure):
9798
rule_type:str="XML"

‎uncoder-core/app/translator/core/exceptions/render.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,5 @@ class FunctionRenderException(BaseRenderException):
1414

1515
classUnsupportedRenderMethod(BaseRenderException):
1616
def__init__(self,platform_name:str,method:str):
17-
message=f"Cannot translate.{platform_name} backend does not support{method}."
17+
message=f'Cannot translate.{platform_name} backend does not support"{method}".'
1818
super().__init__(message)

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525

2626
fromapp.translator.core.exceptions.functionsimportNotSupportedFunctionException
2727
fromapp.translator.core.mappingimportSourceMapping
28-
fromapp.translator.core.models.fieldimportAlias,Field
2928
fromapp.translator.core.models.functions.baseimportFunction,ParsedFunctions,RenderedFunctions
29+
fromapp.translator.core.models.query_tokens.fieldimportAlias,Field,PredefinedField
3030
fromapp.translator.tools.utilsimportexecute_module
3131
fromsettingsimportINIT_FUNCTIONS
3232

@@ -83,7 +83,6 @@ def parse(self, func_body: str, raw: str) -> Function:
8383
classFunctionRender(ABC):
8484
function_names_map:ClassVar[dict[str,str]]= {}
8585
order_to_render:int=0
86-
in_query_render:bool=False
8786
render_to_prefix:bool=False
8887
manager:PlatformFunctionsManager=None
8988

@@ -95,17 +94,19 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe
9594
defrender(self,function:Function,source_mapping:SourceMapping)->str:
9695
raiseNotImplementedError
9796

98-
@staticmethod
99-
defmap_field(field:Union[Alias,Field],source_mapping:SourceMapping)->str:
97+
defmap_field(self,field:Union[Alias,Field],source_mapping:SourceMapping)->str:
10098
ifisinstance(field,Alias):
10199
returnfield.name
102100

103-
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
104-
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
105-
ifisinstance(mapped_field,list):
106-
mapped_field=mapped_field[0]
101+
ifisinstance(field,Field):
102+
mappings=self.manager.platform_functions.platform_query_render.mappings
103+
mapped_fields=mappings.map_field(field,source_mapping)
104+
returnmapped_fields[0]
107105

108-
returnmapped_fieldifmapped_fieldelsefield.source_name
106+
ifisinstance(field,PredefinedField):
107+
returnself.manager.platform_functions.platform_query_render.map_predefined_field(field)
108+
109+
raiseNotSupportedFunctionException
109110

110111

111112
classPlatformFunctionsManager:
@@ -117,7 +118,6 @@ def __init__(self):
117118
self._parsers_map:dict[str,FunctionParser]= {}# {platform_func_name: FunctionParser}
118119

119120
self._renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
120-
self._in_query_renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
121121
self._order_to_render:dict[str,int]= {}# {generic_func_name: int}
122122

123123
defregister_render(self,render_class:type[FunctionRender])->type[FunctionRender]:
@@ -126,8 +126,6 @@ def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRe
126126
forgeneric_function_nameinrender.function_names_map:
127127
self._renders_map[generic_function_name]=render
128128
self._order_to_render[generic_function_name]=render.order_to_render
129-
ifrender.in_query_render:
130-
self._in_query_renders_map[generic_function_name]=render
131129

132130
returnrender_class
133131

@@ -149,24 +147,16 @@ def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
149147

150148
raiseNotSupportedFunctionException
151149

152-
defget_parser(self,platform_func_name:str)->FunctionParser:
150+
defget_parser(self,platform_func_name:str)->Optional[FunctionParser]:
153151
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(platform_func_name)):
154152
returnparser
155153

156-
raiseNotSupportedFunctionException
157-
158154
defget_render(self,generic_func_name:str)->FunctionRender:
159155
ifINIT_FUNCTIONSand (render:=self._renders_map.get(generic_func_name)):
160156
returnrender
161157

162158
raiseNotSupportedFunctionException
163159

164-
defget_in_query_render(self,generic_func_name:str)->FunctionRender:
165-
ifINIT_FUNCTIONSand (render:=self._in_query_renders_map.get(generic_func_name)):
166-
returnrender
167-
168-
raiseNotSupportedFunctionException
169-
170160
@property
171161
deforder_to_render(self)->dict[str,int]:
172162
ifINIT_FUNCTIONS:

‎uncoder-core/app/translator/core/mapping.py‎

Lines changed: 88 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from __future__importannotations
22

33
fromabcimportABC,abstractmethod
4-
fromtypingimportOptional,TypeVar
4+
fromtypingimportTYPE_CHECKING,Optional,TypeVar,Union
55

6+
fromapp.translator.core.exceptions.coreimportStrictPlatformException,UnsupportedMappingsException
7+
fromapp.translator.core.models.platform_detailsimportPlatformDetails
68
fromapp.translator.mappings.utils.load_from_filesimportLoaderFileMappings
79

10+
ifTYPE_CHECKING:
11+
fromapp.translator.core.models.query_tokens.fieldimportField
12+
13+
814
DEFAULT_MAPPING_NAME="default"
915

1016

@@ -13,9 +19,14 @@ class LogSourceSignature(ABC):
1319
wildcard_symbol="*"
1420

1521
@abstractmethod
16-
defis_suitable(self,*args,**kwargs)->bool:
22+
defis_suitable(self,**kwargs)->bool:
1723
raiseNotImplementedError("Abstract method")
1824

25+
@staticmethod
26+
def_check_conditions(conditions:list[Union[bool,None]])->bool:
27+
conditions= [conditionforconditioninconditionsifconditionisnotNone]
28+
returnbool(conditions)andall(conditions)
29+
1930
@abstractmethod
2031
def__str__(self)->str:
2132
raiseNotImplementedError("Abstract method")
@@ -64,7 +75,7 @@ def update(self, fields_mapping: FieldsMapping) -> None:
6475
self.__render_mapping.update(fields_mapping.__render_mapping)
6576

6677
defis_suitable(self,field_names:list[str])->bool:
67-
returnset(field_names).issubset(set(self.__parser_mapping.keys()))
78+
returnbool(field_names)andset(field_names).issubset(set(self.__parser_mapping.keys()))
6879

6980

7081
_LogSourceSignatureType=TypeVar("_LogSourceSignatureType",bound=LogSourceSignature)
@@ -85,12 +96,16 @@ def __init__(
8596

8697

8798
classBasePlatformMappings:
99+
details:PlatformDetails=None
100+
101+
is_strict_mapping:bool=False
88102
skip_load_default_mappings:bool=True
89103
extend_default_mapping_with_all_fields:bool=False
90104

91-
def__init__(self,platform_dir:str):
105+
def__init__(self,platform_dir:str,platform_details:PlatformDetails):
92106
self._loader=LoaderFileMappings()
93107
self._platform_dir=platform_dir
108+
self.details=platform_details
94109
self._source_mappings=self.prepare_mapping()
95110

96111
defupdate_default_source_mapping(self,default_mapping:SourceMapping,fields_mapping:FieldsMapping)->None:
@@ -137,9 +152,34 @@ def prepare_fields_mapping(field_mapping: dict) -> FieldsMapping:
137152
defprepare_log_source_signature(self,mapping:dict)->LogSourceSignature:
138153
raiseNotImplementedError("Abstract method")
139154

140-
@abstractmethod
141-
defget_suitable_source_mappings(self,*args,**kwargs)->list[SourceMapping]:
142-
raiseNotImplementedError("Abstract method")
155+
defget_source_mappings_by_fields_and_log_sources(
156+
self,field_names:list[str],log_sources:dict[str,list[Union[int,str]]]
157+
)->list[SourceMapping]:
158+
by_log_sources_and_fields= []
159+
by_fields= []
160+
forsource_mappinginself._source_mappings.values():
161+
ifsource_mapping.source_id==DEFAULT_MAPPING_NAME:
162+
continue
163+
164+
ifsource_mapping.fields_mapping.is_suitable(field_names):
165+
by_fields.append(source_mapping)
166+
167+
log_source_signature:LogSourceSignature=source_mapping.log_source_signature
168+
iflog_source_signatureandlog_source_signature.is_suitable(**log_sources):
169+
by_log_sources_and_fields.append(source_mapping)
170+
171+
returnby_log_sources_and_fieldsorby_fieldsor [self._source_mappings[DEFAULT_MAPPING_NAME]]
172+
173+
defget_source_mappings_by_ids(self,source_mapping_ids:list[str])->list[SourceMapping]:
174+
source_mappings= []
175+
forsource_mapping_idinsource_mapping_ids:
176+
ifsource_mapping:=self.get_source_mapping(source_mapping_id):
177+
source_mappings.append(source_mapping)
178+
179+
ifnotsource_mappings:
180+
source_mappings= [self.get_source_mapping(DEFAULT_MAPPING_NAME)]
181+
182+
returnsource_mappings
143183

144184
defget_source_mapping(self,source_id:str)->Optional[SourceMapping]:
145185
returnself._source_mappings.get(source_id)
@@ -148,6 +188,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
148188
defdefault_mapping(self)->SourceMapping:
149189
returnself._source_mappings[DEFAULT_MAPPING_NAME]
150190

191+
defcheck_fields_mapping_existence(self,field_tokens:list[Field],source_mapping:SourceMapping)->list[str]:
192+
unmapped= []
193+
forfieldinfield_tokens:
194+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
195+
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
196+
ifnotmapped_fieldandfield.source_namenotinunmapped:
197+
unmapped.append(field.source_name)
198+
199+
ifself.is_strict_mappingandunmapped:
200+
raiseStrictPlatformException(
201+
platform_name=self.details.name,fields=unmapped,mapping=source_mapping.source_id
202+
)
203+
204+
returnunmapped
205+
206+
@staticmethod
207+
defmap_field(field:Field,source_mapping:SourceMapping)->list[str]:
208+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
209+
# field can be mapped to corresponding platform field name or list of platform field names
210+
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
211+
212+
ifisinstance(mapped_field,str):
213+
mapped_field= [mapped_field]
214+
215+
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
216+
151217

152218
classBaseCommonPlatformMappings(ABC,BasePlatformMappings):
153219
defprepare_mapping(self)->dict[str,SourceMapping]:
@@ -163,3 +229,18 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
163229
)
164230

165231
returnsource_mappings
232+
233+
234+
classBaseStrictLogSourcesPlatformMappings(ABC,BasePlatformMappings):
235+
defget_source_mappings_by_ids(self,source_mapping_ids:list[str])->list[SourceMapping]:
236+
source_mappings= []
237+
forsource_mapping_idinsource_mapping_ids:
238+
ifsource_mapping_id==DEFAULT_MAPPING_NAME:
239+
continue
240+
ifsource_mapping:=self.get_source_mapping(source_mapping_id):
241+
source_mappings.append(source_mapping)
242+
243+
ifnotsource_mappings:
244+
raiseUnsupportedMappingsException(platform_name=self.details.name,mappings=source_mapping_ids)
245+
246+
returnsource_mappings

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp