Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd3dba4e

Browse files
authored
Merge pull request#112 from UncoderIO/gis-7814
Improve AQL mapping method is_suitable
2 parents2b3836c +d4614b8 commitd3dba4e

File tree

3 files changed

+224
-18
lines changed

3 files changed

+224
-18
lines changed

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VAL
126126

127127
classQueryRender(ABC):
128128
comment_symbol:str=None
129+
details:PlatformDetails=None
129130
is_single_line_comment:bool=False
130131
unsupported_functions_text="Unsupported functions were excluded from the result query:"
131132

@@ -146,7 +147,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
146147

147148
classPlatformQueryRender(QueryRender):
148149
mappings:BasePlatformMappings=None
149-
details:PlatformDetails=None
150150
is_strict_mapping:bool=False
151151

152152
or_token="or"
@@ -295,28 +295,35 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
295295

296296
def_generate_from_tokenized_query_container(self,query_container:TokenizedQueryContainer)->str:
297297
queries_map= {}
298+
errors= []
298299
source_mappings=self._get_source_mappings(query_container.meta_info.source_mapping_ids)
299300

300301
forsource_mappinginsource_mappings:
301302
prefix=self.generate_prefix(source_mapping.log_source_signature)
302-
ifsource_mapping.raw_log_fields:
303-
defined_raw_log_fields=self.generate_raw_log_fields(
304-
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
303+
try:
304+
ifsource_mapping.raw_log_fields:
305+
defined_raw_log_fields=self.generate_raw_log_fields(
306+
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
307+
)
308+
prefix+=f"\n{defined_raw_log_fields}\n"
309+
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
310+
exceptStrictPlatformExceptionaserr:
311+
errors.append(err)
312+
continue
313+
else:
314+
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
315+
not_supported_functions=query_container.functions.not_supported+rendered_functions.not_supported
316+
finalized_query=self.finalize_query(
317+
prefix=prefix,
318+
query=result,
319+
functions=rendered_functions.rendered,
320+
not_supported_functions=not_supported_functions,
321+
meta_info=query_container.meta_info,
322+
source_mapping=source_mapping,
305323
)
306-
prefix+=f"\n{defined_raw_log_fields}\n"
307-
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
308-
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
309-
not_supported_functions=query_container.functions.not_supported+rendered_functions.not_supported
310-
finalized_query=self.finalize_query(
311-
prefix=prefix,
312-
query=result,
313-
functions=rendered_functions.rendered,
314-
not_supported_functions=not_supported_functions,
315-
meta_info=query_container.meta_info,
316-
source_mapping=source_mapping,
317-
)
318-
queries_map[source_mapping.source_id]=finalized_query
319-
324+
queries_map[source_mapping.source_id]=finalized_query
325+
ifnotqueries_mapanderrors:
326+
raiseerrors[0]
320327
returnself.finalize(queries_map)
321328

322329
defgenerate(self,query_container:Union[RawQueryContainer,TokenizedQueryContainer])->str:
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
fromtypingimportOptional
2+
3+
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
4+
5+
6+
classAQLLogSourceSignature(LogSourceSignature):
7+
def__init__(
8+
self,
9+
device_types:Optional[list[int]],
10+
categories:Optional[list[int]],
11+
qids:Optional[list[int]],
12+
qid_event_categories:Optional[list[int]],
13+
default_source:dict,
14+
):
15+
self.device_types=set(device_typesor [])
16+
self.categories=set(categoriesor [])
17+
self.qids=set(qidsor [])
18+
self.qid_event_categories=set(qid_event_categoriesor [])
19+
self._default_source=default_sourceor {}
20+
21+
defis_suitable(
22+
self,
23+
devicetype:Optional[list[int]],
24+
category:Optional[list[int]],
25+
qid:Optional[list[int]],
26+
qideventcategory:Optional[list[int]],
27+
)->bool:
28+
device_type_match=set(devicetype).issubset(self.device_types)ifdevicetypeelseNone
29+
category_match=set(category).issubset(self.categories)ifcategoryelseNone
30+
qid_match=set(qid).issubset(self.qids)ifqidelseNone
31+
qid_event_category_match=set(qideventcategory).issubset(self.qid_event_categories)ifqideventcategoryelseNone
32+
returnall(
33+
conditionforconditionin (
34+
device_type_match,category_match,
35+
qid_match,qid_event_category_match)
36+
ifconditionisnotNone
37+
)
38+
39+
def__str__(self)->str:
40+
returnself._default_source.get("table","events")
41+
42+
@property
43+
defextra_condition(self)->str:
44+
default_source=self._default_source
45+
return" AND ".join((f"{key}={value}"forkey,valueindefault_source.items()ifkey!="table"andvalue))
46+
47+
48+
classAQLMappings(BasePlatformMappings):
49+
defprepare_log_source_signature(self,mapping:dict)->AQLLogSourceSignature:
50+
log_source=mapping.get("log_source", {})
51+
default_log_source=mapping["default_log_source"]
52+
returnAQLLogSourceSignature(
53+
device_types=log_source.get("devicetype"),
54+
categories=log_source.get("category"),
55+
qids=log_source.get("qid"),
56+
qid_event_categories=log_source.get("qideventcategory"),
57+
default_source=default_log_source,
58+
)
59+
60+
defget_suitable_source_mappings(
61+
self,
62+
field_names:list[str],
63+
devicetype:Optional[list[int]]=None,
64+
category:Optional[list[int]]=None,
65+
qid:Optional[list[int]]=None,
66+
qideventcategory:Optional[list[int]]=None,
67+
)->list[SourceMapping]:
68+
suitable_source_mappings= []
69+
forsource_mappinginself._source_mappings.values():
70+
ifsource_mapping.source_id==DEFAULT_MAPPING_NAME:
71+
continue
72+
73+
log_source_signature:AQLLogSourceSignature=source_mapping.log_source_signature
74+
iflog_source_signature.is_suitable(devicetype,category,qid,qideventcategory):
75+
ifsource_mapping.fields_mapping.is_suitable(field_names):
76+
suitable_source_mappings.append(source_mapping)
77+
elifsource_mapping.fields_mapping.is_suitable(field_names):
78+
suitable_source_mappings.append(source_mapping)
79+
80+
ifnotsuitable_source_mappings:
81+
suitable_source_mappings= [self._source_mappings[DEFAULT_MAPPING_NAME]]
82+
83+
returnsuitable_source_mappings
84+
85+
86+
aql_mappings=AQLMappings(platform_dir="qradar")
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
"""
2+
Uncoder IO Commercial Edition License
3+
-----------------------------------------------------------------
4+
Copyright (c) 2024 SOC Prime, Inc.
5+
6+
This file is part of the Uncoder IO Commercial Edition ("CE") and is
7+
licensed under the Uncoder IO Non-Commercial License (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
https://github.com/UncoderIO/UncoderIO/blob/main/LICENSE
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
-----------------------------------------------------------------
17+
"""
18+
19+
importre
20+
fromtypingimportUnion
21+
22+
fromapp.translator.core.models.query_containerimportRawQueryContainer,TokenizedQueryContainer
23+
fromapp.translator.core.parserimportPlatformQueryParser
24+
fromapp.translator.platforms.base.aql.constimportNUM_VALUE_PATTERN,SINGLE_QUOTES_VALUE_PATTERN
25+
fromapp.translator.platforms.base.aql.mappingimportAQLMappings,aql_mappings
26+
fromapp.translator.platforms.base.aql.tokenizerimportAQLTokenizer
27+
fromapp.translator.tools.utilsimportget_match_group
28+
29+
30+
classAQLQueryParser(PlatformQueryParser):
31+
tokenizer=AQLTokenizer()
32+
mappings:AQLMappings=aql_mappings
33+
34+
log_source_functions= ("LOGSOURCENAME","LOGSOURCEGROUPNAME","LOGSOURCETYPENAME","CATEGORYNAME")
35+
log_source_function_pattern=r"\(?(?P<key>___func_name___\([a-zA-Z]+\))(?:\s+like\s+|\s+ilike\s+|\s*=\s*)'(?P<value>[%a-zA-Z\s]+)'\s*\)?\s+(?:and|or)?\s"# noqa: E501
36+
37+
log_source_key_types= ("devicetype","category","qid","qideventcategory")
38+
log_source_pattern=rf"___source_type___(?:\s+like\s+|\s+ilike\s+|\s*=\s*)(?:{SINGLE_QUOTES_VALUE_PATTERN}|{NUM_VALUE_PATTERN})(?:\s+(?:and|or)\s+|\s+)?"# noqa: E501
39+
num_value_pattern=r"[0-9]+"
40+
multi_num_log_source_pattern= (
41+
rf"___source_type___\s+in\s+\((?P<value>(?:{num_value_pattern}(?:\s*,\s*)?)+)\)(?:\s+(?:and|or)\s+|\s+)?"
42+
)
43+
str_value_pattern=r"""(?:')(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')+)(?:')"""
44+
multi_str_log_source_pattern= (
45+
rf"""___source_type___\s+in\s+\((?P<value>(?:{str_value_pattern}(?:\s*,\s*)?)+)\)(?:\s+(?:and|or)\s+|\s+)?"""
46+
)
47+
48+
table_pattern=r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"
49+
50+
def__clean_query(self,query:str)->str:
51+
forfunc_nameinself.log_source_functions:
52+
pattern=self.log_source_function_pattern.replace("___func_name___",func_name)
53+
whilesearch:=re.search(pattern,query,flags=re.IGNORECASE):
54+
pos_start=search.start()
55+
pos_end=search.end()
56+
query=query[:pos_start]+query[pos_end:]
57+
58+
returnquery
59+
60+
@staticmethod
61+
def__parse_multi_value_log_source(
62+
match:re.Match,query:str,pattern:str
63+
)->tuple[str,Union[list[str],list[int]]]:
64+
value=match.group("value")
65+
pos_start=match.start()
66+
pos_end=match.end()
67+
query=query[:pos_start]+query[pos_end:]
68+
returnquery,re.findall(pattern,value)
69+
70+
def__parse_log_sources(self,query:str)->tuple[dict[str,Union[list[str],list[int]]],str]:
71+
log_sources= {}
72+
73+
ifsearch:=re.search(self.table_pattern,query,flags=re.IGNORECASE):
74+
pos_end=search.end()
75+
query=query[pos_end:]
76+
77+
forlog_source_keyinself.log_source_key_types:
78+
pattern=self.log_source_pattern.replace("___source_type___",log_source_key)
79+
whilesearch:=re.search(pattern,query,flags=re.IGNORECASE):
80+
num_value=get_match_group(search,group_name="num_value")
81+
str_value=get_match_group(search,group_name="s_q_value")
82+
value=num_valueandint(num_value)orstr_value
83+
log_sources.setdefault(log_source_key, []).append(value)
84+
pos_start=search.start()
85+
pos_end=search.end()
86+
query=query[:pos_start]+query[pos_end:]
87+
88+
pattern=self.multi_num_log_source_pattern.replace("___source_type___",log_source_key)
89+
ifsearch:=re.search(pattern,query,flags=re.IGNORECASE):
90+
query,values=self.__parse_multi_value_log_source(search,query,self.num_value_pattern)
91+
values= [int(v)forvinvalues]
92+
log_sources.setdefault(log_source_key, []).extend(values)
93+
94+
pattern=self.multi_str_log_source_pattern.replace("___source_type___",log_source_key)
95+
ifsearch:=re.search(pattern,query,flags=re.IGNORECASE):
96+
query,values=self.__parse_multi_value_log_source(search,query,self.str_value_pattern)
97+
log_sources.setdefault(log_source_key, []).extend(values)
98+
99+
returnlog_sources,query
100+
101+
def_parse_query(self,text:str)->tuple[str,dict[str,Union[list[str],list[int]]]]:
102+
query=self.__clean_query(text)
103+
log_sources,query=self.__parse_log_sources(query)
104+
returnquery,log_sources
105+
106+
defparse(self,raw_query_container:RawQueryContainer)->TokenizedQueryContainer:
107+
query,log_sources=self._parse_query(raw_query_container.query)
108+
tokens,source_mappings=self.get_tokens_and_source_mappings(query,log_sources)
109+
fields_tokens=self.get_fields_tokens(tokens=tokens)
110+
meta_info=raw_query_container.meta_info
111+
meta_info.query_fields=fields_tokens
112+
meta_info.source_mapping_ids= [source_mapping.source_idforsource_mappinginsource_mappings]
113+
returnTokenizedQueryContainer(tokens=tokens,meta_info=meta_info)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp