Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit5481f1e

Browse files
committed
Improve AQL mapping method is_suitable
1 parent2b3836c commit5481f1e

File tree

3 files changed

+220
-17
lines changed

3 files changed

+220
-17
lines changed

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VAL
126126

127127
classQueryRender(ABC):
128128
comment_symbol:str=None
129+
details:PlatformDetails=None
129130
is_single_line_comment:bool=False
130131
unsupported_functions_text="Unsupported functions were excluded from the result query:"
131132

@@ -146,7 +147,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
146147

147148
classPlatformQueryRender(QueryRender):
148149
mappings:BasePlatformMappings=None
149-
details:PlatformDetails=None
150150
is_strict_mapping:bool=False
151151

152152
or_token="or"
@@ -299,23 +299,27 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
299299

300300
forsource_mappinginsource_mappings:
301301
prefix=self.generate_prefix(source_mapping.log_source_signature)
302-
ifsource_mapping.raw_log_fields:
303-
defined_raw_log_fields=self.generate_raw_log_fields(
304-
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
302+
try:
303+
ifsource_mapping.raw_log_fields:
304+
defined_raw_log_fields=self.generate_raw_log_fields(
305+
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
306+
)
307+
prefix+=f"\n{defined_raw_log_fields}\n"
308+
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
309+
exceptStrictPlatformException:
310+
continue
311+
else:
312+
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
313+
not_supported_functions=query_container.functions.not_supported+rendered_functions.not_supported
314+
finalized_query=self.finalize_query(
315+
prefix=prefix,
316+
query=result,
317+
functions=rendered_functions.rendered,
318+
not_supported_functions=not_supported_functions,
319+
meta_info=query_container.meta_info,
320+
source_mapping=source_mapping,
305321
)
306-
prefix+=f"\n{defined_raw_log_fields}\n"
307-
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
308-
rendered_functions=self.generate_functions(query_container.functions.functions,source_mapping)
309-
not_supported_functions=query_container.functions.not_supported+rendered_functions.not_supported
310-
finalized_query=self.finalize_query(
311-
prefix=prefix,
312-
query=result,
313-
functions=rendered_functions.rendered,
314-
not_supported_functions=not_supported_functions,
315-
meta_info=query_container.meta_info,
316-
source_mapping=source_mapping,
317-
)
318-
queries_map[source_mapping.source_id]=finalized_query
322+
queries_map[source_mapping.source_id]=finalized_query
319323

320324
returnself.finalize(queries_map)
321325

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
fromtypingimportOptional
2+
3+
fromapp.translator.core.mappingimportDEFAULT_MAPPING_NAME,BasePlatformMappings,LogSourceSignature,SourceMapping
4+
5+
6+
classAQLLogSourceSignature(LogSourceSignature):
7+
def__init__(
8+
self,
9+
device_types:Optional[list[int]],
10+
categories:Optional[list[int]],
11+
qids:Optional[list[int]],
12+
qid_event_categories:Optional[list[int]],
13+
default_source:dict,
14+
):
15+
self.device_types=set(device_typesor [])
16+
self.categories=set(categoriesor [])
17+
self.qids=set(qidsor [])
18+
self.qid_event_categories=set(qid_event_categoriesor [])
19+
self._default_source=default_sourceor {}
20+
21+
defis_suitable(
22+
self,
23+
devicetype:Optional[list[int]],
24+
category:Optional[list[int]],
25+
qid:Optional[list[int]],
26+
qideventcategory:Optional[list[int]],
27+
)->bool:
28+
device_type_match=set(devicetype).issubset(self.device_types)ifdevicetypeelseNone
29+
category_match=set(category).issubset(self.categories)ifcategoryelseNone
30+
qid_match=set(qid).issubset(self.qids)ifqidelseNone
31+
qid_event_category_match=set(qideventcategory).issubset(self.qid_event_categories)ifqideventcategoryelseNone
32+
returnall(
33+
conditionforconditionin (
34+
device_type_match,category_match,
35+
qid_match,qid_event_category_match)
36+
ifconditionisnotNone
37+
)
38+
39+
def__str__(self)->str:
40+
returnself._default_source.get("table","events")
41+
42+
@property
43+
defextra_condition(self)->str:
44+
default_source=self._default_source
45+
return" AND ".join((f"{key}={value}"forkey,valueindefault_source.items()ifkey!="table"andvalue))
46+
47+
48+
classAQLMappings(BasePlatformMappings):
49+
defprepare_log_source_signature(self,mapping:dict)->AQLLogSourceSignature:
50+
log_source=mapping.get("log_source", {})
51+
default_log_source=mapping["default_log_source"]
52+
returnAQLLogSourceSignature(
53+
device_types=log_source.get("devicetype"),
54+
categories=log_source.get("category"),
55+
qids=log_source.get("qid"),
56+
qid_event_categories=log_source.get("qideventcategory"),
57+
default_source=default_log_source,
58+
)
59+
60+
defget_suitable_source_mappings(
61+
self,
62+
field_names:list[str],
63+
devicetype:Optional[list[int]]=None,
64+
category:Optional[list[int]]=None,
65+
qid:Optional[list[int]]=None,
66+
qideventcategory:Optional[list[int]]=None,
67+
)->list[SourceMapping]:
68+
suitable_source_mappings= []
69+
forsource_mappinginself._source_mappings.values():
70+
ifsource_mapping.source_id==DEFAULT_MAPPING_NAME:
71+
continue
72+
73+
log_source_signature:AQLLogSourceSignature=source_mapping.log_source_signature
74+
iflog_source_signature.is_suitable(devicetype,category,qid,qideventcategory):
75+
ifsource_mapping.fields_mapping.is_suitable(field_names):
76+
suitable_source_mappings.append(source_mapping)
77+
elifsource_mapping.fields_mapping.is_suitable(field_names):
78+
suitable_source_mappings.append(source_mapping)
79+
80+
ifnotsuitable_source_mappings:
81+
suitable_source_mappings= [self._source_mappings[DEFAULT_MAPPING_NAME]]
82+
83+
returnsuitable_source_mappings
84+
85+
86+
aql_mappings=AQLMappings(platform_dir="qradar")
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
"""
2+
Uncoder IO Commercial Edition License
3+
-----------------------------------------------------------------
4+
Copyright (c) 2024 SOC Prime, Inc.
5+
6+
This file is part of the Uncoder IO Commercial Edition ("CE") and is
7+
licensed under the Uncoder IO Non-Commercial License (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
https://github.com/UncoderIO/UncoderIO/blob/main/LICENSE
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
-----------------------------------------------------------------
17+
"""
18+
19+
importre
20+
fromtypingimportUnion
21+
22+
fromapp.translator.core.models.query_containerimportRawQueryContainer,TokenizedQueryContainer
23+
fromapp.translator.core.parserimportPlatformQueryParser
24+
fromapp.translator.platforms.base.aql.constimportNUM_VALUE_PATTERN,SINGLE_QUOTES_VALUE_PATTERN
25+
fromapp.translator.platforms.base.aql.mappingimportAQLMappings,aql_mappings
26+
fromapp.translator.platforms.base.aql.tokenizerimportAQLTokenizer
27+
fromapp.translator.tools.utilsimportget_match_group
28+
29+
30+
classAQLQueryParser(PlatformQueryParser):
31+
tokenizer=AQLTokenizer()
32+
mappings:AQLMappings=aql_mappings
33+
34+
log_source_functions= ("LOGSOURCENAME","LOGSOURCEGROUPNAME","LOGSOURCETYPENAME","CATEGORYNAME")
35+
log_source_function_pattern=r"\(?(?P<key>___func_name___\([a-zA-Z]+\))(?:\s+like\s+|\s+ilike\s+|\s*=\s*)'(?P<value>[%a-zA-Z\s]+)'\s*\)?\s+(?:and|or)?\s"# noqa: E501
36+
37+
log_source_key_types= ("devicetype","category","qid","qideventcategory")
38+
log_source_pattern=rf"___source_type___(?:\s+like\s+|\s+ilike\s+|\s*=\s*)(?:{SINGLE_QUOTES_VALUE_PATTERN}|{NUM_VALUE_PATTERN})(?:\s+(?:and|or)\s+|\s+)?"# noqa: E501
39+
num_value_pattern=r"[0-9]+"
40+
multi_num_log_source_pattern= (
41+
rf"___source_type___\s+in\s+\((?P<value>(?:{num_value_pattern}(?:\s*,\s*)?)+)\)(?:\s+(?:and|or)\s+|\s+)?"
42+
)
43+
str_value_pattern=r"""(?:')(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')+)(?:')"""
44+
multi_str_log_source_pattern= (
45+
rf"""___source_type___\s+in\s+\((?P<value>(?:{str_value_pattern}(?:\s*,\s*)?)+)\)(?:\s+(?:and|or)\s+|\s+)?"""
46+
)
47+
48+
table_pattern=r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"
49+
50+
def__clean_query(self,query:str)->str:
51+
forfunc_nameinself.log_source_functions:
52+
pattern=self.log_source_function_pattern.replace("___func_name___",func_name)
53+
whilesearch:=re.search(pattern,query,flags=re.IGNORECASE):
54+
pos_start=search.start()
55+
pos_end=search.end()
56+
query=query[:pos_start]+query[pos_end:]
57+
58+
returnquery
59+
60+
@staticmethod
61+
def__parse_multi_value_log_source(
62+
match:re.Match,query:str,pattern:str
63+
)->tuple[str,Union[list[str],list[int]]]:
64+
value=match.group("value")
65+
pos_start=match.start()
66+
pos_end=match.end()
67+
query=query[:pos_start]+query[pos_end:]
68+
returnquery,re.findall(pattern,value)
69+
70+
def__parse_log_sources(self,query:str)->tuple[dict[str,Union[list[str],list[int]]],str]:
71+
log_sources= {}
72+
73+
ifsearch:=re.search(self.table_pattern,query,flags=re.IGNORECASE):
74+
pos_end=search.end()
75+
query=query[pos_end:]
76+
77+
forlog_source_keyinself.log_source_key_types:
78+
pattern=self.log_source_pattern.replace("___source_type___",log_source_key)
79+
whilesearch:=re.search(pattern,query,flags=re.IGNORECASE):
80+
num_value=get_match_group(search,group_name="num_value")
81+
str_value=get_match_group(search,group_name="s_q_value")
82+
value=num_valueandint(num_value)orstr_value
83+
log_sources.setdefault(log_source_key, []).append(value)
84+
pos_start=search.start()
85+
pos_end=search.end()
86+
query=query[:pos_start]+query[pos_end:]
87+
88+
pattern=self.multi_num_log_source_pattern.replace("___source_type___",log_source_key)
89+
ifsearch:=re.search(pattern,query,flags=re.IGNORECASE):
90+
query,values=self.__parse_multi_value_log_source(search,query,self.num_value_pattern)
91+
values= [int(v)forvinvalues]
92+
log_sources.setdefault(log_source_key, []).extend(values)
93+
94+
pattern=self.multi_str_log_source_pattern.replace("___source_type___",log_source_key)
95+
ifsearch:=re.search(pattern,query,flags=re.IGNORECASE):
96+
query,values=self.__parse_multi_value_log_source(search,query,self.str_value_pattern)
97+
log_sources.setdefault(log_source_key, []).extend(values)
98+
99+
returnlog_sources,query
100+
101+
def_parse_query(self,text:str)->tuple[str,dict[str,Union[list[str],list[int]]]]:
102+
query=self.__clean_query(text)
103+
log_sources,query=self.__parse_log_sources(query)
104+
returnquery,log_sources
105+
106+
defparse(self,raw_query_container:RawQueryContainer)->TokenizedQueryContainer:
107+
query,log_sources=self._parse_query(raw_query_container.query)
108+
tokens,source_mappings=self.get_tokens_and_source_mappings(query,log_sources)
109+
fields_tokens=self.get_fields_tokens(tokens=tokens)
110+
meta_info=raw_query_container.meta_info
111+
meta_info.query_fields=fields_tokens
112+
meta_info.source_mapping_ids= [source_mapping.source_idforsource_mappinginsource_mappings]
113+
returnTokenizedQueryContainer(tokens=tokens,meta_info=meta_info)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp