Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitc34e622

Browse files
committed
parser, tokenizer, render fixes
1 parentc026f1d commitc34e622

File tree

19 files changed

+71
-63
lines changed

19 files changed

+71
-63
lines changed

‎siem-converter/app/converter/backends/athena/renders/athena.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ class AthenaQueryRender(BaseQueryRender):
6767

6868
field_value_map=AthenaFieldValue(or_token=or_token)
6969
query_pattern="{prefix} WHERE {query} {functions}"
70+
comment_symbol="--"
71+
is_multi_line_comment=True
7072

7173
defgenerate_prefix(self,log_source_signature:LogSourceSignature)->str:
7274
table=str(log_source_signature)ifstr(log_source_signature)else"eventlog"

‎siem-converter/app/converter/backends/athena/tokenizer.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class AthenaTokenizer(QueryTokenizer):
3030
match_operator_pattern=r"""(?:___field___\s?(?P<match_operator>like|in|=|>|<|>=|<=|<>|!=))\s?"""
3131
num_value_pattern=r"(?P<num_value>\d+(?:\.\d+)*)\s*"
3232
bool_value_pattern=r"(?P<bool_value>true|false)\s*"
33-
single_quotes_value_pattern=r"""'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')+)'"""
33+
single_quotes_value_pattern=r"""'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')*)'"""
3434
_value_pattern=fr"{num_value_pattern}|{bool_value_pattern}|{single_quotes_value_pattern}"
3535
multi_value_pattern=r"""\((?P<value>\d+(?:,\s*\d+)*|'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')*'(?:,\s*'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')*')*)\)"""
3636

@@ -49,13 +49,13 @@ def should_process_value_wildcard_symbols(operator: str) -> bool:
4949
returnoperator.lower()in ("like",)
5050

5151
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
52-
ifnum_value:=get_match_group(match,group_name='num_value'):
52+
if(num_value:=get_match_group(match,group_name='num_value'))isnotNone:
5353
returnoperator,num_value
5454

55-
elifbool_value:=get_match_group(match,group_name='bool_value'):
55+
elif(bool_value:=get_match_group(match,group_name='bool_value'))isnotNone:
5656
returnoperator,bool_value
5757

58-
elifs_q_value:=get_match_group(match,group_name='s_q_value'):
58+
elif(s_q_value:=get_match_group(match,group_name='s_q_value'))isnotNone:
5959
returnoperator,s_q_value
6060

6161
returnsuper().get_operator_and_value(match,operator)

‎siem-converter/app/converter/backends/chronicle/tokenizer.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,22 +31,22 @@ class ChronicleQueryTokenizer(QueryTokenizer):
3131
num_value_pattern=r"(?P<num_value>\d+(?:\.\d+)*)\s*"
3232
bool_value_pattern=r"(?P<bool_value>true|false)\s*"
3333
double_quotes_value_pattern=r'"(?P<d_q_value>(?:[:a-zA-Z\*0-9=+%#\-_/,\'\.$&^@!\(\)\{\}\s]|\\\"|\\\\)*)"\s*(?:nocase)?'
34-
re_value_pattern=r"/(?P<re_value>[:a-zA-Z\*0-9=+%#\\\-_\,\"\'\.$&^@!\(\)\{\}\s?]*)/\s*(?:nocase)?"
34+
re_value_pattern=r"/(?P<re_value>(?:\\\/|[:a-zA-Z\*0-9=+%#\\\-_\,\"\'\.$&^@!\(\)\{\}\s?])+)/\s*(?:nocase)?"
3535
_value_pattern=fr"{num_value_pattern}|{bool_value_pattern}|{double_quotes_value_pattern}|{re_value_pattern}"
3636

3737
wildcard_symbol=".*"
3838

3939
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
40-
ifnum_value:=get_match_group(match,group_name='num_value'):
40+
if(num_value:=get_match_group(match,group_name='num_value'))isnotNone:
4141
returnoperator,num_value
4242

43-
elifbool_value:=get_match_group(match,group_name='bool_value'):
43+
elif(bool_value:=get_match_group(match,group_name='bool_value'))isnotNone:
4444
returnoperator,bool_value
4545

46-
elifd_q_value:=get_match_group(match,group_name='d_q_value'):
46+
elif(d_q_value:=get_match_group(match,group_name='d_q_value'))isnotNone:
4747
returnoperator,d_q_value
4848

49-
elifre_value:=get_match_group(match,group_name='re_value'):
49+
elif(re_value:=get_match_group(match,group_name='re_value'))isnotNone:
5050
returnOperatorType.REGEX,re_value
5151

5252
returnsuper().get_operator_and_value(match,operator)
@@ -94,10 +94,10 @@ def search_field_value(self, query):
9494
returnsuper().search_field_value(query=query)
9595

9696
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
97-
ifd_q_value:=get_match_group(match,group_name='d_q_value'):
97+
if(d_q_value:=get_match_group(match,group_name='d_q_value'))isnotNone:
9898
returnoperator,d_q_value
9999

100-
elifb_q_value:=get_match_group(match,group_name='b_q_value'):
100+
elif(b_q_value:=get_match_group(match,group_name='b_q_value'))isnotNone:
101101
returnoperator,b_q_value
102102

103103
returnsuper().get_operator_and_value(match,operator)

‎siem-converter/app/converter/backends/elasticsearch/renders/elasticsearch.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ class ElasticSearchQueryRender(BaseQueryRender):
8282

8383
field_value_map=ElasticSearchFieldValue(or_token=or_token)
8484
query_pattern="{query} {functions}"
85+
comment_symbol="//"
86+
is_multi_line_comment=True
8587

8688
defgenerate_prefix(self,logsource:dict)->str:
8789
return""

‎siem-converter/app/converter/backends/elasticsearch/tokenizer.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,16 @@ def clean_quotes(value: Union[str, int]):
6464
returnvalue
6565

6666
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
67-
ifnum_value:=get_match_group(match,group_name='num_value'):
67+
if(num_value:=get_match_group(match,group_name='num_value'))isnotNone:
6868
returnoperator,num_value
6969

70-
elifre_value:=get_match_group(match,group_name='re_value'):
70+
elif(re_value:=get_match_group(match,group_name='re_value'))isnotNone:
7171
returnOperatorType.REGEX,re_value
7272

73-
elifn_q_value:=get_match_group(match,group_name='n_q_value'):
73+
elif(n_q_value:=get_match_group(match,group_name='n_q_value'))isnotNone:
7474
returnoperator,n_q_value
7575

76-
elifd_q_value:=get_match_group(match,group_name='d_q_value'):
76+
elif(d_q_value:=get_match_group(match,group_name='d_q_value'))isnotNone:
7777
returnoperator,d_q_value
7878

7979
returnsuper().get_operator_and_value(match)

‎siem-converter/app/converter/backends/logscale/tokenizer.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,20 +30,20 @@ class LogScaleTokenizer(QueryTokenizer):
3030
match_operator_pattern=r"""(?:___field___\s?(?P<match_operator>=|!=))\s?"""
3131
num_value_pattern=r"(?P<num_value>\d+(?:\.\d+)*)\s*"
3232
double_quotes_value_pattern=r'"(?P<d_q_value>(?:[:a-zA-Z\*0-9=+%#\-_/,\'\.$&^@!\(\)\{\}\s]|\\\"|\\)*)"\s*'
33-
re_value_pattern=r"/(?P<re_value>[:a-zA-Z\*0-9=+%#\\\-_\,\"\'\.$&^@!\(\)\{\}\s?]*)/i?\s*"
33+
re_value_pattern=r"/(?P<re_value>[:a-zA-Z\*0-9=+%#\\\-_\,\"\'\.$&^@!\(\)\{\}\s?]+)/i?\s*"
3434
_value_pattern=fr"""{num_value_pattern}|{re_value_pattern}|{double_quotes_value_pattern}"""
3535
keyword_pattern=double_quotes_value_pattern
3636

3737
wildcard_symbol="*"
3838

3939
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
40-
ifnum_value:=get_match_group(match,group_name='num_value'):
40+
if(num_value:=get_match_group(match,group_name='num_value'))isnotNone:
4141
returnoperator,num_value
4242

43-
elifd_q_value:=get_match_group(match,group_name='d_q_value'):
43+
elif(d_q_value:=get_match_group(match,group_name='d_q_value'))isnotNone:
4444
returnoperator,d_q_value
4545

46-
elifre_value:=get_match_group(match,group_name='re_value'):
46+
elif(re_value:=get_match_group(match,group_name='re_value'))isnotNone:
4747
returnOperatorType.REGEX,re_value
4848

4949
returnsuper().get_operator_and_value(match,operator)

‎siem-converter/app/converter/backends/microsoft/renders/microsoft_sentinel.py‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
fromtypingimportUnion
1920

2021
fromapp.converter.backends.microsoft.constimportmicrosoft_sentinel_query_details
2122
fromapp.converter.backends.microsoft.mappingimportMicrosoftSentinelMappings,microsoft_sentinel_mappings
@@ -28,32 +29,36 @@
2829
classMicrosoftSentinelFieldValue(BaseQueryFieldValue):
2930
details:PlatformDetails=microsoft_sentinel_query_details
3031

32+
@staticmethod
33+
def__escape_value(value:Union[int,str])->Union[int,str]:
34+
returnvalue.replace("'","''")ifisinstance(value,str)elsevalue
35+
3136
defequal_modifier(self,field,value):
3237
ifisinstance(value,str):
33-
returnf"{field} =~ @'{value}'"
38+
returnf"{field} =~ @'{self.__escape_value(value)}'"
3439
elifisinstance(value,list):
35-
prepared_values=", ".join(f"@'{v}'"forvinvalue)
40+
prepared_values=", ".join(f"@'{self.__escape_value(v)}'"forvinvalue)
3641
operator="in~"ifall(isinstance(v,str)forvinvalue)else"in"
3742
returnf'{field}{operator} ({prepared_values})'
3843
returnf'{field} =={value}'
3944

4045
defcontains_modifier(self,field,value):
4146
ifisinstance(value,list):
4247
returnf"({self.or_token.join(self.contains_modifier(field=field,value=v)forvinvalue)})"
43-
returnf"{field} contains @'{value}'"
48+
returnf"{field} contains @'{self.__escape_value(value)}'"
4449

4550
defendswith_modifier(self,field,value):
4651
ifisinstance(value,list):
4752
returnf"({self.or_token.join(self.endswith_modifier(field=field,value=v)forvinvalue)})"
48-
returnf"{field} endswith @'{value}'"
53+
returnf"{field} endswith @'{self.__escape_value(value)}'"
4954

5055
defstartswith_modifier(self,field,value):
5156
ifisinstance(value,list):
5257
returnf"({self.or_token.join(self.startswith_modifier(field=field,value=v)forvinvalue)})"
53-
returnf"{field} startswith @'{value}'"
58+
returnf"{field} startswith @'{self.__escape_value(value)}'"
5459

5560
def__regex_modifier(self,field,value):
56-
returnf"{field} matches regex @'(?i){value}'"
61+
returnf"{field} matches regex @'(?i){self.__escape_value(value)}'"
5762

5863
defregex_modifier(self,field,value):
5964
ifisinstance(value,list):
@@ -63,7 +68,7 @@ def regex_modifier(self, field, value):
6368
defkeywords(self,field,value):
6469
ifisinstance(value,list):
6570
returnf"({self.or_token.join(self.keywords(field=field,value=v)forvinvalue)})"
66-
returnf"* contains @'{value}'"
71+
returnf"* contains @'{self.__escape_value(value)}'"
6772

6873

6974
classMicrosoftSentinelQueryRender(BaseQueryRender):
@@ -78,14 +83,11 @@ class MicrosoftSentinelQueryRender(BaseQueryRender):
7883

7984
mappings:MicrosoftSentinelMappings=microsoft_sentinel_mappings
8085
comment_symbol="//"
86+
is_multi_line_comment=True
8187

8288
defgenerate_prefix(self,log_source_signature:LogSourceSignature)->str:
8389
returnstr(log_source_signature)
8490

85-
defrender_not_supported_functions(self,not_supported_functions:list)->str:
86-
render_not_suported="\n".join([f'//{i}'foriinnot_supported_functions])
87-
return"\n\n"+f"//{self.unsupported_functions_text}"+render_not_suported
88-
8991
defgenerate_functions(self,functions:list)->str:
9092
ifnotfunctions:
9193
return""

‎siem-converter/app/converter/backends/microsoft/tokenizer.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class MicrosoftSentinelTokenizer(QueryTokenizer, OperatorBasedMixin):
3434
single_quotes_value_pattern=r"@?'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-_/,\"\.$&^@!\(\)\{\}\s]|\\\'|\\\\)*)'\s*"
3535
str_value_pattern=fr"""{double_quotes_value_pattern}|{single_quotes_value_pattern}"""
3636
_value_pattern=fr"""{bool_value_pattern}|{num_value_pattern}|{str_value_pattern}"""
37-
multi_value_pattern=r"""\((?P<value>[:a-zA-Z\"\*0-9=+%#\-_\/\\'\,.&^@!\(\s]*)\)"""
37+
multi_value_pattern=r"""\((?P<value>[:a-zA-Z\"\*0-9=+%#\-_\/\\'\,.&^@!\(\s]+)\)"""
3838
keyword_pattern=fr"\*\s+contains\s+(?:{str_value_pattern})"
3939

4040
multi_value_operators= ("in","in~")
@@ -50,16 +50,16 @@ def __init__(self, *args, **kwargs):
5050
self.operators_map.update(super().operators_map)
5151

5252
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
53-
ifnum_value:=get_match_group(match,group_name='num_value'):
53+
if(num_value:=get_match_group(match,group_name='num_value'))isnotNone:
5454
returnoperator,num_value
5555

56-
elifbool_value:=get_match_group(match,group_name='bool_value'):
56+
elif(bool_value:=get_match_group(match,group_name='bool_value'))isnotNone:
5757
returnoperator,bool_value
5858

59-
elifd_q_value:=get_match_group(match,group_name='d_q_value'):
59+
elif(d_q_value:=get_match_group(match,group_name='d_q_value'))isnotNone:
6060
returnoperator,d_q_value
6161

62-
elifs_q_value:=get_match_group(match,group_name='s_q_value'):
62+
elif(s_q_value:=get_match_group(match,group_name='s_q_value'))isnotNone:
6363
returnoperator,s_q_value
6464

6565
returnsuper().get_operator_and_value(match,operator)

‎siem-converter/app/converter/backends/opensearch/renders/opensearch.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ class OpenSearchQueryRender(BaseQueryRender):
7171

7272
field_value_map=OpenSearchFieldValue(or_token=or_token)
7373
query_pattern="{query} {functions}"
74+
comment_symbol="//"
75+
is_multi_line_comment=True
7476

7577
defgenerate_prefix(self,logsource:dict)->str:
7678
return""

‎siem-converter/app/converter/backends/opensearch/tokenizer.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,16 @@ def clean_quotes(value: Union[str, int]):
6464
returnvalue
6565

6666
defget_operator_and_value(self,match:re.Match,operator:str=OperatorType.EQ)->Tuple[str,Any]:
67-
ifnum_value:=get_match_group(match,group_name='num_value'):
67+
if(num_value:=get_match_group(match,group_name='num_value'))isnotNone:
6868
returnoperator,num_value
6969

70-
elifre_value:=get_match_group(match,group_name='re_value'):
70+
elif(re_value:=get_match_group(match,group_name='re_value'))isnotNone:
7171
returnOperatorType.REGEX,re_value
7272

73-
elifn_q_value:=get_match_group(match,group_name='n_q_value'):
73+
elif(n_q_value:=get_match_group(match,group_name='n_q_value'))isnotNone:
7474
returnoperator,n_q_value
7575

76-
elifd_q_value:=get_match_group(match,group_name='d_q_value'):
76+
elif(d_q_value:=get_match_group(match,group_name='d_q_value'))isnotNone:
7777
returnoperator,d_q_value
7878

7979
returnsuper().get_operator_and_value(match)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp