Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit441545e

Browse files
authored
Merge pull request#18 from UncoderIO/space-as-and
Interpret a space as and
2 parents5bde75e +f72ca55 commit441545e

File tree

4 files changed

+47
-7
lines changed

4 files changed

+47
-7
lines changed
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
fromtypingimportList,Union
2+
3+
fromapp.converter.core.models.fieldimportField,Keyword
4+
fromapp.converter.core.models.identifierimportIdentifier
5+
fromapp.converter.core.custom_types.tokensimportLogicalOperatorType,GroupType
6+
7+
8+
classANDLogicOperatorMixin:
9+
10+
@staticmethod
11+
defget_missed_and_token_indices(tokens:List[Union[Field,Keyword,Identifier]])->List[int]:
12+
missed_and_indices= []
13+
forindexinrange(len(tokens)-1):
14+
token=tokens[index]
15+
next_token=tokens[index+1]
16+
if (isinstance(token, (Field,Keyword))
17+
andnot (isinstance(next_token,Identifier)and (
18+
next_token.token_typeinLogicalOperatorType
19+
ornext_token.token_type==GroupType.R_PAREN))):
20+
missed_and_indices.append(index+1)
21+
returnreversed(missed_and_indices)
22+
23+
defadd_and_token_if_missed(self,tokens:List[Union[Field,Keyword,Identifier]])->List[Union[Field,Keyword,Identifier]]:
24+
indices=self.get_missed_and_token_indices(tokens=tokens)
25+
forindexinindices:
26+
tokens.insert(index,Identifier(token_type=LogicalOperatorType.AND))
27+
returntokens

‎siem-converter/app/converter/platforms/base/lucene/tokenizer.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@
2020
fromtypingimportTuple,Union,List,Any
2121

2222
fromapp.converter.core.exceptions.parserimportTokenizerGeneralException
23+
fromapp.converter.core.mixins.logicimportANDLogicOperatorMixin
2324
fromapp.converter.core.models.fieldimportKeyword,Field
2425
fromapp.converter.core.models.identifierimportIdentifier
2526
fromapp.converter.core.tokenizerimportQueryTokenizer
2627
fromapp.converter.core.custom_types.tokensimportOperatorType
2728
fromapp.converter.tools.utilsimportget_match_group
2829

2930

30-
classLuceneTokenizer(QueryTokenizer):
31+
classLuceneTokenizer(QueryTokenizer,ANDLogicOperatorMixin):
3132
field_pattern=r"(?P<field_name>[a-zA-Z\.\-_]+)"
3233
match_operator_pattern=r"(?:___field___\s*(?P<match_operator>:))\s*"
3334

@@ -107,3 +108,7 @@ def search_keyword(self, query: str) -> Tuple[Keyword, str]:
107108
keyword=Keyword(value=value)
108109
pos=keyword_search.end()-1
109110
returnkeyword,query[pos:]
111+
112+
deftokenize(self,query:str)->List[Union[Field,Keyword,Identifier]]:
113+
tokens=super().tokenize(query=query)
114+
returnself.add_and_token_if_missed(tokens=tokens)

‎siem-converter/app/converter/platforms/base/spl/tokenizer.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,17 @@
1717
"""
1818

1919
importre
20-
fromtypingimportTuple,Any
20+
fromtypingimportTuple,Any,List,Union
2121

22+
fromapp.converter.core.mixins.logicimportANDLogicOperatorMixin
23+
fromapp.converter.core.models.fieldimportField,Keyword
24+
fromapp.converter.core.models.identifierimportIdentifier
2225
fromapp.converter.core.tokenizerimportQueryTokenizer
2326
fromapp.converter.core.custom_types.tokensimportOperatorType
2427
fromapp.converter.tools.utilsimportget_match_group
2528

2629

27-
classSplTokenizer(QueryTokenizer):
30+
classSplTokenizer(QueryTokenizer,ANDLogicOperatorMixin):
2831
field_pattern=r"(?P<field_name>[a-zA-Z\.\-_\{\}]+)"
2932
num_value_pattern=r"(?P<num_value>\d+(?:\.\d+)*)\s*"
3033
double_quotes_value_pattern=r'"(?P<d_q_value>(?:[:a-zA-Z\*0-9=+%#\-_/,;\'\.$&^@!\(\)\{\}\s]|\\\"|\\)*)"\s*'
@@ -51,3 +54,7 @@ def get_operator_and_value(self, match: re.Match, operator: str = OperatorType.E
5154
returnoperator,s_q_value
5255

5356
returnsuper().get_operator_and_value(match)
57+
58+
deftokenize(self,query:str)->List[Union[Field,Keyword,Identifier]]:
59+
tokens=super().tokenize(query=query)
60+
returnself.add_and_token_if_missed(tokens=tokens)

‎siem-converter/app/converter/platforms/logscale/tokenizer.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,17 @@
1717
"""
1818

1919
importre
20-
fromtypingimportTuple,Any
20+
fromtypingimportTuple,Any,List,Union
2121

22+
fromapp.converter.core.mixins.logicimportANDLogicOperatorMixin
2223
fromapp.converter.core.models.fieldimportKeyword,Field
2324
fromapp.converter.core.models.identifierimportIdentifier
2425
fromapp.converter.core.custom_types.tokensimportGroupType,LogicalOperatorType,OperatorType
2526
fromapp.converter.core.tokenizerimportQueryTokenizer
2627
fromapp.converter.tools.utilsimportget_match_group
2728

2829

29-
classLogScaleTokenizer(QueryTokenizer):
30+
classLogScaleTokenizer(QueryTokenizer,ANDLogicOperatorMixin):
3031
match_operator_pattern=r"""(?:___field___\s?(?P<match_operator>=|!=))\s?"""
3132
num_value_pattern=r"(?P<num_value>\d+(?:\.\d+)*)\s*"
3233
double_quotes_value_pattern=r'"(?P<d_q_value>(?:[:a-zA-Z\*0-9=+%#\-_/,\'\.$&^@!\(\)\{\}\s]|\\\"|\\)*)"\s*'
@@ -65,7 +66,7 @@ def __get_identifier(self, query: str) -> (list, str):
6566
else:
6667
returnself.search_field_value(query)
6768

68-
deftokenize(self,query:str)->list:
69+
deftokenize(self,query:str)->List[Union[Field,Keyword,Identifier]]:
6970
tokenized= []
7071
whilequery:
7172
identifier,query=self.__get_identifier(query=query)
@@ -78,4 +79,4 @@ def tokenize(self, query: str) -> list:
7879
tokenized.append(Identifier(token_type=LogicalOperatorType.AND))
7980
tokenized.append(identifier)
8081
self._validate_parentheses(tokenized)
81-
returntokenized
82+
returnself.add_and_token_if_missed(tokens=tokenized)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp