Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit416f5ca

Browse files
authored
Merge pull request#199 from UncoderIO/gis-8639
Gis 8639 add ElasticSearchEQLQueryParser
2 parents25a55d0 +608a1f4 commit416f5ca

File tree

7 files changed

+129
-3
lines changed

7 files changed

+129
-3
lines changed

‎uncoder-core/app/translator/platforms/elasticsearch/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
ElasticSearchRuleTOMLParser,# noqa: F401
44
)
55
fromapp.translator.platforms.elasticsearch.parsers.elasticsearchimportElasticSearchQueryParser# noqa: F401
6+
fromapp.translator.platforms.elasticsearch.parsers.elasticsearch_eqlimportElasticSearchEQLQueryParser# noqa: F401
67
fromapp.translator.platforms.elasticsearch.renders.detection_ruleimportElasticSearchRuleRender# noqa: F401
78
fromapp.translator.platforms.elasticsearch.renders.elast_alertimportElastAlertRuleRender# noqa: F401
89
fromapp.translator.platforms.elasticsearch.renders.elasticsearchimportElasticSearchQueryRender# noqa: F401

‎uncoder-core/app/translator/platforms/elasticsearch/const.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
_ELASTIC_WATCHER_RULE="elastic-watcher-rule"
1212
_ELASTIC_ESQL_QUERY="elastic-esql-query"
1313
_ELASTIC_ESQL_RULE="elastic-esql-rule"
14+
_ELASTIC_EQL_QUERY="elastic-eql-query"
1415

1516
ELASTIC_QUERY_TYPES= {
1617
_ELASTIC_LUCENE_QUERY,
@@ -83,6 +84,13 @@
8384
**PLATFORM_DETAILS,
8485
}
8586

87+
ELASTICSEARCH_EQL_QUERY_DETAILS= {
88+
"platform_id":_ELASTIC_EQL_QUERY,
89+
"name":"Elasticsearch EQL Query",
90+
"platform_name":"Query (EQL)",
91+
**PLATFORM_DETAILS,
92+
}
93+
8694
elasticsearch_lucene_query_details=PlatformDetails(**ELASTICSEARCH_LUCENE_QUERY_DETAILS)
8795
elasticsearch_esql_query_details=PlatformDetails(**ELASTICSEARCH_ESQL_QUERY_DETAILS)
8896
elasticsearch_esql_rule_details=PlatformDetails(**ELASTICSEARCH_ESQL_RULE_DETAILS)
@@ -91,6 +99,7 @@
9199
elastalert_details=PlatformDetails(**ELASTALERT_DETAILS)
92100
kibana_rule_details=PlatformDetails(**KIBANA_DETAILS)
93101
xpack_watcher_details=PlatformDetails(**XPACK_WATCHER_DETAILS)
102+
elastic_eql_query_details=PlatformDetails(**ELASTICSEARCH_EQL_QUERY_DETAILS)
94103

95104
ELASTICSEARCH_DETECTION_RULE= {
96105
"description":"Autogenerated ElasticSearch Detection Rule.",

‎uncoder-core/app/translator/platforms/elasticsearch/mapping.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
fromapp.translator.platforms.elasticsearch.constimport (
33
elastalert_details,
44
elasticsearch_esql_query_details,
5+
elastic_eql_query_details,
56
elasticsearch_lucene_query_details,
67
elasticsearch_rule_details,
78
kibana_rule_details,
@@ -17,6 +18,7 @@
1718
elastalert_mappings=LuceneMappings(platform_dir="elasticsearch",platform_details=elastalert_details)
1819
kibana_rule_mappings=LuceneMappings(platform_dir="elasticsearch",platform_details=kibana_rule_details)
1920
xpack_watcher_mappings=LuceneMappings(platform_dir="elasticsearch",platform_details=xpack_watcher_details)
21+
elastic_eql_query_mappings=LuceneMappings(platform_dir="elasticsearch",platform_details=elastic_eql_query_details)
2022

2123

2224
classElasticESQLMappings(LuceneMappings):
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
importre
2+
3+
fromapp.translator.core.models.platform_detailsimportPlatformDetails
4+
fromapp.translator.core.models.query_containerimportRawQueryContainer,TokenizedQueryContainer
5+
fromapp.translator.core.parserimportPlatformQueryParser
6+
fromapp.translator.managersimportparser_manager
7+
fromapp.translator.platforms.base.lucene.mappingimportLuceneMappings
8+
fromapp.translator.platforms.elasticsearch.constimportelastic_eql_query_details
9+
fromapp.translator.platforms.elasticsearch.mappingimportelastic_eql_query_mappings
10+
fromapp.translator.platforms.elasticsearch.tokenizerimportElasticSearchEQLTokenizer
11+
12+
13+
@parser_manager.register_supported_by_roota
14+
classElasticSearchEQLQueryParser(PlatformQueryParser):
15+
details:PlatformDetails=elastic_eql_query_details
16+
tokenizer=ElasticSearchEQLTokenizer()
17+
mappings:LuceneMappings=elastic_eql_query_mappings
18+
query_delimiter_pattern=r"\swhere\s"
19+
20+
def_parse_query(self,query:str)->tuple[str,dict[str,list[str]]]:
21+
log_source= {"category": []}
22+
ifre.search(self.query_delimiter_pattern,query,flags=re.IGNORECASE):
23+
sp_query=re.split(self.query_delimiter_pattern,query,flags=re.IGNORECASE)
24+
ifsp_query[0].lower()!="all":
25+
log_source["category"].append(sp_query[0])
26+
returnsp_query[1],log_source
27+
returnquery,log_source
28+
29+
defparse(self,raw_query_container:RawQueryContainer)->TokenizedQueryContainer:
30+
query,log_sources=self._parse_query(raw_query_container.query)
31+
query_tokens=self.get_query_tokens(query)
32+
field_tokens=self.get_field_tokens(query_tokens)
33+
source_mappings=self.get_source_mappings(field_tokens,log_sources)
34+
meta_info=raw_query_container.meta_info
35+
meta_info.query_fields=field_tokens
36+
meta_info.source_mapping_ids= [source_mapping.source_idforsource_mappinginsource_mappings]
37+
returnTokenizedQueryContainer(tokens=query_tokens,meta_info=meta_info)

‎uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,13 @@
2929
fromapp.translator.platforms.elasticsearch.mappingimportElasticESQLMappings,esql_query_mappings
3030
fromapp.translator.platforms.elasticsearch.str_value_managerimport (
3131
ESQLQueryStrValueManager,
32-
esql_query_str_value_manager
32+
esql_str_value_manager
3333
)
3434

3535

3636
classESQLFieldValueRender(BaseFieldValueRender):
3737
details:PlatformDetails=elasticsearch_esql_query_details
38-
str_value_manager:ESQLQueryStrValueManager=esql_query_str_value_manager
38+
str_value_manager:ESQLQueryStrValueManager=esql_str_value_manager
3939

4040
@staticmethod
4141
def_make_case_insensitive(value:str)->str:

‎uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
ReDigitalSymbol,
2424
ReWhiteSpaceSymbol,
2525
ReWordSymbol,
26+
SingleSymbolWildCard,
27+
StrValue,
2628
StrValueManager,
2729
)
2830
fromapp.translator.platforms.elasticsearch.escape_managerimportESQLQueryEscapeManager,esql_query_escape_manager
@@ -37,4 +39,13 @@ class ESQLQueryStrValueManager(StrValueManager):
3739
}
3840

3941

40-
esql_query_str_value_manager=ESQLQueryStrValueManager()
42+
classEQLStrValueManager(StrValueManager):
43+
str_spec_symbols_map:ClassVar[dict[str,type[BaseSpecSymbol]]]= {"*":SingleSymbolWildCard}
44+
45+
deffrom_str_to_container(self,value:str)->StrValue:
46+
split= [self.str_spec_symbols_map[char]()ifcharinself.str_spec_symbols_mapelsecharforcharinvalue]
47+
returnStrValue(value,self._concat(split))
48+
49+
50+
esql_str_value_manager=ESQLQueryStrValueManager()
51+
eql_str_value_manager=EQLStrValueManager()

‎uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,75 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
importre
19+
fromtypingimportAny,ClassVar,Optional,Union
1820

21+
fromapp.translator.core.custom_types.tokensimportOperatorType
22+
fromapp.translator.core.custom_types.valuesimportValueType
23+
fromapp.translator.core.models.query_tokens.field_valueimportFieldValue
24+
fromapp.translator.core.models.query_tokens.identifierimportIdentifier
25+
fromapp.translator.core.tokenizerimportQueryTokenizer
1926
fromapp.translator.platforms.base.lucene.tokenizerimportLuceneTokenizer
27+
fromapp.translator.platforms.elasticsearch.str_value_managerimporteql_str_value_manager
28+
fromapp.translator.tools.utilsimportget_match_group
2029

2130

2231
classElasticSearchTokenizer(LuceneTokenizer):
2332
pass
33+
34+
35+
classElasticSearchEQLTokenizer(QueryTokenizer):
36+
single_value_operators_map:ClassVar[dict[str,str]]= {
37+
":":OperatorType.EQ,
38+
"==":OperatorType.EQ,
39+
"<=":OperatorType.LTE,
40+
"<":OperatorType.LT,
41+
">=":OperatorType.GTE,
42+
">":OperatorType.GT,
43+
"!=":OperatorType.NOT_EQ,
44+
"regex~":OperatorType.REGEX,
45+
"regex":OperatorType.REGEX,
46+
}
47+
48+
multi_value_operators_map:ClassVar[dict[str,str]]= {
49+
"in":OperatorType.EQ,
50+
"in~":OperatorType.EQ,
51+
":":OperatorType.EQ,
52+
}
53+
wildcard_symbol="*"
54+
field_pattern=r"(?P<field_name>[a-zA-Z\.\-_`]+)"
55+
re_value_pattern= (
56+
rf'"(?P<{ValueType.regex_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)\[\^[z|Z]\]\.\?"'# noqa: RUF001
57+
)
58+
double_quotes_value_pattern= (
59+
rf'"(?P<{ValueType.double_quotes_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)"'# noqa: RUF001
60+
)
61+
_value_pattern=rf"{re_value_pattern}|{double_quotes_value_pattern}"
62+
multi_value_pattern=rf"""\((?P<{ValueType.multi_value}>[:a-zA-Z\"\*0-9=+%#№;\-_\/\\'\,.$&^@!\(\[\]\s|]+)\)"""
63+
multi_value_check_pattern=r"___field___\s*___operator___\s*\("
64+
keyword_pattern= (
65+
rf'"(?P<{ValueType.double_quotes_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)"'# noqa: RUF001
66+
)
67+
68+
str_value_manager=eql_str_value_manager
69+
70+
defget_operator_and_value(
71+
self,match:re.Match,mapped_operator:str=OperatorType.EQ,operator:Optional[str]=None
72+
)->tuple[str,Any]:
73+
if (re_value:=get_match_group(match,group_name=ValueType.regex_value))isnotNone:
74+
returnOperatorType.REGEX,self.str_value_manager.from_re_str_to_container(re_value)
75+
76+
if (d_q_value:=get_match_group(match,group_name=ValueType.double_quotes_value))isnotNone:
77+
returnmapped_operator,self.str_value_manager.from_str_to_container(d_q_value)
78+
79+
returnsuper().get_operator_and_value(match,mapped_operator,operator)
80+
81+
defis_multi_value_flow(self,field_name:str,operator:str,query:str)->bool:
82+
check_pattern=self.multi_value_check_pattern
83+
check_regex=check_pattern.replace("___field___",field_name).replace("___operator___",operator)
84+
returnbool(re.match(check_regex,query))
85+
86+
@staticmethod
87+
defcreate_field_value(field_name:str,operator:Identifier,value:Union[str,list])->FieldValue:
88+
field_name=field_name.replace("`","")
89+
returnFieldValue(source_name=field_name,operator=operator,value=value)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp