Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

sql str value manager#206

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
alexvolha merged 9 commits intomainfromgis-7011
Oct 22, 2024
Merged
Show file tree
Hide file tree
Changes from1 commit
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
PrevPrevious commit
NextNext commit
parse sql escape symbol
  • Loading branch information
@alexvolha
alexvolha committedOct 19, 2024
commit40d51c4064f51cde1eeea0a781782365de33889c
2 changes: 1 addition & 1 deletionuncoder-core/app/translator/core/str_value_manager.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -182,7 +182,7 @@ class StrValueManager:
container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = CONTAINER_SPEC_SYMBOLS_MAP

@staticmethod
def from_str_to_container(value: str) -> StrValue:
def from_str_to_container(value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG004
return StrValue(value=value, split_value=[value])

def from_re_str_to_container(self, value: str) -> StrValue:
Expand Down
3 changes: 1 addition & 2 deletionsuncoder-core/app/translator/core/tokenizer.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -162,8 +162,7 @@ def search_multi_value(

def _get_field_value_match(self, query: str, operator: str, field_name: str, value_pattern: str) -> re.Match:
field_value_pattern = self.get_field_value_pattern(operator, field_name, value_pattern)
field_value_regex = re.compile(field_value_pattern, re.IGNORECASE)
field_value_match = re.match(field_value_regex, query)
field_value_match = re.match(field_value_pattern, query, re.IGNORECASE)
if field_value_match is None:
raise TokenizerGeneralException(error=f"Value couldn't be found in query part: {query}")

Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
"""

import copy
from typing import ClassVar
from typing import ClassVar, Optional

from app.translator.core.custom_types.values import ValueType
from app.translator.core.str_value_manager import (
Expand DownExpand Up@@ -55,7 +55,7 @@ class AQLStrValueManager(StrValueManager):
"%": UnboundLenWildCard,
}

def from_str_to_container(self, value: str) -> StrValue:
def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002
split = []
prev_char = None
for char in value:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,7 +17,7 @@
-----------------------------------------------------------------
"""

from typing import ClassVar
from typing import ClassVar, Optional

from app.translator.core.str_value_manager import (
BaseSpecSymbol,
Expand DownExpand Up@@ -68,7 +68,7 @@ class LuceneStrValueManager(StrValueManager):
}
re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP

def from_str_to_container(self, value: str) -> StrValue:
def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002
split = []
prev_char = None
for char in value:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,7 +16,7 @@
limitations under the License.
-----------------------------------------------------------------
"""
from typing import ClassVar
from typing import ClassVar, Optional

from app.translator.core.str_value_manager import BaseSpecSymbol, StrValue, StrValueManager, UnboundLenWildCard
from app.translator.platforms.base.spl.escape_manager import spl_escape_manager
Expand All@@ -26,7 +26,7 @@ class SplStrValueManager(StrValueManager):
escape_manager = spl_escape_manager
str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": UnboundLenWildCard}

def from_str_to_container(self, value: str) -> StrValue:
def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002
split = []
prev_char = None
for char in value:
Expand Down
Empty file.
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
from app.translator.core.custom_types.values import ValueType


class SQLValueType(ValueType):
like_value = "like_value"
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
from typing import ClassVar

from app.translator.core.custom_types.values import ValueType
from app.translator.core.escape_manager import EscapeManager
from app.translator.core.models.escape_details import EscapeDetails
from app.translator.platforms.base.sql.custom_types.values import SQLValueType


class SQLEscapeManager(EscapeManager):
escape_map: ClassVar[dict[str, list[EscapeDetails]]] = {
ValueType.value: [EscapeDetails(pattern=r"(')", escape_symbols=r"'\1")],
ValueType.regex_value: [
SQLValueType.value: [EscapeDetails(pattern=r"(')", escape_symbols=r"'\1")],
SQLValueType.like_value: [EscapeDetails(pattern=r"(['%_\\])", escape_symbols=r"\\\1")],
SQLValueType.regex_value: [
EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1"),
EscapeDetails(pattern=r"(')", escape_symbols=r"'\1"),
],
Expand Down
13 changes: 10 additions & 3 deletionsuncoder-core/app/translator/platforms/base/sql/renders/sql.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@
from app.translator.core.custom_types.values import ValueType
from app.translator.core.mapping import LogSourceSignature
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
from app.translator.platforms.base.sql.custom_types.values import SQLValueType
from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager


Expand DownExpand Up@@ -56,17 +57,23 @@ def greater_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> st
def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
if isinstance(value, list):
return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})"
return f"{field} like '%{self._pre_process_value(field, value)}%'"

value = f"'%{self._pre_process_value(field, value, value_type=SQLValueType.like_value)}%' escape '\\'"
return f"{field} like {value}"

def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
if isinstance(value, list):
return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})"
return f"{field} like '%{self._pre_process_value(field, value)}'"

value = f"'%{self._pre_process_value(field, value, value_type=SQLValueType.like_value)}' escape '\\'"
return f"{field} like {value}"

def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
if isinstance(value, list):
return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})"
return f"{field} like '{self._pre_process_value(field, value)}%'"

value = f"'{self._pre_process_value(field, value, value_type=SQLValueType.like_value)}%' escape '\\'"
return f"{field} like {value}"

def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
if isinstance(value, list):
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,7 +18,7 @@
"""

import copy
from typing import ClassVar
from typing import ClassVar, Optional

from app.translator.core.custom_types.values import ValueType
from app.translator.core.str_value_manager import (
Expand DownExpand Up@@ -55,20 +55,28 @@ class SQLStrValueManager(StrValueManager):
"%": UnboundLenWildCard,
}

def from_str_to_container(self, value: str) -> StrValue:
def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue:
split = []
prev_char = None
for char in value:
if char in self.str_spec_symbols_map:
split.append(self.str_spec_symbols_map[char]())
else:
if char == "'":
if prev_char == "'":
split.append(char)
prev_char = None
continue
prev_char = char
if escape_symbol and char == escape_symbol:
if prev_char == escape_symbol:
split.append(char)
prev_char = None
continue
prev_char = char
continue
if not escape_symbol and char == "'":
if prev_char == "'":
split.append(char)
prev_char = None
continue
elif char in ("'", "_", "%"):
if escape_symbol and prev_char == escape_symbol:
split.append(char)
elif char in self.str_spec_symbols_map:
split.append(self.str_spec_symbols_map[char]())
else:
split.append(char)

prev_char = char
Expand Down
13 changes: 5 additions & 8 deletionsuncoder-core/app/translator/platforms/base/sql/tokenizer.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,8 @@
from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager
from app.translator.tools.utils import get_match_group

_ESCAPE_SYMBOL_GROUP_NAME = "escape_symbol"


class SqlTokenizer(QueryTokenizer):
single_value_operators_map: ClassVar[dict[str, str]] = {
Expand All@@ -46,9 +48,7 @@ class SqlTokenizer(QueryTokenizer):
field_pattern = r'(?P<field_name>"[a-zA-Z\._\-\s]+"|[a-zA-Z\._\-]+)'
num_value_pattern = rf"(?P<{ValueType.number_value}>\d+(?:\.\d+)*)\s*"
bool_value_pattern = rf"(?P<{ValueType.bool_value}>true|false)\s*"
single_quotes_value_pattern = (
rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*)'"""
)
single_quotes_value_pattern = rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Z\*0-9=+%#\-\/,_".$&^@!\(\)\{{\}}\s]|''|\\\'|\\\%|\\\_|\\\\)*)'(?:\s+escape\s+'(?P<{_ESCAPE_SYMBOL_GROUP_NAME}>.)')?""" # noqa: E501
_value_pattern = rf"{num_value_pattern}|{bool_value_pattern}|{single_quotes_value_pattern}"
multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>\d+(?:,\s*\d+)*|'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*'(?:,\s*'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*')*)\)""" # noqa: E501
re_field_value_pattern = rf"""regexp_like\({field_pattern},\s*'(?P<{ValueType.regex_value}>(?:[:a-zA-Z\*\?0-9=+%#№;\-_,"\.$&^@!\{{\}}\[\]\s?<>|]|\\\'|\\)+)'\)""" # noqa: E501
Expand All@@ -71,7 +71,8 @@ def get_operator_and_value(
return mapped_operator, bool_value

if (s_q_value := get_match_group(match, group_name=ValueType.single_quotes_value)) is not None:
return mapped_operator, self.str_value_manager.from_str_to_container(s_q_value)
escape_symbol = get_match_group(match, group_name=_ESCAPE_SYMBOL_GROUP_NAME)
return mapped_operator, self.str_value_manager.from_str_to_container(s_q_value, escape_symbol=escape_symbol)

return super().get_operator_and_value(match, mapped_operator, operator)

Expand All@@ -88,10 +89,6 @@ def _search_re_field_value(self, query: str) -> Optional[tuple[FieldValue, str]]
operator = Identifier(token_type=OperatorType.REGEX)
return self.create_field_value(field_name, operator, value), query[match.end() :]

def tokenize(self, query: str) -> list:
query = re.sub(r"\s*ESCAPE\s*'.'", "", query) # remove `ESCAPE 'escape_char'` in LIKE expr
return super().tokenize(query)

def _get_next_token(
self, query: str
) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,19 +16,21 @@
limitations under the License.
-----------------------------------------------------------------
"""
from typing import ClassVar
from typing import ClassVar, Optional

from app.translator.core.str_value_manager import (
BaseSpecSymbol,
ReDigitalSymbol,
ReWhiteSpaceSymbol,
ReWordSymbol,
SingleSymbolWildCard,
StrValue,
StrValueManager,
)
from app.translator.platforms.elasticsearch.escape_manager import ESQLQueryEscapeManager, esql_query_escape_manager


classESQLQueryStrValueManager(StrValueManager):
classESQLStrValueManager(StrValueManager):
escape_manager: ESQLQueryEscapeManager = esql_query_escape_manager
re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {
"w": ReWordSymbol,
Expand All@@ -37,4 +39,13 @@ class ESQLQueryStrValueManager(StrValueManager):
}


esql_query_str_value_manager = ESQLQueryStrValueManager()
class EQLStrValueManager(StrValueManager):
str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": SingleSymbolWildCard}

def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002
split = [self.str_spec_symbols_map[char]() if char in self.str_spec_symbols_map else char for char in value]
return StrValue(value, self._concat(split))


esql_str_value_manager = ESQLStrValueManager()
eql_str_value_manager = EQLStrValueManager()
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,6 +16,7 @@
limitations under the License.
-----------------------------------------------------------------
"""
from typing import Optional

from app.translator.core.str_value_manager import (
RE_STR_SPEC_SYMBOLS_MAP,
Expand All@@ -42,7 +43,7 @@ class SigmaStrValueManager(StrValueManager):
}
re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP

def from_str_to_container(self, value: str) -> StrValue:
def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002
split = []
prev_char = None
for char in value:
Expand Down
[8]ページ先頭
©2009-2025 Movatter.jp