Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

mapping flow changes, render unmapped fields to comment#173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
alexvolha merged 7 commits intomainfromgis-8193
Jul 16, 2024
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletionsuncoder-core/app/translator/core/exceptions/core.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,19 +10,14 @@ class BasePlatformException(BaseException):


class StrictPlatformException(BasePlatformException):
field_name: str = None

def __init__(
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
):
def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None):
message = (
f"Platform {platform_name} has strict mapping. "
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
f"Source fields: {', '.join(fields)} have no mapping."
f" Mapping file: {mapping}."
if mapping
else ""
)
self.field_name = field_name
super().__init__(message)


Expand Down
13 changes: 6 additions & 7 deletionsuncoder-core/app/translator/core/functions.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -94,17 +94,16 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe
def render(self, function: Function, source_mapping: SourceMapping) -> str:
raise NotImplementedError

@staticmethod
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
def map_field(self, field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
if isinstance(field, Alias):
return field.name

generic_field_name = field.get_generic_field_name(source_mapping.source_id)
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
if isinstance(mapped_field, list):
mapped_field = mapped_field[0]
if isinstance(field, Field):
mappings = self.manager.platform_functions.platform_query_render.mappings
mapped_fields = mappings.map_field(field, source_mapping)
return mapped_fields[0]

return mapped_field if mapped_field else field.source_name
raise NotSupportedFunctionException


class PlatformFunctionsManager:
Expand Down
40 changes: 38 additions & 2 deletionsuncoder-core/app/translator/core/mapping.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
from __future__ import annotations

from abc import ABC, abstractmethod
from typing import Optional, TypeVar
from typing importTYPE_CHECKING,Optional, TypeVar

from app.translator.core.exceptions.core import StrictPlatformException
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.mappings.utils.load_from_files import LoaderFileMappings

if TYPE_CHECKING:
from app.translator.core.models.query_tokens.field import Field


DEFAULT_MAPPING_NAME = "default"


Expand DownExpand Up@@ -85,12 +91,16 @@ def __init__(


class BasePlatformMappings:
details: PlatformDetails = None

is_strict_mapping: bool = False
skip_load_default_mappings: bool = True
extend_default_mapping_with_all_fields: bool = False

def __init__(self, platform_dir: str):
def __init__(self, platform_dir: str, platform_details: PlatformDetails):
self._loader = LoaderFileMappings()
self._platform_dir = platform_dir
self.details = platform_details
self._source_mappings = self.prepare_mapping()

def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None:
Expand DownExpand Up@@ -148,6 +158,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
def default_mapping(self) -> SourceMapping:
return self._source_mappings[DEFAULT_MAPPING_NAME]

def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
unmapped = []
for field in field_tokens:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
if not mapped_field and field.source_name not in unmapped:
unmapped.append(field.source_name)

if self.is_strict_mapping and unmapped:
raise StrictPlatformException(
platform_name=self.details.name, fields=unmapped, mapping=source_mapping.source_id
)

return unmapped

@staticmethod
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
# field can be mapped to corresponding platform field name or list of platform field names
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)

if isinstance(mapped_field, str):
mapped_field = [mapped_field]

return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]


class BaseCommonPlatformMappings(ABC, BasePlatformMappings):
def prepare_mapping(self) -> dict[str, SourceMapping]:
Expand Down
54 changes: 27 additions & 27 deletionsuncoder-core/app/translator/core/render.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -184,6 +184,7 @@ class QueryRender(ABC):
details: PlatformDetails = None
is_single_line_comment: bool = False
unsupported_functions_text = "Unsupported functions were excluded from the result query:"
unmapped_fields_text = "Unmapped fields: "

platform_functions: PlatformFunctions = None

Expand All@@ -206,6 +207,11 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:

return query

def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str:
if fields:
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}")
return query

def wrap_with_comment(self, value: str) -> str:
return f"{self.comment_symbol} {value}"

Expand All@@ -216,7 +222,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta

class PlatformQueryRender(QueryRender):
mappings: BasePlatformMappings = None
is_strict_mapping: bool = False

or_token = "or"
and_token = "and"
Expand DownExpand Up@@ -247,22 +252,10 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu
def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
return self.platform_functions.render(functions, source_mapping)

def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
# field can be mapped to corresponding platform field name or list of platform field names
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
if not mapped_field and self.is_strict_mapping:
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)

if isinstance(mapped_field, str):
mapped_field = [mapped_field]

return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]

def map_predefined_field(self, predefined_field: PredefinedField) -> str:
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
if self.is_strict_mapping:
raise StrictPlatformException(field_name=predefined_field.name,platform_name=self.details.name)
if self.mappings.is_strict_mapping:
raise StrictPlatformException(platform_name=self.details.name,fields=[predefined_field.name])

return predefined_field.name

Expand All@@ -275,7 +268,7 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
elif token.predefined_field:
mapped_fields = [self.map_predefined_field(token.predefined_field)]
else:
mapped_fields = self.map_field(token.field, source_mapping)
mapped_fields = self.mappings.map_field(token.field, source_mapping)
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
[
self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
Expand All@@ -285,9 +278,13 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
return self.group_token % joined if len(mapped_fields) > 1 else joined
if isinstance(token, FieldField):
alias_left, field_left = token.alias_left, token.field_left
mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping)
mapped_fields_left = (
[alias_left.name] if alias_left else self.mappings.map_field(field_left, source_mapping)
)
alias_right, field_right = token.alias_right, token.field_right
mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping)
mapped_fields_right = (
[alias_right.name] if alias_right else self.mappings.map_field(field_right, source_mapping)
)
cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right))
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
[
Expand All@@ -311,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->

def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
result_values = []
unmapped_fields = set()
for token in tokens:
try:
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
except StrictPlatformException as err:
unmapped_fields.add(err.field_name)
if unmapped_fields:
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))

return "".join(result_values)

def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str:
Expand DownExpand Up@@ -351,11 +343,13 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
query = self._join_query_parts(prefix, query, functions)
query = self.wrap_with_meta_info(query, meta_info)
query = self.wrap_with_unmapped_fields(query, unmapped_fields)
return self.wrap_with_not_supported_functions(query, not_supported_functions)

@staticmethod
Expand DownExpand Up@@ -417,8 +411,10 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
mapped_field = source_mapping.fields_mapping.get_platform_field_name(
generic_field_name=generic_field_name
)
if not mapped_field and self.is_strict_mapping:
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
if not mapped_field and self.mappings.is_strict_mapping:
raise StrictPlatformException(
platform_name=self.details.name, fields=[field.source_name], mapping=source_mapping.source_id
)
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
for prefix in prefix_list:
if prefix not in defined_raw_log_fields:
Expand All@@ -428,6 +424,9 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
def _generate_from_tokenized_query_container_by_source_mapping(
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
) -> str:
unmapped_fields = self.mappings.check_fields_mapping_existence(
query_container.meta_info.query_fields, source_mapping
)
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)

Expand All@@ -443,6 +442,7 @@ def _generate_from_tokenized_query_container_by_source_mapping(
query=query,
functions=rendered_functions.rendered,
not_supported_functions=not_supported_functions,
unmapped_fields=unmapped_fields,
meta_info=query_container.meta_info,
source_mapping=source_mapping,
)
Expand Down
2 changes: 1 addition & 1 deletionuncoder-core/app/translator/platforms/athena/const.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,4 +9,4 @@
"alt_platform_name": "OCSF",
}

athena_details = PlatformDetails(**ATHENA_QUERY_DETAILS)
athena_query_details = PlatformDetails(**ATHENA_QUERY_DETAILS)
3 changes: 2 additions & 1 deletionuncoder-core/app/translator/platforms/athena/mapping.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
from typing import Optional

from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
from app.translator.platforms.athena.const import athena_query_details


class AthenaLogSourceSignature(LogSourceSignature):
Expand DownExpand Up@@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s
return suitable_source_mappings


athena_mappings = AthenaMappings(platform_dir="athena")
athena_query_mappings = AthenaMappings(platform_dir="athena", platform_details=athena_query_details)
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,14 +18,14 @@

from app.translator.core.models.platform_details import PlatformDetails
from app.translator.managers import parser_manager
from app.translator.platforms.athena.const importathena_details
from app.translator.platforms.athena.mapping import AthenaMappings,athena_mappings
from app.translator.platforms.athena.const importathena_query_details
from app.translator.platforms.athena.mapping import AthenaMappings,athena_query_mappings
from app.translator.platforms.base.sql.parsers.sql import SqlQueryParser


@parser_manager.register_supported_by_roota
class AthenaQueryParser(SqlQueryParser):
details: PlatformDetails =athena_details
mappings: AthenaMappings =athena_mappings
details: PlatformDetails =athena_query_details
mappings: AthenaMappings =athena_query_mappings
query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s"
table_pattern = r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,19 +19,19 @@

from app.translator.core.models.platform_details import PlatformDetails
from app.translator.managers import render_manager
from app.translator.platforms.athena.const importathena_details
from app.translator.platforms.athena.mapping import AthenaMappings,athena_mappings
from app.translator.platforms.athena.const importathena_query_details
from app.translator.platforms.athena.mapping import AthenaMappings,athena_query_mappings
from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender


class AthenaFieldValueRender(SqlFieldValueRender):
details: PlatformDetails =athena_details
details: PlatformDetails =athena_query_details


@render_manager.register
class AthenaQueryRender(SqlQueryRender):
details: PlatformDetails =athena_details
mappings: AthenaMappings =athena_mappings
details: PlatformDetails =athena_query_details
mappings: AthenaMappings =athena_query_mappings

or_token = "OR"

Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,13 +20,13 @@
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.render_cti import RenderCTI
from app.translator.managers import render_cti_manager
from app.translator.platforms.athena.const importathena_details
from app.translator.platforms.athena.const importathena_query_details
from app.translator.platforms.athena.mappings.athena_cti import DEFAULT_ATHENA_MAPPING


@render_cti_manager.register
class AthenaCTI(RenderCTI):
details: PlatformDetails =athena_details
details: PlatformDetails =athena_query_details

field_value_template: str = "{key} = '{value}'"
or_operator: str = " OR "
Expand Down
3 changes: 0 additions & 3 deletionsuncoder-core/app/translator/platforms/base/aql/mapping.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,6 +90,3 @@ def get_suitable_source_mappings(
suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]]

return suitable_source_mappings


aql_mappings = AQLMappings(platform_dir="qradar")
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,14 +26,12 @@
from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, TABLE_GROUP_PATTERN
from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions
from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP
from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings
from app.translator.platforms.base.aql.tokenizer import AQLTokenizer
from app.translator.tools.utils import get_match_group


class AQLQueryParser(PlatformQueryParser):
tokenizer: AQLTokenizer = AQLTokenizer(aql_functions)
mappings: AQLMappings = aql_mappings
platform_functions: AQLFunctions = aql_functions

log_source_functions = ("LOGSOURCENAME", "LOGSOURCEGROUPNAME")
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,7 @@
from app.translator.core.custom_types.values import ValueType
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
from app.translator.core.str_value_manager import StrValue
from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings
from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature
from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager


Expand DownExpand Up@@ -121,8 +121,6 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:


class AQLQueryRender(PlatformQueryRender):
mappings: AQLMappings = aql_mappings

or_token = "OR"
and_token = "AND"
not_token = "NOT"
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
from app.translator.platforms.chronicle.const import chronicle_query_details, chronicle_rule_details


class ChronicleLogSourceSignature(LogSourceSignature):
Expand All@@ -10,6 +11,8 @@ def __str__(self) -> str:


class ChronicleMappings(BasePlatformMappings):
is_strict_mapping = True

def prepare_log_source_signature(self, mapping: dict) -> ChronicleLogSourceSignature:
...

Expand All@@ -28,4 +31,5 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap
return suitable_source_mappings


chronicle_mappings = ChronicleMappings(platform_dir="chronicle")
chronicle_query_mappings = ChronicleMappings(platform_dir="chronicle", platform_details=chronicle_query_details)
chronicle_rule_mappings = ChronicleMappings(platform_dir="chronicle", platform_details=chronicle_rule_details)
Loading
[8]ページ先頭
©2009-2025 Movatter.jp