Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

mapping flow changes, render unmapped fields to comment#173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
alexvolha merged 7 commits intomainfromgis-8193
Jul 16, 2024
Merged
Show file tree
Hide file tree
Changes from1 commit
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
PrevPrevious commit
NextNext commit
render unmapped fields comment
  • Loading branch information
@alexvolha
alexvolha committedJul 11, 2024
commit83c12c8436ff7c9b64dc0b1f4b7f723d9314a5c0
9 changes: 2 additions & 7 deletionsuncoder-core/app/translator/core/exceptions/core.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,19 +10,14 @@ class BasePlatformException(BaseException):


class StrictPlatformException(BasePlatformException):
field_name: str = None

def __init__(
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
):
def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None):
message = (
f"Platform {platform_name} has strict mapping. "
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
f"Source fields: {', '.join(fields)} have no mapping."
f" Mapping file: {mapping}."
if mapping
else ""
)
self.field_name = field_name
super().__init__(message)


Expand Down
15 changes: 8 additions & 7 deletionsuncoder-core/app/translator/core/mapping.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -158,17 +158,18 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
def default_mapping(self) -> SourceMapping:
return self._source_mappings[DEFAULT_MAPPING_NAME]

def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[Field]:
not_mapped = []
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
unmapped = []
for field in field_tokens:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
if not mapped_field:
if self.is_strict_mapping:
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
not_mapped.append(field)
if not mapped_field and field.source_name not in unmapped:
unmapped.append(field.source_name)

return not_mapped
if self.is_strict_mapping and unmapped:
raise StrictPlatformException(platform_name=self.details.name, fields=unmapped)

return unmapped

@staticmethod
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
Expand Down
20 changes: 7 additions & 13 deletionsuncoder-core/app/translator/core/render.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -207,10 +207,9 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:

return query

def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[Field]]) -> str:
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str:
if fields:
joined = ", ".join(field.source_name for field in fields)
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{joined}")
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}")
return query

def wrap_with_comment(self, value: str) -> str:
Expand DownExpand Up@@ -256,7 +255,7 @@ def generate_functions(self, functions: list[Function], source_mapping: SourceMa
def map_predefined_field(self, predefined_field: PredefinedField) -> str:
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
if self.mappings.is_strict_mapping:
raise StrictPlatformException(field_name=predefined_field.name,platform_name=self.details.name)
raise StrictPlatformException(platform_name=self.details.name,fields=[predefined_field.name])

return predefined_field.name

Expand DownExpand Up@@ -309,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->

def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
result_values = []
unmapped_fields = set()
for token in tokens:
try:
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
except StrictPlatformException as err:
unmapped_fields.add(err.field_name)
if unmapped_fields:
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))

return "".join(result_values)

def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str:
Expand DownExpand Up@@ -349,7 +343,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand DownExpand Up@@ -418,7 +412,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
generic_field_name=generic_field_name
)
if not mapped_field and self.mappings.is_strict_mapping:
raise StrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
raise StrictPlatformException(platform_name=self.details.name, fields=[field.source_name])
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
for prefix in prefix_list:
if prefix not in defined_raw_log_fields:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details
from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings
Expand DownExpand Up@@ -112,7 +111,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None, # ,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,7 +25,6 @@
from app.translator.core.mitre import MitreConfig
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.base.lucene.mapping import LuceneMappings
from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details
Expand DownExpand Up@@ -88,7 +87,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None,
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.base.lucene.mapping import LuceneMappings
from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_ALERT, elastalert_details
Expand DownExpand Up@@ -61,7 +60,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.base.lucene.mapping import LuceneMappings
from app.translator.platforms.elasticsearch.const import KIBANA_RULE, KIBANA_SEARCH_SOURCE_JSON, kibana_rule_details
Expand DownExpand Up@@ -57,7 +56,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.base.lucene.mapping import LuceneMappings
from app.translator.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details
Expand DownExpand Up@@ -57,7 +56,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None,
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.core.models.query_tokens.field_value import FieldValue
from app.translator.core.models.query_tokens.identifier import Identifier
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
Expand DownExpand Up@@ -304,7 +303,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
fields: Optional[set[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,7 +25,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details
from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager
Expand DownExpand Up@@ -66,7 +65,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None,
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details
from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_alert_mappings
Expand DownExpand Up@@ -53,7 +52,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,7 +25,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details
from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings
Expand DownExpand Up@@ -78,7 +77,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,7 +26,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.core.models.query_tokens.field_value import FieldValue
from app.translator.managers import render_manager
from app.translator.platforms.base.lucene.mapping import LuceneMappings
Expand DownExpand Up@@ -65,7 +64,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,18 +16,15 @@
limitations under the License.
-----------------------------------------------------------------
"""
from contextlib import suppress
from typing import ClassVar, Optional, Union

from app.translator.const import DEFAULT_VALUE_TYPE
from app.translator.core.const import QUERY_TOKEN_TYPE
from app.translator.core.context_vars import preset_log_source_str_ctx_var, return_only_first_query_ctx_var
from app.translator.core.context_vars import preset_log_source_str_ctx_var
from app.translator.core.custom_types.tokens import OperatorType
from app.translator.core.custom_types.values import ValueType
from app.translator.core.exceptions.core import StrictPlatformException
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import TokenizedQueryContainer
from app.translator.core.models.query_tokens.field_value import FieldValue
from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender
from app.translator.core.str_value_manager import StrValue
Expand DownExpand Up@@ -224,32 +221,3 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
@staticmethod
def _finalize_search_query(query: str) -> str:
return f"| filter {query}" if query else ""

def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
queries_map = {}
errors = []
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)

last_mapping_index = len(source_mappings) - 1
for index, source_mapping in enumerate(source_mappings):
try:
finalized_query = self._generate_from_tokenized_query_container_by_source_mapping(
query_container, source_mapping
)
if return_only_first_query_ctx_var.get() is True:
return finalized_query
queries_map[source_mapping.source_id] = finalized_query
except StrictPlatformException as err:
errors.append(err)
if index != last_mapping_index or source_mapping.source_id == DEFAULT_MAPPING_NAME or queries_map:
continue

with suppress(StrictPlatformException):
finalized_query = self._generate_from_tokenized_query_container_by_source_mapping(
query_container, self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME)
)
queries_map[source_mapping.source_id] = finalized_query

if not queries_map and errors:
raise errors[0]
return self.finalize(queries_map)
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,7 +23,6 @@
from app.translator.core.mapping import SourceMapping
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer
from app.translator.core.models.query_tokens.field import Field
from app.translator.managers import render_manager
from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details
from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_alert_mappings
Expand DownExpand Up@@ -63,7 +62,7 @@ def finalize_query(
meta_info: Optional[MetaInfoContainer] = None,
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
not_supported_functions: Optional[list] = None,
unmapped_fields: Optional[list[Field]] = None,
unmapped_fields: Optional[list[str]] = None,
*args, # noqa: ARG002
**kwargs, # noqa: ARG002
) -> str:
Expand Down
[8]ページ先頭
©2009-2025 Movatter.jp