Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Gis 8825 added sentinel one power query render#205

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
nazargesyk merged 12 commits intomainfromgis-8825
Dec 17, 2024
Merged
Show file tree
Hide file tree
Changes from1 commit
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
PrevPrevious commit
NextNext commit
Merge branch 'prod' into gis-8825
  • Loading branch information
@nazargesyk
nazargesyk committedOct 17, 2024
commit9231d12f85e01901e74a208b4b26f3780ddf98f5
88 changes: 88 additions & 0 deletionsuncoder-core/app/routers/meta_info.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
from dataclasses import asdict

from fastapi import APIRouter, Body, HTTPException

from app.models.meta_info import (
MetaInfo,
MetaInfoResponse,
MitreInfoContainer,
MitreTacticContainer,
MitreTechniqueContainer,
ParsedLogSources,
RawMetaInfo,
)
from app.translator.core.exceptions.core import UnsupportedPlatform
from app.translator.translator import app_translator

meta_info_router = APIRouter()


@meta_info_router.post("/get_meta_info/", tags=["meta_info"], description="Get Rule MetaInfo")
@meta_info_router.post("/get_meta_info/", include_in_schema=False)
def get_meta_info_data(
source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
) -> MetaInfoResponse:
try:
logsources, raw_query_container = app_translator.parse_meta_info(text=text, source=source_platform_id)
except UnsupportedPlatform as exc:
raise HTTPException(status_code=400, detail="Unsuported platform") from exc
except Exception as exc:
raise HTTPException(status_code=400, detail="Unexpected error.") from exc
if not raw_query_container:
raise HTTPException(status_code=400, detail="Can't parse metadata")
most_frequent_product = max(logsources.get("product"), key=logsources.get("product").get, default=None)
most_frequent_service = max(logsources.get("service"), key=logsources.get("service").get, default=None)
most_frequent_category = max(logsources.get("category"), key=logsources.get("category").get, default=None)

logsources.get("product", {}).pop(most_frequent_product, None)
logsources.get("service", {}).pop(most_frequent_service, None)
logsources.get("category", {}).pop(most_frequent_category, None)

parsed_logsources = ParsedLogSources(
most_frequent_product=most_frequent_product,
most_frequent_service=most_frequent_service,
most_frequent_category=most_frequent_category,
least_frequent_products=list(logsources.get("product", {}).keys()),
least_frequent_services=list(logsources.get("service", {}).keys()),
least_frequent_categories=list(logsources.get("category", {}).keys()),
)
return MetaInfoResponse(
query=raw_query_container.query,
language=raw_query_container.language,
meta_info=MetaInfo(
id_=raw_query_container.meta_info.id,
title=raw_query_container.meta_info.title,
description=raw_query_container.meta_info.description,
author=raw_query_container.meta_info.author,
date=raw_query_container.meta_info.date,
false_positives=raw_query_container.meta_info.false_positives,
license_=raw_query_container.meta_info.license,
mitre_attack=MitreInfoContainer(
tactics=[
MitreTacticContainer(**asdict(tactic_container))
for tactic_container in raw_query_container.meta_info.mitre_attack.tactics
],
techniques=[
MitreTechniqueContainer(**asdict(tactic_container))
for tactic_container in raw_query_container.meta_info.mitre_attack.techniques
],
),
output_table_fields=raw_query_container.meta_info.output_table_fields,
parsed_log_sources=parsed_logsources,
query_fields=raw_query_container.meta_info.query_fields + raw_query_container.meta_info.function_fields,
query_period=raw_query_container.meta_info.query_period,
raw_metainfo_container=RawMetaInfo(
trigger_operator=raw_query_container.meta_info.raw_metainfo_container.trigger_operator,
trigger_threshold=raw_query_container.meta_info.raw_metainfo_container.trigger_threshold,
query_frequency=raw_query_container.meta_info.raw_metainfo_container.query_frequency,
query_period=raw_query_container.meta_info.raw_metainfo_container.query_period,
),
raw_mitre_attack=raw_query_container.meta_info.raw_mitre_attack,
references=raw_query_container.meta_info.references,
severity=raw_query_container.meta_info.severity,
source_mapping_ids=raw_query_container.meta_info.source_mapping_ids,
status=raw_query_container.meta_info.status,
tags=raw_query_container.meta_info.tags,
timeframe=raw_query_container.meta_info.timeframe,
),
)
53 changes: 43 additions & 10 deletionsuncoder-core/app/translator/core/mapping.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,12 @@ class LogSourceSignature(ABC):
def is_suitable(self, **kwargs) -> bool:
raise NotImplementedError("Abstract method")

def is_probably_suitable(self, **kwargs) -> bool:
"""
Performs check with more options, but the result is less accurate than the "is_suitable" method
"""
raise NotImplementedError("Abstract method")

@staticmethod
def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
conditions = [condition for condition in conditions if condition is not None]
Expand DownExpand Up@@ -88,11 +94,13 @@ def __init__(
log_source_signature: _LogSourceSignatureType = None,
fields_mapping: Optional[FieldsMapping] = None,
raw_log_fields: Optional[dict] = None,
conditions: Optional[dict] = None,
):
self.source_id = source_id
self.log_source_signature = log_source_signature
self.fields_mapping = fields_mapping or FieldsMapping([])
self.raw_log_fields = raw_log_fields
self.conditions = conditions


class BasePlatformMappings:
Expand DownExpand Up@@ -123,6 +131,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:

field_mappings_dict = mapping_dict.get("field_mapping", {})
raw_log_fields = mapping_dict.get("raw_log_fields", {})
conditions = mapping_dict.get("conditions", {})
field_mappings_dict.update({field: field for field in raw_log_fields})
fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)
Expand All@@ -131,6 +140,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
log_source_signature=log_source_signature,
fields_mapping=fields_mapping,
raw_log_fields=raw_log_fields,
conditions=conditions,
)

if self.skip_load_default_mappings:
Expand DownExpand Up@@ -170,31 +180,47 @@ def get_source_mappings_by_fields_and_log_sources(

return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]

def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
return self._source_mappings.get(source_id)

def get_source_mappings_by_ids(
self, source_mapping_ids: list[str], return_default: bool = True
) -> list[SourceMapping]:
source_mappings = []
for source_mapping_id in source_mapping_ids:
if source_mapping_id == DEFAULT_MAPPING_NAME:
continue
if source_mapping := self.get_source_mapping(source_mapping_id):
source_mappings.append(source_mapping)

if not source_mappings:
if not source_mappings and return_default:
source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)]

return source_mappings

defget_source_mapping(self,source_id: str) -> Optional[SourceMapping]:
return self._source_mappings.get(source_id)
defget_source_mappings_by_log_sources(self,log_sources: dict) -> Optional[list[str]]:
raise NotImplementedError("Abstract method")

@property
def default_mapping(self) -> SourceMapping:
return self._source_mappings[DEFAULT_MAPPING_NAME]

def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
def check_fields_mapping_existence(
self,
query_field_tokens: list[Field],
function_field_tokens_map: dict[str, list[Field]],
supported_func_render_names: set[str],
source_mapping: SourceMapping,
) -> list[str]:
unmapped = []
for field in field_tokens:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
if not mapped_field and field.source_name not in unmapped:
unmapped.append(field.source_name)

for field in query_field_tokens:
self._check_field_mapping_existence(field, source_mapping, unmapped)

for func_name, function_field_tokens in function_field_tokens_map.items():
if func_name in supported_func_render_names:
for field in function_field_tokens:
self._check_field_mapping_existence(field, source_mapping, unmapped)

if self.is_strict_mapping and unmapped:
raise StrictPlatformException(
Expand All@@ -203,6 +229,13 @@ def check_fields_mapping_existence(self, field_tokens: list[Field], source_mappi

return unmapped

@staticmethod
def _check_field_mapping_existence(field: Field, source_mapping: SourceMapping, unmapped: list[str]) -> None:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
if not mapped_field and field.source_name not in unmapped:
unmapped.append(field.source_name)

@staticmethod
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
Expand Down
2 changes: 1 addition & 1 deletionuncoder-core/app/translator/core/mixins/rule.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,7 +42,7 @@ def parse_mitre_attack(self, tags: list[str]) -> MitreInfoContainer:
tag = tag.lower()
if tag.startswith("attack."):
tag = tag[7::]
if tag.startswith("t"):
if tag[-1].isdigit():
parsed_techniques.append(tag)
else:
parsed_tactics.append(tag)
Expand Down
10 changes: 7 additions & 3 deletionsuncoder-core/app/translator/core/models/query_container.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -65,6 +65,8 @@ def __init__(
date: Optional[str] = None,
output_table_fields: Optional[list[Field]] = None,
query_fields: Optional[list[Field]] = None,
function_fields: Optional[list[Field]] = None,
function_fields_map: Optional[dict[str, list[Field]]] = None,
license_: Optional[str] = None,
severity: Optional[str] = None,
references: Optional[list[str]] = None,
Expand All@@ -76,7 +78,7 @@ def __init__(
parsed_logsources: Optional[dict] = None,
timeframe: Optional[timedelta] = None,
query_period: Optional[timedelta] = None,
mitre_attack: MitreInfoContainer =MitreInfoContainer(),
mitre_attack:Optional[MitreInfoContainer] =None,
raw_metainfo_container: Optional[RawMetaInfoContainer] = None,
) -> None:
self.id = id_ or str(uuid.uuid4())
Expand All@@ -90,19 +92,21 @@ def __init__(
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
self.output_table_fields = output_table_fields or []
self.query_fields = query_fields or []
self.function_fields = function_fields or []
self.function_fields_map = function_fields_map or {}
self.license = license_ or "DRL 1.1"
self.severity = severity or SeverityType.low
self.references = references or []
self.tags = tags or []
self.mitre_attack = mitre_attack orNone
self.mitre_attack = mitre_attack orMitreInfoContainer()
self.raw_mitre_attack = raw_mitre_attack or []
self.status = status or "stable"
self.false_positives = false_positives or []
self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME]
self.parsed_logsources = parsed_logsources or {}
self.timeframe = timeframe
self.query_period = query_period
self.raw_metainfo_container = raw_metainfo_container
self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()

@property
def author_str(self) -> str:
Expand Down
25 changes: 18 additions & 7 deletionsuncoder-core/app/translator/core/parser.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,7 +24,7 @@
from app.translator.core.exceptions.parser import TokenizerGeneralException
from app.translator.core.functions import PlatformFunctions
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
from app.translator.core.models.functions.base import Function
from app.translator.core.models.functions.base import Function, ParsedFunctions
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
from app.translator.core.models.query_tokens.field import Field
Expand All@@ -51,6 +51,9 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer:
def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
raise NotImplementedError("Abstract method")

def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]:
raise NotImplementedError("Abstract method")


class PlatformQueryParser(QueryParser, ABC):
mappings: BasePlatformMappings = None
Expand All@@ -65,16 +68,19 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
@staticmethod
def get_field_tokens(
query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
) -> list[Field]:
field_tokens = []
) -> tuple[list[Field], list[Field], dict[str, list[Field]]]:
query_field_tokens = []
function_field_tokens = []
function_field_tokens_map = {}
for token in query_tokens:
if isinstance(token, (FieldField, FieldValue, FunctionValue)):
field_tokens.extend(token.fields)
query_field_tokens.extend(token.fields)

if functions:
field_tokens.extend([field for func in functions for field in func.fields])
for func in functions or []:
function_field_tokens.extend(func.fields)
function_field_tokens_map[func.name] = func.fields

returnfield_tokens
returnquery_field_tokens, function_field_tokens, function_field_tokens_map

def get_source_mappings(
self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]]
Expand All@@ -85,3 +91,8 @@ def get_source_mappings(
)
self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
return source_mappings

def get_source_mapping_ids_by_logsources(self, query: str) -> Optional[list[str]]:
_, parsed_logsources, _ = self._parse_query(query=query)
if parsed_logsources:
return self.mappings.get_source_mappings_by_log_sources(parsed_logsources)
8 changes: 6 additions & 2 deletionsuncoder-core/app/translator/core/render.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -428,14 +428,18 @@ def _generate_from_tokenized_query_container_by_source_mapping(
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
) -> str:
unmapped_fields = self.mappings.check_fields_mapping_existence(
query_container.meta_info.query_fields, source_mapping
query_container.meta_info.query_fields,
query_container.meta_info.function_fields_map,
self.platform_functions.manager.supported_render_names,
source_mapping,
)
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)

if source_mapping.raw_log_fields:
defined_raw_log_fields = self.generate_raw_log_fields(
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields,
source_mapping=source_mapping,
)
prefix += f"\n{defined_raw_log_fields}"
query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
platform: CarbonBlack
source: default
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
platform: CarbonBlack
source: linux_dns_query


field_mapping:
User:
- childproc_username
- process_username
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
platform: CarbonBlack
source: linux_network_connection


field_mapping:
DestinationHostname:
- netconn_domain
- netconn_proxy_domain
DestinationPort: netconn_port
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
platform: CarbonBlack
source: macos_dns_query


field_mapping:
User:
- childproc_username
- process_username
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
platform: CarbonBlack
source: macos_network_connection


field_mapping:
DestinationHostname:
- netconn_domain
- netconn_proxy_domain
DestinationPort: netconn_port
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
platform: CarbonBlack
source: windows_create_remote_thread


field_mapping:
SourceImage: parent_name
StartModule: modload_name
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
platform: CarbonBlack
source: windows_dns_query


field_mapping:
User:
- childproc_username
- process_username
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
platform: CarbonBlack
source: windows_file_event


field_mapping:
User:
- childproc_username
- process_username
Loading
[8]ページ先頭
©2009-2025 Movatter.jp