Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

IOC`s fix mapping and hash types#8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
nazargesyk merged 1 commit intomainfromiocs_mapping_hash_fixes
Nov 14, 2023
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
DEFAULT_GRAYLOG_MAPPING = {
"SourceIP": "sourceAddress",
"DestinationIP": "destinationAddress",
"Domain": "destinationDnsDomain",
"URL": "requestUrl",
"HashMd5": "fileHash",
"HashSha1": "fileHash",
"HashSha256": "fileHash",
"HashSha512": "fileHash",
"SourceIP": "source.ip",
"DestinationIP": "destination.ip",
"Domain": "destination.domain",
"URL": "url.original",
"HashMd5": "file.hash.md5",
"HashSha1": "file.hash.sha1",
"HashSha256": "file.hash.sha256",
"HashSha512": "file.hash.sha512",
"Emails": "emails",
"Files": "filePath"
}
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
DEFAULT_MICROSOFT_SENTINEL_MAPPING = {
"DestinationIP": "DestinationIP",
"DestinationIP": "DestinationIp",
"SourceIP": "SourceIp",
"HashSha512": "FileHashSha512",
"HashSha256": "FileHashSha256",
"HashMd5": "FileHashMd5",
"Emails": "SenderFromAddress",
"Domain": "DestinationHostname",
"HashSha1": "FileHashSha1",
"Files": "FileName",
"Files": "TargetFileName",
"URL": "URL"
}
26 changes: 17 additions & 9 deletionssiem-converter/app/converter/core/parser_cti.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,23 +5,29 @@
from pydantic import BaseModel

from app.converter.core.exceptions.iocs import IocsLimitExceededException, EmptyIOCSException
from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN,\
hash_regexes, IOCType, HashType, IocParsingRule
from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN,\
hash_regexes, IOCType, HashType, IocParsingRule, HASH_MAP


class Iocs(BaseModel):
ip: list[str] = []
url: list[str] = []
domain: list[str] = []
hash: list[str] =[]
hash_dict: dict ={}

def get_total_count(self) -> int:
return len(self.ip) + len(self.url) + len(self.domain) + len(self.hash)
hash_len = 0
for value in self.hash_dict.values():
hash_len += len(value)
return len(self.ip) + len(self.url) + len(self.domain) + hash_len

def return_iocs(self) -> dict:
if all(not value for value in [self.ip, self.url, self.domain, self.hash]):
if all(not value for value in [self.ip, self.url, self.domain, self.hash_dict]):
raise EmptyIOCSException()
return {"ip": self.ip, "url": self.url, "domain": self.domain, "hash": self.hash}
result = {"DestinationIP": self.ip, "URL": self.url, "Domain": self.domain}
for key, value in self.hash_dict.items():
result[HASH_MAP[key]] = value
return result


class CTIParser:
Expand All@@ -47,7 +53,7 @@ def get_iocs_from_string(
if not include_hash_types:
include_hash_types = list(hash_regexes.keys())
for hash_type in include_hash_types:
iocs.hash.extend(self._find_all_str_by_regex(string, hash_regexes[hash_type]))
iocs.hash_dict[hash_type] =self._find_all_str_by_regex(string, hash_regexes[hash_type])
iocs = self.remove_duplicates(iocs)
iocs = self.remove_exceptions(iocs, exceptions)
if ioc_parsing_rules is None or "remove_private_and_reserved_ips" in ioc_parsing_rules:
Expand All@@ -69,14 +75,16 @@ def remove_duplicates(self, iocs):
iocs.ip = self._remove_duplicates_from_list(iocs.ip)
iocs.domain = self._remove_duplicates_from_list(iocs.domain)
iocs.url = self._remove_duplicates_from_list(iocs.url)
iocs.hash = self._remove_duplicates_from_list(iocs.hash)
for key, value in iocs.hash_dict.items():
iocs.hash_dict[key] = self._remove_duplicates_from_list(value)
return iocs

def remove_exceptions(self, iocs, exceptions=None):
iocs.ip = self._remove_exceptions(iocs.ip, exceptions)
iocs.domain = self._remove_exceptions(iocs.domain, exceptions)
iocs.url = self._remove_exceptions(iocs.url, exceptions)
iocs.hash = self._remove_exceptions(iocs.hash, exceptions)
for key, value in iocs.hash_dict.items():
iocs.hash_dict[key] = self._remove_exceptions(value, exceptions)
return iocs

@staticmethod
Expand Down
29 changes: 4 additions & 25 deletionssiem-converter/app/converter/core/render_cti.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,9 +34,6 @@ class RenderCTI:
final_result_for_one: str = "union * | where {result}\n"
default_mapping = None

def get_default_mapping(self, include_source_ip=False):
return self.prepare_mapping(self.default_mapping, include_source_ip)

def create_field_value(self, field: str, value: str, generic_field: str):
return self.data_map.format(key=field, value=value)

Expand All@@ -53,35 +50,17 @@ def render(self, data: List[List[IocsChunkValue]]) -> list[str]:
def collect_data_values(self, chunk):
data_values = []
key_chunk = []
processing_key = chunk[0].generic_field
processing_key = chunk[0].platform_field
for value in chunk:
if processing_key != value.generic_field:
if processing_key != value.platform_field:
data_values.append(self.or_group.format(or_group=self.or_operator.join(key_chunk),
processing_key=processing_key))
key_chunk = []
processing_key = value.generic_field
key_chunk.append(self.create_field_value(field=value.generic_field,
processing_key = value.platform_field
key_chunk.append(self.create_field_value(field=value.platform_field,
value=value.value,
generic_field=value.generic_field))
if key_chunk:
data_values.append(
self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key))
return data_values

def prepare_mapping(self, mapping: dict, include_source_ip: bool = False) -> dict:
m = {
"DestinationIP": "ip",
"Domain": "domain",
"URL": "url",
"HashMd5": "hash",
"HashSha1": "hash",
"HashSha256": "hash",
"HashSha512": "hash",
}
if include_source_ip:
m["SourceIP"] = "ip"
res = {}
for key, new_field_name in mapping.items():
if key in m:
res[new_field_name] = m[key]
return res
22 changes: 8 additions & 14 deletionssiem-converter/app/converter/cti_converter.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,14 +17,8 @@ def __init__(self):
self.logger = logging.getLogger("cti_converter")
self.parser = CTIParser()

def _reverse_mapping(self, platform: CTIPlatform, data: Dict[str, List[str]],
include_source_ip: bool = False) -> Dict[str, str]:
mapping: Dict = self.renders.get(platform.name).get_default_mapping(include_source_ip)
reverse_mapping = {}
for platform_field, generic_field in mapping.items():
if data.get(generic_field):
reverse_mapping.update({generic_field: platform_field})
return reverse_mapping
def _get_render_mapping(self, platform: CTIPlatform, include_source_ip: bool = False) -> Dict[str, str]:
return self.renders.get(platform.name).default_mapping

@handle_translation_exceptions
def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, include_hash_types: list = None,
Expand All@@ -39,11 +33,10 @@ def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, in
@handle_translation_exceptions
def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int,
include_source_ip: bool = False) -> List[str]:
reverse_mapping = self._reverse_mapping(data=parsed_data,
include_source_ip=include_source_ip, platform=platform_data)
mapping = self._get_render_mapping(platform=platform_data, include_source_ip=include_source_ip)
platform = self.renders.get(platform_data.name)
platform_generation = self.generate(data=parsed_data, platform=platform, iocs_per_query=iocs_per_query,
mapping=reverse_mapping)
mapping=mapping)
return platform_generation

def convert(self, text: str,
Expand DownExpand Up@@ -73,9 +66,10 @@ def _get_iocs_chunk(chunks_size: int, data: Dict[str, List[str]],
result = []
for generic_field, iocs_list in data.items():
for ioc in iocs_list:
result.append(IocsChunkValue(generic_field=generic_field,
platform_field=mapping.get(generic_field, generic_field),
value=ioc))
if mapping.get(generic_field):
result.append(IocsChunkValue(generic_field=generic_field,
platform_field=mapping[generic_field],
value=ioc))
return [result[i:i + chunks_size] for i in range(0, len(result), chunks_size)]

def generate(self, platform: RenderCTI, iocs_per_query, data: Dict[str, List[str]],
Expand Down
7 changes: 7 additions & 0 deletionssiem-converter/app/converter/tools/const.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,13 @@
"replace_dots", "remove_private_and_reserved_ips", "replace_hxxp"
]

HASH_MAP = {
"md5": "HashMd5",
"sha1": "HashSha1",
"sha256": "HashSha256",
"sha512": "HashSha512"
}

hash_regexes = {
"md5": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)",
"sha1": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)",
Expand Down
[8]ページ先頭
©2009-2025 Movatter.jp