Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0236ff1

Browse files
committed
gis-9379 add new endpoint /iocs/generate
1 parentc868b92 commit0236ff1

File tree

7 files changed

+209
-10
lines changed

7 files changed

+209
-10
lines changed

‎uncoder-core/app/routers/ioc_translate.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,10 @@
44

55
fromapp.models.ioc_translationimportCTIPlatform,OneTranslationCTIData
66
fromapp.models.translationimportInfoMessage
7-
fromapp.translator.cti_translatorimportCTITranslator
7+
fromapp.translator.cti_translatorimportcti_translator
88
fromapp.translator.tools.constimportHashType,IocParsingRule,IOCType
99

1010
iocs_router=APIRouter()
11-
cti_translator=CTITranslator()
1211

1312

1413
@iocs_router.post("/iocs/translate",description="Parse IOCs from text.")
@@ -46,3 +45,32 @@ def parse_and_translate_iocs(
4645

4746
info_message=InfoMessage(message=translations,severity="error")
4847
returnOneTranslationCTIData(info=info_message,status=status,target_platform_id=platform.id)
48+
49+
50+
@iocs_router.post("/iocs/generate",description="Parse IOCs from text and based on input data generate translation")
51+
@iocs_router.post("/iocs/generate",include_in_schema=False)
52+
defparse_iocs_and_generate_rule(
53+
text:str=Body(...,description="Text to parse IOCs from",embed=True),
54+
platform:CTIPlatform=Body(...,description="Platform to parse IOCs to",embed=True),
55+
iocs_per_query:int=Body(25,description="IOCs per query limit",embed=True),
56+
title:str=Body(...,description="Title",embed=True),
57+
description:str=Body(...,description="Description",embed=True),
58+
references:list[str]=Body(...,description="References",embed=True),
59+
created_date:str=Body(...,description="Rule created date",embed=True),
60+
mitre_tags:Optional[list[str]]=Body(...,description="Mitra tactics and techniques",embed=True),
61+
)->OneTranslationCTIData:
62+
status,translations=cti_translator.generate(
63+
title=title,
64+
text=text,
65+
platform_data=platform,
66+
description=description,
67+
references=references,
68+
created_date=created_date,
69+
mitre_tags=mitre_tags,
70+
iocs_per_query=iocs_per_query,
71+
)
72+
ifstatus:
73+
returnOneTranslationCTIData(status=status,translations=translations,target_platform_id=platform.id)
74+
75+
info_message=InfoMessage(message=translations,severity="error")
76+
returnOneTranslationCTIData(info=info_message,status=status,target_platform_id=platform.id)
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
fromapp.translator.core.constimportQUERY_TOKEN_TYPE
2+
fromapp.translator.core.custom_types.tokensimportLogicalOperatorType,OperatorType
3+
fromapp.translator.core.mappingimportSourceMapping
4+
fromapp.translator.core.models.query_tokens.field_valueimportFieldValue
5+
fromapp.translator.core.models.query_tokens.identifierimportIdentifier
6+
7+
8+
classExtraConditionMixin:
9+
defgenerate_extra_conditions(self,source_mapping:SourceMapping)->list[QUERY_TOKEN_TYPE]:
10+
extra_tokens= []
11+
forfield,valueinsource_mapping.conditions.items():
12+
extra_tokens.extend(
13+
[
14+
FieldValue(source_name=field,operator=Identifier(token_type=OperatorType.EQ),value=value),
15+
Identifier(token_type=LogicalOperatorType.AND),
16+
]
17+
)
18+
returnextra_tokens

‎uncoder-core/app/translator/core/render_cti.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
fromabcimportabstractmethod
1920

2021
fromapp.translator.core.models.iocsimportIocsChunkValue
2122
fromapp.translator.core.models.platform_detailsimportPlatformDetails
@@ -46,6 +47,10 @@ def render(self, data: list[list[IocsChunkValue]]) -> list[str]:
4647
final_result.append(self.final_result_for_one.format(result=data_values[0]))
4748
returnfinal_result
4849

50+
@abstractmethod
51+
defgenerate(self,data:dict[str,list[list[IocsChunkValue]]],**kwargs)->list[str]:
52+
raiseNotImplementedError("Abstract method")
53+
4954
defcollect_data_values(self,chunk:list[IocsChunkValue])->list[str]:
5055
data_values= []
5156
key_chunk= []

‎uncoder-core/app/translator/cti_translator.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
fromapp.translator.core.models.iocsimportIocsChunkValue
77
fromapp.translator.core.parser_ctiimportCTIParser
88
fromapp.translator.managersimportRenderCTIManager,render_cti_manager
9+
fromapp.translator.tools.constimportDefaultHashType,DefaultIocParsingRule,DefaultIOCType,iocs_types_map
910
fromapp.translator.tools.decoratorsimporthandle_translation_exceptions
1011

1112

@@ -45,6 +46,31 @@ def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, io
4546
)
4647
returnrender_cti.render(chunked_iocs)
4748

49+
def__sort_iocs_by_type(self,parsed_data:dict)->dict:
50+
result= {}
51+
forkey,valuesiniocs_types_map.items():
52+
ifnotresult.get(key):
53+
result[key]= {}
54+
forgeneric_field,iocs_listinparsed_data.items():
55+
ifgeneric_fieldinvalues:
56+
result[key][generic_field]=iocs_list
57+
returnresult
58+
59+
@handle_translation_exceptions
60+
def__generate_translation(
61+
self,parsed_data:dict,platform_data:CTIPlatform,iocs_per_query:int,**kwargs
62+
)->list[str]:
63+
render_cti=self.render_manager.get(platform_data.id)
64+
65+
sorted_data=self.__sort_iocs_by_type(parsed_data)
66+
chunked_iocs= {}
67+
forkey,chunkinsorted_data.items():
68+
ifioc_chuck:=self.__get_iocs_chunk(
69+
chunks_size=iocs_per_query,data=chunk,mapping=render_cti.default_mapping
70+
):
71+
chunked_iocs[key]=ioc_chuck
72+
returnrender_cti.generate(chunked_iocs,**kwargs)
73+
4874
deftranslate(
4975
self,
5076
text:str,
@@ -70,6 +96,37 @@ def translate(
7096
)
7197
returnstatus,parsed_data
7298

99+
defgenerate(
100+
self,
101+
text:str,
102+
title:str,
103+
description:str,
104+
references:list[str],
105+
created_date:str,
106+
mitre_tags:Optional[list[str]],
107+
platform_data:CTIPlatform,
108+
iocs_per_query:int=CTI_IOCS_PER_QUERY_LIMIT,
109+
)-> (bool,list[str]):
110+
status,parsed_data=self.__parse_iocs_from_string(
111+
text=text,
112+
include_ioc_types=DefaultIOCType,
113+
include_hash_types=DefaultHashType,
114+
ioc_parsing_rules=DefaultIocParsingRule,
115+
include_source_ip=True,
116+
)
117+
ifstatus:
118+
kwargs= {
119+
"title":title,
120+
"description":description,
121+
"references":references,
122+
"created_date":created_date,
123+
"mitre_tags":mitre_tags,
124+
}
125+
returnself.__generate_translation(
126+
parsed_data=parsed_data,platform_data=platform_data,iocs_per_query=iocs_per_query,**kwargs
127+
)
128+
returnstatus,parsed_data
129+
73130
@staticmethod
74131
def__get_iocs_chunk(
75132
chunks_size:int,data:dict[str,list[str]],mapping:dict[str,str]
@@ -86,3 +143,6 @@ def __get_iocs_chunk(
86143
@classmethod
87144
defget_renders(cls)->list:
88145
returncls.render_manager.get_platforms_details
146+
147+
148+
cti_translator=CTITranslator()

‎uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
fromapp.translator.core.models.platform_detailsimportPlatformDetails
22
fromapp.translator.core.render_ctiimportRenderCTI
33
fromapp.translator.managersimportrender_cti_manager
4-
fromapp.translator.platforms.arcsight.constimportARCSIGHT_QUERY_DETAILS
5-
fromapp.translator.platforms.arcsight.mappings.arcsight_ctiimportDEFAULT_ARCSIGHT_MAPPING
4+
fromapp.translator.platforms.arcsight.constimportDEFAULT_ARCSIGHT_CTI_MAPPING,arcsight_query_details
65

76

87
@render_cti_manager.register
98
classArcsightKeyword(RenderCTI):
10-
details:PlatformDetails=PlatformDetails(**ARCSIGHT_QUERY_DETAILS)
9+
details:PlatformDetails=arcsight_query_details
1110

12-
default_mapping=DEFAULT_ARCSIGHT_MAPPING
11+
default_mapping=DEFAULT_ARCSIGHT_CTI_MAPPING
1312
field_value_template:str="{key} = {value}"
1413
or_operator:str=" OR "
1514
group_or_operator:str=" OR "
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
importuuid
2+
importyaml
3+
4+
fromapp.translator.core.custom_types.meta_infoimportSeverityType
5+
fromapp.translator.core.models.iocsimportIocsChunkValue
6+
fromapp.translator.core.models.platform_detailsimportPlatformDetails
7+
fromapp.translator.core.render_ctiimportRenderCTI
8+
fromapp.translator.managersimportrender_cti_manager
9+
fromapp.translator.platforms.sigma.constimportsigma_rule_details,DEFAULT_SIGMA_CTI_MAPPING
10+
fromapp.translator.tools.constimportLOGSOURCE_MAP
11+
12+
13+
@render_cti_manager.register
14+
classSigmaRenderCTI(RenderCTI):
15+
details:PlatformDetails=sigma_rule_details
16+
default_mapping=DEFAULT_SIGMA_CTI_MAPPING
17+
18+
defrender(self,data:list[list[IocsChunkValue]])->list[str]:
19+
final_result= []
20+
foriocs_chunkindata:
21+
data_values=self.collect_sigma_data_values(iocs_chunk)
22+
rule= {
23+
"title":"Sigma automatically generated based on IOCs",
24+
"id":uuid.uuid4().__str__(),
25+
"description":"Detects suspicious activity based on IOCs.",
26+
"status":"experimental",
27+
"author":"SOC Prime",
28+
"logsource": {"product":"windows"},
29+
"fields":list(data_values.keys()),
30+
"detection": {"selection":data_values,"condition":"selection"},
31+
"level":SeverityType.low,
32+
"falsepositives":"",
33+
}
34+
final_result.append(yaml.dump(rule,default_flow_style=False,sort_keys=False))
35+
returnfinal_result
36+
37+
defcollect_sigma_data_values(self,chunk:list[IocsChunkValue])->dict:
38+
raw_data_values= {}
39+
forvalueinchunk:
40+
ifvalue.platform_fieldinraw_data_values.keys():
41+
raw_data_values[value.platform_field].append(value.value)
42+
else:
43+
raw_data_values[value.platform_field]= [value.value]
44+
returnraw_data_values
45+
46+
defgenerate(self,data:dict[list[list[IocsChunkValue]]],**kwargs):
47+
final_result= []
48+
forkey,iocs_chunksindata.items():
49+
foriocs_chunkiniocs_chunks:
50+
data_values=self.collect_sigma_data_values(iocs_chunk)
51+
rule= {
52+
"title":f"IOCs ({key}) to detect:{kwargs['title']}",
53+
"id":uuid.uuid4().__str__(),
54+
"description":kwargs["description"],
55+
"status":"stable",
56+
"author":"SOC Prime Team",
57+
"logsource":LOGSOURCE_MAP.get(key),
58+
"fields":list(data_values.keys()),
59+
"detection": {"selection":data_values,"condition":"selection"},
60+
"level":SeverityType.medium,
61+
"falsepositives":"",
62+
"references":kwargs["references"],
63+
"date":kwargs["created_date"],
64+
"modified":kwargs["created_date"],
65+
}
66+
ifkwargs.get("mitre_tags"):
67+
rule["tags"]=kwargs["mitre_tags"]
68+
final_result.append(yaml.dump(rule,default_flow_style=False,sort_keys=False))
69+
returnfinal_result

‎uncoder-core/app/translator/tools/const.py‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,35 @@
1-
fromtypingimportLiteral
1+
importtyping
22

33
IP_IOC_REGEXP_PATTERN=r"(?:^|[ \/\[(\"',;>|])((?:25[0-5]|2[0-4]\d|[0-1]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d{1,2})){3})(?=[\s)\]\"',;:\/?\n<|]|$)"# noqa: E501
44
DOMAIN_IOC_REGEXP_PATTERN=r"(?:^|[\s\/\[\]@(\"',;{>|])(?:(?:http[s]?|ftp):\/\/?)?([^:\\\/\s({\[\]@\"'`,]+\.[a-zA-Z]+)(?:(?:(?:[/|:]\w+)*\/)(?:[\w\-.]+[^#?\s]+)?(?:[\w/\-&?=%.#]+(?:\(\))?)?)?(?=[\s)\]\"',;<|]|$)"# noqa: E501
55
URL_IOC_REGEXP_PATTERN=r"(?:^|[\s\/\[\]@(\"',;{>|])((?:(?:http[s]?|ftp):\/\/?)+(?:[^:\\\/\s({\[\]@\"'`,]+\.[a-zA-Z0-9]+)(?:(?:(?:[/|:]\w+)*\/)(?:[\w\-.]+[^#?\s<']+)?(?:[\w/\-&?=%.#]+(?:\(\))?)?)?)(?=[\s)\]\"',;<|]|$)"# noqa: E501
66

7-
IOCType=Literal["ip","domain","url","hash"]
8-
HashType=Literal["md5","sha1","sha256","sha512"]
9-
IocParsingRule=Literal["replace_dots","remove_private_and_reserved_ips","replace_hxxp"]
7+
IOCType=typing.Literal["ip","domain","url","hash"]
8+
HashType=typing.Literal["md5","sha1","sha256","sha512"]
9+
IocParsingRule=typing.Literal["replace_dots","remove_private_and_reserved_ips","replace_hxxp"]
10+
11+
DefaultIOCType=list(typing.get_args(IOCType))
12+
DefaultHashType=list(typing.get_args(HashType))
13+
DefaultIocParsingRule=list(typing.get_args(IocParsingRule))
1014

1115
HASH_MAP= {"md5":"HashMd5","sha1":"HashSha1","sha256":"HashSha256","sha512":"HashSha512"}
1216

17+
iocs_types_map= {
18+
"url": ["URL"],
19+
"domain": ["Domain"],
20+
"ip": ["DestinationIP","SourceIP"],
21+
"hash": ["HashMd5","HashSha1","HashSha256","HashSha512"],
22+
}
23+
24+
LOGSOURCE_MAP= {
25+
"hash": {"category":"process_creation"},
26+
"domain": {"category":"proxy"},
27+
"url": {"category":"proxy"},
28+
"ip": {"category":"proxy"},
29+
"emails": {"category":"mail"},
30+
"files": {"category":"file_event"},
31+
}
32+
1333
hash_regexes= {
1434
"md5":r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)",
1535
"sha1":r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)",

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp