Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commit6f1173b

Browse files
committed
merge main
2 parents9a9f093 +8b77550 commit6f1173b

File tree

26 files changed

+221
-151
lines changed

26 files changed

+221
-151
lines changed

‎uncoder-core/app/routers/translate.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
fromfastapiimportAPIRouter,Body
22

33
fromapp.models.translationimportInfoMessage,OneTranslationData,Platform,TranslatorPlatforms
4+
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var
45
fromapp.translator.cti_translatorimportCTITranslator
56
fromapp.translator.translatorimportTranslator
67

@@ -15,7 +16,9 @@ def translate_one(
1516
source_platform_id:str=Body(...,embed=True),
1617
target_platform_id:str=Body(...,embed=True),
1718
text:str=Body(...,embed=True),
19+
return_only_first_query:bool=False,
1820
)->OneTranslationData:
21+
return_only_first_query_ctx_var.set(return_only_first_query)
1922
status,data=translator.translate_one(text=text,source=source_platform_id,target=target_platform_id)
2023
ifstatus:
2124
returnOneTranslationData(status=status,translation=data,target_platform_id=target_platform_id)
@@ -27,8 +30,11 @@ def translate_one(
2730
@st_router.post("/translate/all",tags=["translator"],description="Generate all translations")
2831
@st_router.post("/translate/all/",include_in_schema=False)
2932
deftranslate_all(
30-
source_platform_id:str=Body(...,embed=True),text:str=Body(...,embed=True)
33+
source_platform_id:str=Body(...,embed=True),
34+
text:str=Body(...,embed=True),
35+
return_only_first_query:bool=False,
3136
)->list[OneTranslationData]:
37+
return_only_first_query_ctx_var.set(return_only_first_query)
3238
result=translator.translate_all(text=text,source=source_platform_id)
3339
translations= []
3440
forplatform_resultinresult:

‎uncoder-core/app/translator/core/custom_types/functions.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min="min"
1010
sum="sum"
1111

12-
divide="divide"
12+
values="values"
1313

1414
earliest="earliest"
1515
latest="latest"
1616

17+
divide="divide"
18+
1719
lower="lower"
20+
split="split"
1821
upper="upper"
1922

20-
compare_boolean="compare_boolean"
21-
23+
array_length="array_length"
24+
compare="compare"
25+
extract_time="extract_time"
2226
ipv4_is_in_range="ipv4_is_in_range"
2327

2428
bin="bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats="stats"
3135
table="table"
3236
timeframe="timeframe"
33-
values="values"
37+
union="union"
38+
39+
aggregation_data_function="aggregation_data_function"

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
importre
2222
fromabcimportABC,abstractmethod
2323
fromdataclassesimportdataclass
24-
fromfunctoolsimportcached_property
25-
fromtypingimportTYPE_CHECKING,Any,Optional
24+
fromtypingimportTYPE_CHECKING,Any,ClassVar,Optional,Union
2625

27-
fromapp.translator.core.exceptions.functionsimportInvalidFunctionSignature,NotSupportedFunctionException
26+
fromapp.translator.core.exceptions.functionsimportNotSupportedFunctionException
2827
fromapp.translator.core.mappingimportSourceMapping
29-
fromapp.translator.core.models.fieldimportField
28+
fromapp.translator.core.models.fieldimportAlias,Field
3029
fromapp.translator.core.models.functions.baseimportFunction,ParsedFunctions,RenderedFunctions
30+
fromapp.translator.tools.utilsimportexecute_module
3131
fromsettingsimportINIT_FUNCTIONS
3232

3333
ifTYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
classBaseFunctionParser(ABC):
44+
function_names_map:ClassVar[dict[str,str]]= {}
45+
functions_group_name:str=None
46+
manager:PlatformFunctionsManager=None
47+
48+
defset_functions_manager(self,manager:PlatformFunctionsManager)->BaseFunctionParser:
49+
self.manager=manager
50+
returnself
51+
4452
@abstractmethod
4553
defparse(self,*args,**kwargs)->Function:
4654
raiseNotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
classFunctionRender(ABC):
84+
function_names_map:ClassVar[dict[str,str]]= {}
85+
order_to_render:int=0
86+
in_query_render:bool=False
87+
render_to_prefix:bool=False
88+
manager:PlatformFunctionsManager=None
89+
90+
defset_functions_manager(self,manager:PlatformFunctionsManager)->FunctionRender:
91+
self.manager=manager
92+
returnself
93+
7694
@abstractmethod
7795
defrender(self,function:Function,source_mapping:SourceMapping)->str:
7896
raiseNotImplementedError
7997

8098
@staticmethod
81-
defconcat_kwargs(kwargs:dict[str,str])->str:
82-
result=""
83-
forkey,valueinkwargs.items():
84-
ifvalue:
85-
result=f"{result},{key}={value}"ifresultelsef"{key}={value}"
99+
defmap_field(field:Union[Alias,Field],source_mapping:SourceMapping)->str:
100+
ifisinstance(field,Alias):
101+
returnfield.name
86102

87-
returnresult
88-
89-
@staticmethod
90-
defmap_field(field:Field,source_mapping:SourceMapping)->str:
91103
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
ifisinstance(mapped_field,list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
classPlatformFunctionsManager:
112+
platform_functions:PlatformFunctions=None
113+
100114
def__init__(self):
101-
self._parsers_map:dict[str,HigherOrderFunctionParser]= {}
102-
self._renders_map:dict[str,FunctionRender]= {}
103-
self._in_query_renders_map:dict[str,FunctionRender]= {}
104-
self._names_map:dict[str,str]= {}
105-
self._order_to_render:dict[str,int]= {}
106-
self._render_to_prefix_functions:list[str]= []
107-
108-
defpost_init_configure(self,platform_render:PlatformQueryRender)->None:
109-
raiseNotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map:dict[str,HigherOrderFunctionParser]= {}
117+
self._parsers_map:dict[str,FunctionParser]= {}# {platform_func_name: FunctionParser}
118+
119+
self._renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
120+
self._in_query_renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
121+
self._order_to_render:dict[str,int]= {}# {generic_func_name: int}
122+
123+
defregister_render(self,render_class:type[FunctionRender])->type[FunctionRender]:
124+
render=render_class()
125+
render.manager=self
126+
forgeneric_function_nameinrender.function_names_map:
127+
self._renders_map[generic_function_name]=render
128+
self._order_to_render[generic_function_name]=render.order_to_render
129+
ifrender.in_query_render:
130+
self._in_query_renders_map[generic_function_name]=render
131+
132+
returnrender_class
133+
134+
defregister_parser(self,parser_class:type[BaseFunctionParser])->type[BaseFunctionParser]:
135+
parser=parser_class()
136+
parser.manager=self
137+
parsers_map=self._hof_parsers_mapifisinstance(parser,HigherOrderFunctionParser)elseself._parsers_map
138+
forplatform_function_nameinparser.function_names_map:
139+
parsers_map[platform_function_name]=parser
140+
141+
ifparser.functions_group_name:
142+
parsers_map[parser.functions_group_name]=parser
143+
144+
returnparser_class
145+
146+
defget_hof_parser(self,platform_func_name:str)->HigherOrderFunctionParser:
147+
ifINIT_FUNCTIONSand (parser:=self._hof_parsers_map.get(platform_func_name)):
148+
returnparser
110149

111-
@cached_property
112-
def_inverted_names_map(self)->dict[str,str]:
113-
return {value:keyforkey,valueinself._names_map.items()}
150+
raiseNotSupportedFunctionException
114151

115-
defget_parser(self,generic_func_name:str)->HigherOrderFunctionParser:
116-
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(generic_func_name)):
152+
defget_parser(self,platform_func_name:str)->FunctionParser:
153+
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(platform_func_name)):
117154
returnparser
118155

119156
raiseNotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raiseNotSupportedFunctionException
132169

133-
defget_generic_func_name(self,platform_func_name:str)->Optional[str]:
134-
ifINIT_FUNCTIONSand (generic_func_name:=self._names_map.get(platform_func_name)):
135-
returngeneric_func_name
136-
137-
raiseNotSupportedFunctionException
138-
139-
defget_platform_func_name(self,generic_func_name:str)->Optional[str]:
140-
ifINIT_FUNCTIONS:
141-
returnself._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
deforder_to_render(self)->dict[str,int]:
145172
ifINIT_FUNCTIONS:
146173
returnself._order_to_render
147174

148175
return {}
149176

150-
@property
151-
defrender_to_prefix_functions(self)->list[str]:
152-
ifINIT_FUNCTIONS:
153-
returnself._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
classPlatformFunctions:
179+
dir_path:str=None
180+
platform_query_render:PlatformQueryRender=None
159181
manager:PlatformFunctionsManager=PlatformFunctionsManager()
182+
160183
function_delimiter="|"
161184

162-
defparse(self,query:str)->ParsedFunctions:
163-
parsed= []
164-
not_supported= []
165-
invalid= []
166-
functions=query.split(self.function_delimiter)
167-
forfuncinfunctions:
168-
split_func=func.strip().split(" ")
169-
func_name,func_body=split_func[0]," ".join(split_func[1:])
170-
try:
171-
func_parser=self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body,func))
173-
exceptNotSupportedFunctionException:
174-
not_supported.append(func)
175-
exceptInvalidFunctionSignature:
176-
invalid.append(func)
185+
def__init__(self):
186+
self.manager.platform_functions=self
187+
ifself.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
returnParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func)forfuncinnot_supported],
181-
invalid=invalid,
182-
)
191+
defparse(self,query:str)->ParsedFunctions:# noqa: ARG002
192+
returnParsedFunctions()
183193

184194
def_sort_functions_to_render(self,functions:list[Function])->list[Function]:
185195
returnsorted(functions,key=lambdafunc:self.manager.order_to_render.get(func.name,0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render=self.manager.get_render(func.name)
195205
_rendered=func_render.render(func,source_mapping)
196-
iffunc.nameinself.manager.render_to_prefix_functions:
206+
iffunc_render.render_to_prefix:
197207
rendered_prefix+=_rendered
198208
else:
199209
rendered+=self.wrap_function_with_delimiter(_rendered)

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ field_mapping:
1010
#dns-record: dns-record
1111
dns_query_name:xdm.network.dns.dns_question.name
1212
QueryName:xdm.network.dns.dns_question.name
13-
query:xdm.network.dns.dns_question.name
13+
query:xdm.network.dns.dns_question.name
14+
dns-record-type:xdm.network.dns.dns_question.type

‎uncoder-core/app/translator/mappings/platforms/qradar/default.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ field_mapping:
3535
-userName
3636
-EventUserName
3737
CommandLine:Command
38-
Protocol:IPProtocol
38+
Protocol:
39+
-IPProtocol
40+
-protocol
3941
Application:
4042
-Application
4143
-application
@@ -61,6 +63,7 @@ field_mapping:
6163
SourceMAC:
6264
-SourceMAC
6365
-MAC
66+
-sourceMAC
6467
DestinationMAC:DestinationMAC
6568
SourceOS:
6669
-SourceOS
@@ -69,4 +72,7 @@ field_mapping:
6972
TargetUserName:DestinationUserName
7073
SourceUserName:SourceUserName
7174
url_category:XForceCategoryByURL
72-
EventSeverity:EventSeverity
75+
EventSeverity:EventSeverity
76+
Source:
77+
-Source
78+
-source

‎uncoder-core/app/translator/mappings/platforms/qradar/dns.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ field_mapping:
1212
dns-query:URL
1313
parent-domain:parent-domain
1414
dns-answer:dns-answer
15-
dns-record:URL
15+
dns-record:URL
16+
dns-record-type:DNSRecordType

‎uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ field_mapping:
2424
cs-host:
2525
-UrlHost
2626
-URL Host
27+
-URL Domain
2728
cs-referrer:
2829
-URL Referrer
2930
-Referrer URL

‎uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ field_mapping:
4141
LinkName:LinkName
4242
MemberName:MemberName
4343
MemberSid:MemberSid
44-
NewProcessName:Process Name
44+
NewProcessName:
45+
-Process Name
46+
-New Process Name
4547
ObjectClass:ObjectClass
4648
ObjectName:
4749
-Object Name
@@ -122,6 +124,7 @@ field_mapping:
122124
ServiceFileName:
123125
-Service Filename
124126
-ServiceFileName
127+
-Service File Name
125128
SecurityDescriptor:SecurityDescriptor
126129
ServiceName:Service Name
127130
ShareName:
Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
1-
importimportlib.util
21
importos
32

3+
fromapp.translator.tools.utilsimportexecute_module
44
fromconstimportPLATFORMS_PATH
55

66

7-
definit_platforms():
7+
definit_platforms()->None:
88
forplatformin [fforfinos.listdir(PLATFORMS_PATH)ifos.path.isdir(os.path.join(PLATFORMS_PATH,f))]:
99
ifnotplatform.startswith("__")andnotplatform.endswith("__"):
1010
# Platforms __init__.py execution
11-
init_path=f"{PLATFORMS_PATH}/{platform}/__init__.py"
12-
spec=importlib.util.spec_from_file_location("__init__",init_path)
13-
init_module=importlib.util.module_from_spec(spec)
14-
spec.loader.exec_module(init_module)
11+
execute_module(f"{PLATFORMS_PATH}/{platform}/__init__.py")
1512

1613

1714
init_platforms()
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
UTF8_PAYLOAD_PATTERN=r"UTF8\(payload\)"
22
NUM_VALUE_PATTERN=r"(?P<num_value>\d+(?:\.\d+)*)"
3-
SINGLE_QUOTES_VALUE_PATTERN=r"""'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'"""
3+
SINGLE_QUOTES_VALUE_PATTERN= (
4+
r"""'(?P<s_q_value>(?:[:a-zA-Zа-яА-Я\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'"""# noqa: RUF001
5+
)
46
TABLE_PATTERN=r"\s+FROM\s+[a-zA-Z.\-*]+"
57
TABLE_GROUP_PATTERN=r"\s+FROM\s+(?P<table>[a-zA-Z.\-*]+)"

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp