Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commit5f2d770

Browse files
committed
merge main
2 parents10030cd +5a15552 commit5f2d770

File tree

56 files changed

+309
-170
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+309
-170
lines changed

‎uncoder-core/app/routers/translate.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
fromfastapiimportAPIRouter,Body
22

33
fromapp.models.translationimportInfoMessage,OneTranslationData,Platform,TranslatorPlatforms
4+
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var
45
fromapp.translator.cti_translatorimportCTITranslator
56
fromapp.translator.translatorimportTranslator
67

@@ -15,7 +16,9 @@ def translate_one(
1516
source_platform_id:str=Body(...,embed=True),
1617
target_platform_id:str=Body(...,embed=True),
1718
text:str=Body(...,embed=True),
19+
return_only_first_query:bool=False,
1820
)->OneTranslationData:
21+
return_only_first_query_ctx_var.set(return_only_first_query)
1922
status,data=translator.translate_one(text=text,source=source_platform_id,target=target_platform_id)
2023
ifstatus:
2124
returnOneTranslationData(status=status,translation=data,target_platform_id=target_platform_id)
@@ -27,8 +30,11 @@ def translate_one(
2730
@st_router.post("/translate/all",tags=["translator"],description="Generate all translations")
2831
@st_router.post("/translate/all/",include_in_schema=False)
2932
deftranslate_all(
30-
source_platform_id:str=Body(...,embed=True),text:str=Body(...,embed=True)
33+
source_platform_id:str=Body(...,embed=True),
34+
text:str=Body(...,embed=True),
35+
return_only_first_query:bool=False,
3136
)->list[OneTranslationData]:
37+
return_only_first_query_ctx_var.set(return_only_first_query)
3238
result=translator.translate_all(text=text,source=source_platform_id)
3339
translations= []
3440
forplatform_resultinresult:

‎uncoder-core/app/translator/core/custom_types/functions.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min="min"
1010
sum="sum"
1111

12-
divide="divide"
12+
values="values"
1313

1414
earliest="earliest"
1515
latest="latest"
1616

17+
divide="divide"
18+
1719
lower="lower"
20+
split="split"
1821
upper="upper"
1922

20-
compare_boolean="compare_boolean"
21-
23+
array_length="array_length"
24+
compare="compare"
25+
extract_time="extract_time"
2226
ipv4_is_in_range="ipv4_is_in_range"
2327

2428
bin="bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats="stats"
3135
table="table"
3236
timeframe="timeframe"
33-
values="values"
37+
union="union"
38+
39+
aggregation_data_function="aggregation_data_function"
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
fromapp.translator.tools.custom_enumimportCustomEnum
2+
3+
4+
classTimeFrameType(CustomEnum):
5+
years="years"
6+
months="months"
7+
days="days"
8+
hours="hours"
9+
minutes="minutes"
10+
11+
12+
classTimePartType(CustomEnum):
13+
day="day"
14+
day_of_week="day_of_week"
15+
day_of_year="day_of_year"
16+
hour="hour"
17+
microsecond="microsecond"
18+
millisecond="millisecond"
19+
minute="minute"
20+
month="month"
21+
quarter="quarter"
22+
second="second"
23+
year="year"

‎uncoder-core/app/translator/core/exceptions/core.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
fromtypingimportOptional
2+
3+
14
classNotImplementedException(BaseException):
25
...
36

@@ -7,8 +10,19 @@ class BasePlatformException(BaseException):
710

811

912
classStrictPlatformException(BasePlatformException):
10-
def__init__(self,platform_name:str,field_name:str):
11-
message=f"Platform{platform_name} has strict mapping. Source field{field_name} has no mapping."
13+
field_name:str=None
14+
15+
def__init__(
16+
self,platform_name:str,field_name:str,mapping:Optional[str]=None,detected_fields:Optional[list]=None
17+
):
18+
message= (
19+
f"Platform{platform_name} has strict mapping. "
20+
f"Source fields:{', '.join(detected_fields)ifdetected_fieldselsefield_name} has no mapping."
21+
f" Mapping file:{mapping}."
22+
ifmapping
23+
else""
24+
)
25+
self.field_name=field_name
1226
super().__init__(message)
1327

1428

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
importre
2222
fromabcimportABC,abstractmethod
2323
fromdataclassesimportdataclass
24-
fromfunctoolsimportcached_property
25-
fromtypingimportTYPE_CHECKING,Any,Optional
24+
fromtypingimportTYPE_CHECKING,Any,ClassVar,Optional,Union
2625

27-
fromapp.translator.core.exceptions.functionsimportInvalidFunctionSignature,NotSupportedFunctionException
26+
fromapp.translator.core.exceptions.functionsimportNotSupportedFunctionException
2827
fromapp.translator.core.mappingimportSourceMapping
29-
fromapp.translator.core.models.fieldimportField
28+
fromapp.translator.core.models.fieldimportAlias,Field
3029
fromapp.translator.core.models.functions.baseimportFunction,ParsedFunctions,RenderedFunctions
30+
fromapp.translator.tools.utilsimportexecute_module
3131
fromsettingsimportINIT_FUNCTIONS
3232

3333
ifTYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
classBaseFunctionParser(ABC):
44+
function_names_map:ClassVar[dict[str,str]]= {}
45+
functions_group_name:str=None
46+
manager:PlatformFunctionsManager=None
47+
48+
defset_functions_manager(self,manager:PlatformFunctionsManager)->BaseFunctionParser:
49+
self.manager=manager
50+
returnself
51+
4452
@abstractmethod
4553
defparse(self,*args,**kwargs)->Function:
4654
raiseNotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
classFunctionRender(ABC):
84+
function_names_map:ClassVar[dict[str,str]]= {}
85+
order_to_render:int=0
86+
in_query_render:bool=False
87+
render_to_prefix:bool=False
88+
manager:PlatformFunctionsManager=None
89+
90+
defset_functions_manager(self,manager:PlatformFunctionsManager)->FunctionRender:
91+
self.manager=manager
92+
returnself
93+
7694
@abstractmethod
7795
defrender(self,function:Function,source_mapping:SourceMapping)->str:
7896
raiseNotImplementedError
7997

8098
@staticmethod
81-
defconcat_kwargs(kwargs:dict[str,str])->str:
82-
result=""
83-
forkey,valueinkwargs.items():
84-
ifvalue:
85-
result=f"{result},{key}={value}"ifresultelsef"{key}={value}"
99+
defmap_field(field:Union[Alias,Field],source_mapping:SourceMapping)->str:
100+
ifisinstance(field,Alias):
101+
returnfield.name
86102

87-
returnresult
88-
89-
@staticmethod
90-
defmap_field(field:Field,source_mapping:SourceMapping)->str:
91103
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
ifisinstance(mapped_field,list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
classPlatformFunctionsManager:
112+
platform_functions:PlatformFunctions=None
113+
100114
def__init__(self):
101-
self._parsers_map:dict[str,HigherOrderFunctionParser]= {}
102-
self._renders_map:dict[str,FunctionRender]= {}
103-
self._in_query_renders_map:dict[str,FunctionRender]= {}
104-
self._names_map:dict[str,str]= {}
105-
self._order_to_render:dict[str,int]= {}
106-
self._render_to_prefix_functions:list[str]= []
107-
108-
defpost_init_configure(self,platform_render:PlatformQueryRender)->None:
109-
raiseNotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map:dict[str,HigherOrderFunctionParser]= {}
117+
self._parsers_map:dict[str,FunctionParser]= {}# {platform_func_name: FunctionParser}
118+
119+
self._renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
120+
self._in_query_renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
121+
self._order_to_render:dict[str,int]= {}# {generic_func_name: int}
122+
123+
defregister_render(self,render_class:type[FunctionRender])->type[FunctionRender]:
124+
render=render_class()
125+
render.manager=self
126+
forgeneric_function_nameinrender.function_names_map:
127+
self._renders_map[generic_function_name]=render
128+
self._order_to_render[generic_function_name]=render.order_to_render
129+
ifrender.in_query_render:
130+
self._in_query_renders_map[generic_function_name]=render
131+
132+
returnrender_class
133+
134+
defregister_parser(self,parser_class:type[BaseFunctionParser])->type[BaseFunctionParser]:
135+
parser=parser_class()
136+
parser.manager=self
137+
parsers_map=self._hof_parsers_mapifisinstance(parser,HigherOrderFunctionParser)elseself._parsers_map
138+
forplatform_function_nameinparser.function_names_map:
139+
parsers_map[platform_function_name]=parser
140+
141+
ifparser.functions_group_name:
142+
parsers_map[parser.functions_group_name]=parser
143+
144+
returnparser_class
145+
146+
defget_hof_parser(self,platform_func_name:str)->HigherOrderFunctionParser:
147+
ifINIT_FUNCTIONSand (parser:=self._hof_parsers_map.get(platform_func_name)):
148+
returnparser
110149

111-
@cached_property
112-
def_inverted_names_map(self)->dict[str,str]:
113-
return {value:keyforkey,valueinself._names_map.items()}
150+
raiseNotSupportedFunctionException
114151

115-
defget_parser(self,generic_func_name:str)->HigherOrderFunctionParser:
116-
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(generic_func_name)):
152+
defget_parser(self,platform_func_name:str)->FunctionParser:
153+
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(platform_func_name)):
117154
returnparser
118155

119156
raiseNotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raiseNotSupportedFunctionException
132169

133-
defget_generic_func_name(self,platform_func_name:str)->Optional[str]:
134-
ifINIT_FUNCTIONSand (generic_func_name:=self._names_map.get(platform_func_name)):
135-
returngeneric_func_name
136-
137-
raiseNotSupportedFunctionException
138-
139-
defget_platform_func_name(self,generic_func_name:str)->Optional[str]:
140-
ifINIT_FUNCTIONS:
141-
returnself._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
deforder_to_render(self)->dict[str,int]:
145172
ifINIT_FUNCTIONS:
146173
returnself._order_to_render
147174

148175
return {}
149176

150-
@property
151-
defrender_to_prefix_functions(self)->list[str]:
152-
ifINIT_FUNCTIONS:
153-
returnself._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
classPlatformFunctions:
179+
dir_path:str=None
180+
platform_query_render:PlatformQueryRender=None
159181
manager:PlatformFunctionsManager=PlatformFunctionsManager()
182+
160183
function_delimiter="|"
161184

162-
defparse(self,query:str)->ParsedFunctions:
163-
parsed= []
164-
not_supported= []
165-
invalid= []
166-
functions=query.split(self.function_delimiter)
167-
forfuncinfunctions:
168-
split_func=func.strip().split(" ")
169-
func_name,func_body=split_func[0]," ".join(split_func[1:])
170-
try:
171-
func_parser=self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body,func))
173-
exceptNotSupportedFunctionException:
174-
not_supported.append(func)
175-
exceptInvalidFunctionSignature:
176-
invalid.append(func)
185+
def__init__(self):
186+
self.manager.platform_functions=self
187+
ifself.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
returnParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func)forfuncinnot_supported],
181-
invalid=invalid,
182-
)
191+
defparse(self,query:str)->ParsedFunctions:# noqa: ARG002
192+
returnParsedFunctions()
183193

184194
def_sort_functions_to_render(self,functions:list[Function])->list[Function]:
185195
returnsorted(functions,key=lambdafunc:self.manager.order_to_render.get(func.name,0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render=self.manager.get_render(func.name)
195205
_rendered=func_render.render(func,source_mapping)
196-
iffunc.nameinself.manager.render_to_prefix_functions:
206+
iffunc_render.render_to_prefix:
197207
rendered_prefix+=_rendered
198208
else:
199209
rendered+=self.wrap_function_with_delimiter(_rendered)

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
1819
importre
1920
fromabcimportABC,abstractmethod
2021
fromtypingimportUnion

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ class PlatformQueryRender(QueryRender):
197197
not_token="not"
198198

199199
group_token="(%s)"
200+
query_parts_delimiter=" "
200201

201202
field_value_map=BaseQueryFieldValue(or_token=or_token)
202203

@@ -262,8 +263,14 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
262263

263264
defgenerate_query(self,tokens:list[TOKEN_TYPE],source_mapping:SourceMapping)->str:
264265
result_values= []
266+
unmapped_fields=set()
265267
fortokenintokens:
266-
result_values.append(self.apply_token(token=token,source_mapping=source_mapping))
268+
try:
269+
result_values.append(self.apply_token(token=token,source_mapping=source_mapping))
270+
exceptStrictPlatformExceptionaserr:
271+
unmapped_fields.add(err.field_name)
272+
ifunmapped_fields:
273+
raiseStrictPlatformException(self.details.name,"",source_mapping.source_id,sorted(unmapped_fields))
267274
return"".join(result_values)
268275

269276
defwrap_query_with_meta_info(self,meta_info:MetaInfoContainer,query:str)->str:
@@ -284,6 +291,10 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
284291
def_finalize_search_query(query:str)->str:
285292
returnquery
286293

294+
def_join_query_parts(self,prefix:str,query:str,functions:str)->str:
295+
parts=filter(lambdas:bool(s),map(str.strip, [prefix,self._finalize_search_query(query),functions]))
296+
returnself.query_parts_delimiter.join(parts)
297+
287298
deffinalize_query(
288299
self,
289300
prefix:str,
@@ -295,8 +306,7 @@ def finalize_query(
295306
*args,# noqa: ARG002
296307
**kwargs,# noqa: ARG002
297308
)->str:
298-
parts=filter(lambdas:bool(s),map(str.strip, [prefix,self._finalize_search_query(query),functions]))
299-
query=" ".join(parts)
309+
query=self._join_query_parts(prefix,query,functions)
300310
query=self.wrap_query_with_meta_info(meta_info=meta_info,query=query)
301311
ifnot_supported_functions:
302312
rendered_not_supported=self.render_not_supported_functions(not_supported_functions)
@@ -383,7 +393,7 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
383393
defined_raw_log_fields=self.generate_raw_log_fields(
384394
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
385395
)
386-
prefix+=f"\n{defined_raw_log_fields}\n"
396+
prefix+=f"\n{defined_raw_log_fields}"
387397
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
388398
exceptStrictPlatformExceptionaserr:
389399
errors.append(err)

‎uncoder-core/app/translator/core/render_cti.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
fromapp.translator.core.models.iocsimportIocsChunkValue
2221
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2322

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,3 +125,4 @@ field_mapping:
125125
SourceOS:xdm.source.host.os
126126
DestinationOS:xdm.target.host.os
127127
url_category:xdm.network.http.url_category
128+
EventSeverity:xdm.alert.severity

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp