Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commit8a06b99

Browse files
authored
Merge pull request#152 from UncoderIO/gis-7984
Gis 7984
2 parentscf48a62 +13298c9 commit8a06b99

File tree

72 files changed

+353
-238
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+353
-238
lines changed
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
fromcontextvarsimportContextVar
22

33
return_only_first_query_ctx_var:ContextVar[bool]=ContextVar("return_only_first_query_ctx_var",default=False)
4-
"""Set to True to returnony first query if rendered multiple options"""
4+
"""Set to True to returnonly first query if rendered multiple options"""

‎uncoder-core/app/translator/core/custom_types/functions.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min="min"
1010
sum="sum"
1111

12-
divide="divide"
12+
values="values"
1313

1414
earliest="earliest"
1515
latest="latest"
1616

17+
divide="divide"
18+
1719
lower="lower"
20+
split="split"
1821
upper="upper"
1922

20-
compare_boolean="compare_boolean"
21-
23+
array_length="array_length"
24+
compare="compare"
25+
extract_time="extract_time"
2226
ipv4_is_in_range="ipv4_is_in_range"
2327

2428
bin="bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats="stats"
3135
table="table"
3236
timeframe="timeframe"
33-
values="values"
37+
union="union"
38+
39+
aggregation_data_function="aggregation_data_function"
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
fromapp.translator.tools.custom_enumimportCustomEnum
2+
3+
4+
classTimeFrameType(CustomEnum):
5+
years="years"
6+
months="months"
7+
days="days"
8+
hours="hours"
9+
minutes="minutes"

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
importre
2222
fromabcimportABC,abstractmethod
2323
fromdataclassesimportdataclass
24-
fromfunctoolsimportcached_property
25-
fromtypingimportTYPE_CHECKING,Any,Optional
24+
fromtypingimportTYPE_CHECKING,Any,ClassVar,Optional,Union
2625

27-
fromapp.translator.core.exceptions.functionsimportInvalidFunctionSignature,NotSupportedFunctionException
26+
fromapp.translator.core.exceptions.functionsimportNotSupportedFunctionException
2827
fromapp.translator.core.mappingimportSourceMapping
29-
fromapp.translator.core.models.fieldimportField
28+
fromapp.translator.core.models.fieldimportAlias,Field
3029
fromapp.translator.core.models.functions.baseimportFunction,ParsedFunctions,RenderedFunctions
30+
fromapp.translator.tools.utilsimportexecute_module
3131
fromsettingsimportINIT_FUNCTIONS
3232

3333
ifTYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
classBaseFunctionParser(ABC):
44+
function_names_map:ClassVar[dict[str,str]]= {}
45+
functions_group_name:str=None
46+
manager:PlatformFunctionsManager=None
47+
48+
defset_functions_manager(self,manager:PlatformFunctionsManager)->BaseFunctionParser:
49+
self.manager=manager
50+
returnself
51+
4452
@abstractmethod
4553
defparse(self,*args,**kwargs)->Function:
4654
raiseNotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
classFunctionRender(ABC):
84+
function_names_map:ClassVar[dict[str,str]]= {}
85+
order_to_render:int=0
86+
in_query_render:bool=False
87+
render_to_prefix:bool=False
88+
manager:PlatformFunctionsManager=None
89+
90+
defset_functions_manager(self,manager:PlatformFunctionsManager)->FunctionRender:
91+
self.manager=manager
92+
returnself
93+
7694
@abstractmethod
7795
defrender(self,function:Function,source_mapping:SourceMapping)->str:
7896
raiseNotImplementedError
7997

8098
@staticmethod
81-
defconcat_kwargs(kwargs:dict[str,str])->str:
82-
result=""
83-
forkey,valueinkwargs.items():
84-
ifvalue:
85-
result=f"{result},{key}={value}"ifresultelsef"{key}={value}"
99+
defmap_field(field:Union[Alias,Field],source_mapping:SourceMapping)->str:
100+
ifisinstance(field,Alias):
101+
returnfield.name
86102

87-
returnresult
88-
89-
@staticmethod
90-
defmap_field(field:Field,source_mapping:SourceMapping)->str:
91103
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
ifisinstance(mapped_field,list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
classPlatformFunctionsManager:
112+
platform_functions:PlatformFunctions=None
113+
100114
def__init__(self):
101-
self._parsers_map:dict[str,HigherOrderFunctionParser]= {}
102-
self._renders_map:dict[str,FunctionRender]= {}
103-
self._in_query_renders_map:dict[str,FunctionRender]= {}
104-
self._names_map:dict[str,str]= {}
105-
self._order_to_render:dict[str,int]= {}
106-
self._render_to_prefix_functions:list[str]= []
107-
108-
defpost_init_configure(self,platform_render:PlatformQueryRender)->None:
109-
raiseNotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map:dict[str,HigherOrderFunctionParser]= {}
117+
self._parsers_map:dict[str,FunctionParser]= {}# {platform_func_name: FunctionParser}
118+
119+
self._renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
120+
self._in_query_renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
121+
self._order_to_render:dict[str,int]= {}# {generic_func_name: int}
122+
123+
defregister_render(self,render_class:type[FunctionRender])->type[FunctionRender]:
124+
render=render_class()
125+
render.manager=self
126+
forgeneric_function_nameinrender.function_names_map:
127+
self._renders_map[generic_function_name]=render
128+
self._order_to_render[generic_function_name]=render.order_to_render
129+
ifrender.in_query_render:
130+
self._in_query_renders_map[generic_function_name]=render
131+
132+
returnrender_class
133+
134+
defregister_parser(self,parser_class:type[BaseFunctionParser])->type[BaseFunctionParser]:
135+
parser=parser_class()
136+
parser.manager=self
137+
parsers_map=self._hof_parsers_mapifisinstance(parser,HigherOrderFunctionParser)elseself._parsers_map
138+
forplatform_function_nameinparser.function_names_map:
139+
parsers_map[platform_function_name]=parser
140+
141+
ifparser.functions_group_name:
142+
parsers_map[parser.functions_group_name]=parser
143+
144+
returnparser_class
145+
146+
defget_hof_parser(self,platform_func_name:str)->HigherOrderFunctionParser:
147+
ifINIT_FUNCTIONSand (parser:=self._hof_parsers_map.get(platform_func_name)):
148+
returnparser
110149

111-
@cached_property
112-
def_inverted_names_map(self)->dict[str,str]:
113-
return {value:keyforkey,valueinself._names_map.items()}
150+
raiseNotSupportedFunctionException
114151

115-
defget_parser(self,generic_func_name:str)->HigherOrderFunctionParser:
116-
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(generic_func_name)):
152+
defget_parser(self,platform_func_name:str)->FunctionParser:
153+
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(platform_func_name)):
117154
returnparser
118155

119156
raiseNotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raiseNotSupportedFunctionException
132169

133-
defget_generic_func_name(self,platform_func_name:str)->Optional[str]:
134-
ifINIT_FUNCTIONSand (generic_func_name:=self._names_map.get(platform_func_name)):
135-
returngeneric_func_name
136-
137-
raiseNotSupportedFunctionException
138-
139-
defget_platform_func_name(self,generic_func_name:str)->Optional[str]:
140-
ifINIT_FUNCTIONS:
141-
returnself._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
deforder_to_render(self)->dict[str,int]:
145172
ifINIT_FUNCTIONS:
146173
returnself._order_to_render
147174

148175
return {}
149176

150-
@property
151-
defrender_to_prefix_functions(self)->list[str]:
152-
ifINIT_FUNCTIONS:
153-
returnself._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
classPlatformFunctions:
179+
dir_path:str=None
180+
platform_query_render:PlatformQueryRender=None
159181
manager:PlatformFunctionsManager=PlatformFunctionsManager()
182+
160183
function_delimiter="|"
161184

162-
defparse(self,query:str)->ParsedFunctions:
163-
parsed= []
164-
not_supported= []
165-
invalid= []
166-
functions=query.split(self.function_delimiter)
167-
forfuncinfunctions:
168-
split_func=func.strip().split(" ")
169-
func_name,func_body=split_func[0]," ".join(split_func[1:])
170-
try:
171-
func_parser=self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body,func))
173-
exceptNotSupportedFunctionException:
174-
not_supported.append(func)
175-
exceptInvalidFunctionSignature:
176-
invalid.append(func)
185+
def__init__(self):
186+
self.manager.platform_functions=self
187+
ifself.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
returnParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func)forfuncinnot_supported],
181-
invalid=invalid,
182-
)
191+
defparse(self,query:str)->ParsedFunctions:# noqa: ARG002
192+
returnParsedFunctions()
183193

184194
def_sort_functions_to_render(self,functions:list[Function])->list[Function]:
185195
returnsorted(functions,key=lambdafunc:self.manager.order_to_render.get(func.name,0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render=self.manager.get_render(func.name)
195205
_rendered=func_render.render(func,source_mapping)
196-
iffunc.nameinself.manager.render_to_prefix_functions:
206+
iffunc_render.render_to_prefix:
197207
rendered_prefix+=_rendered
198208
else:
199209
rendered+=self.wrap_function_with_delimiter(_rendered)

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
1819
importre
1920
fromabcimportABC,abstractmethod
2021
fromtypingimportUnion

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
fromabcimportABC,abstractmethod
2021
fromcollections.abcimportCallable
21-
fromtypingimportOptional,Union
22+
fromtypingimportClassVar,Optional,Union
2223

2324
fromapp.translator.constimportDEFAULT_VALUE_TYPE
2425
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var
@@ -165,7 +166,14 @@ class QueryRender(ABC):
165166
is_single_line_comment:bool=False
166167
unsupported_functions_text="Unsupported functions were excluded from the result query:"
167168

168-
platform_functions:PlatformFunctions=PlatformFunctions()
169+
platform_functions:PlatformFunctions=None
170+
171+
def__init__(self):
172+
self.init_platform_functions()
173+
174+
definit_platform_functions(self)->None:
175+
self.platform_functions=PlatformFunctions()
176+
self.platform_functions.platform_query_render=self
169177

170178
defrender_not_supported_functions(self,not_supported_functions:list)->str:
171179
line_template=f"{self.comment_symbol} "ifself.comment_symbolandself.is_single_line_commentelse""
@@ -192,19 +200,19 @@ class PlatformQueryRender(QueryRender):
192200

193201
field_value_map=BaseQueryFieldValue(or_token=or_token)
194202

195-
query_pattern="{table} {query} {functions}"
196-
raw_log_field_pattern_map:dict=None
203+
raw_log_field_pattern_map:ClassVar[dict[str,str]]=None
197204

198205
def__init__(self):
206+
super().__init__()
199207
self.operator_map= {
200208
LogicalOperatorType.AND:f"{self.and_token} ",
201209
LogicalOperatorType.OR:f"{self.or_token} ",
202210
LogicalOperatorType.NOT:f"{self.not_token} ",
203211
}
204212

205-
defgenerate_prefix(self,log_source_signature:LogSourceSignature,functions_prefix:str="")->str:# noqa: ARG002
206-
ifstr(log_source_signature):
207-
returnf"{log_source_signature!s}{self.and_token}"
213+
defgenerate_prefix(self,log_source_signature:Optional[LogSourceSignature],functions_prefix:str="")->str:# noqa: ARG002
214+
iflog_source_signatureandstr(log_source_signature):
215+
returnf"{log_source_signature}{self.and_token}"
208216
return""
209217

210218
defgenerate_functions(self,functions:list[Function],source_mapping:SourceMapping)->RenderedFunctions:
@@ -272,6 +280,10 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
272280
query=f"{query}\n\n{query_meta_info}"
273281
returnquery
274282

283+
@staticmethod
284+
def_finalize_search_query(query:str)->str:
285+
returnquery
286+
275287
deffinalize_query(
276288
self,
277289
prefix:str,
@@ -283,8 +295,8 @@ def finalize_query(
283295
*args,# noqa: ARG002
284296
**kwargs,# noqa: ARG002
285297
)->str:
286-
query=self.query_pattern.format(prefix=prefix,query=query,functions=functions).strip()
287-
298+
parts=filter(lambdas:bool(s),map(str.strip, [prefix,self._finalize_search_query(query),functions]))
299+
query=" ".join(parts)
288300
query=self.wrap_query_with_meta_info(meta_info=meta_info,query=query)
289301
ifnot_supported_functions:
290302
rendered_not_supported=self.render_not_supported_functions(not_supported_functions)
@@ -327,15 +339,15 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
327339

328340
defprocess_raw_log_field(self,field:str,field_type:str)->Optional[str]:
329341
ifraw_log_field_pattern:=self.raw_log_field_pattern_map.get(field_type):
330-
returnraw_log_field_pattern.pattern.format(field=field)
342+
returnraw_log_field_pattern.format(field=field)
331343

332344
defprocess_raw_log_field_prefix(self,field:str,source_mapping:SourceMapping)->Optional[list]:
333345
ifisinstance(field,list):
334-
list_of_prefix= []
346+
prefix_list= []
335347
forfinfield:
336-
ifprepared_prefix:=self.process_raw_log_field_prefix(field=f,source_mapping=source_mapping):
337-
list_of_prefix.extend(prepared_prefix)
338-
returnlist_of_prefix
348+
if_prefix_list:=self.process_raw_log_field_prefix(field=f,source_mapping=source_mapping):
349+
prefix_list.extend(_prefix_list)
350+
returnprefix_list
339351
ifraw_log_field_type:=source_mapping.raw_log_fields.get(field):
340352
return [self.process_raw_log_field(field=field,field_type=raw_log_field_type)]
341353

@@ -352,9 +364,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
352364
)
353365
ifnotmapped_fieldandself.is_strict_mapping:
354366
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
355-
iffield_prefix:=self.process_raw_log_field_prefix(field=mapped_field,source_mapping=source_mapping):
356-
defined_raw_log_fields.extend(field_prefix)
357-
return"\n".join(set(defined_raw_log_fields))
367+
ifprefix_list:=self.process_raw_log_field_prefix(field=mapped_field,source_mapping=source_mapping):
368+
forprefixinprefix_list:
369+
ifprefixnotindefined_raw_log_fields:
370+
defined_raw_log_fields.append(prefix)
371+
return"\n".join(defined_raw_log_fields)
358372

359373
def_generate_from_tokenized_query_container(self,query_container:TokenizedQueryContainer)->str:
360374
queries_map= {}

‎uncoder-core/app/translator/core/render_cti.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
fromapp.translator.core.models.iocsimportIocsChunkValue
2221
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2322

‎uncoder-core/app/translator/core/str_value_manager.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
fromtypingimportClassVar,Optional,TypeVar,Union
2021

2122
fromapp.translator.core.custom_types.valuesimportValueType

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp