Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commitff305d5

Browse files
Merge branch 'UncoderIO:main' into main
2 parents01e06fe +7606f21 commitff305d5

File tree

75 files changed

+366
-243
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+366
-243
lines changed
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
fromcontextvarsimportContextVar
22

33
return_only_first_query_ctx_var:ContextVar[bool]=ContextVar("return_only_first_query_ctx_var",default=False)
4-
"""Set to True to returnony first query if rendered multiple options"""
4+
"""Set to True to returnonly first query if rendered multiple options"""

‎uncoder-core/app/translator/core/custom_types/functions.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min="min"
1010
sum="sum"
1111

12-
divide="divide"
12+
values="values"
1313

1414
earliest="earliest"
1515
latest="latest"
1616

17+
divide="divide"
18+
1719
lower="lower"
20+
split="split"
1821
upper="upper"
1922

20-
compare_boolean="compare_boolean"
21-
23+
array_length="array_length"
24+
compare="compare"
25+
extract_time="extract_time"
2226
ipv4_is_in_range="ipv4_is_in_range"
2327

2428
bin="bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats="stats"
3135
table="table"
3236
timeframe="timeframe"
33-
values="values"
37+
union="union"
38+
39+
aggregation_data_function="aggregation_data_function"
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
fromapp.translator.tools.custom_enumimportCustomEnum
2+
3+
4+
classTimeFrameType(CustomEnum):
5+
years="years"
6+
months="months"
7+
days="days"
8+
hours="hours"
9+
minutes="minutes"

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
importre
2222
fromabcimportABC,abstractmethod
2323
fromdataclassesimportdataclass
24-
fromfunctoolsimportcached_property
25-
fromtypingimportTYPE_CHECKING,Any,Optional
24+
fromtypingimportTYPE_CHECKING,Any,ClassVar,Optional,Union
2625

27-
fromapp.translator.core.exceptions.functionsimportInvalidFunctionSignature,NotSupportedFunctionException
26+
fromapp.translator.core.exceptions.functionsimportNotSupportedFunctionException
2827
fromapp.translator.core.mappingimportSourceMapping
29-
fromapp.translator.core.models.fieldimportField
28+
fromapp.translator.core.models.fieldimportAlias,Field
3029
fromapp.translator.core.models.functions.baseimportFunction,ParsedFunctions,RenderedFunctions
30+
fromapp.translator.tools.utilsimportexecute_module
3131
fromsettingsimportINIT_FUNCTIONS
3232

3333
ifTYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
classBaseFunctionParser(ABC):
44+
function_names_map:ClassVar[dict[str,str]]= {}
45+
functions_group_name:str=None
46+
manager:PlatformFunctionsManager=None
47+
48+
defset_functions_manager(self,manager:PlatformFunctionsManager)->BaseFunctionParser:
49+
self.manager=manager
50+
returnself
51+
4452
@abstractmethod
4553
defparse(self,*args,**kwargs)->Function:
4654
raiseNotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
classFunctionRender(ABC):
84+
function_names_map:ClassVar[dict[str,str]]= {}
85+
order_to_render:int=0
86+
in_query_render:bool=False
87+
render_to_prefix:bool=False
88+
manager:PlatformFunctionsManager=None
89+
90+
defset_functions_manager(self,manager:PlatformFunctionsManager)->FunctionRender:
91+
self.manager=manager
92+
returnself
93+
7694
@abstractmethod
7795
defrender(self,function:Function,source_mapping:SourceMapping)->str:
7896
raiseNotImplementedError
7997

8098
@staticmethod
81-
defconcat_kwargs(kwargs:dict[str,str])->str:
82-
result=""
83-
forkey,valueinkwargs.items():
84-
ifvalue:
85-
result=f"{result},{key}={value}"ifresultelsef"{key}={value}"
99+
defmap_field(field:Union[Alias,Field],source_mapping:SourceMapping)->str:
100+
ifisinstance(field,Alias):
101+
returnfield.name
86102

87-
returnresult
88-
89-
@staticmethod
90-
defmap_field(field:Field,source_mapping:SourceMapping)->str:
91103
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
ifisinstance(mapped_field,list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
classPlatformFunctionsManager:
112+
platform_functions:PlatformFunctions=None
113+
100114
def__init__(self):
101-
self._parsers_map:dict[str,HigherOrderFunctionParser]= {}
102-
self._renders_map:dict[str,FunctionRender]= {}
103-
self._in_query_renders_map:dict[str,FunctionRender]= {}
104-
self._names_map:dict[str,str]= {}
105-
self._order_to_render:dict[str,int]= {}
106-
self._render_to_prefix_functions:list[str]= []
107-
108-
defpost_init_configure(self,platform_render:PlatformQueryRender)->None:
109-
raiseNotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map:dict[str,HigherOrderFunctionParser]= {}
117+
self._parsers_map:dict[str,FunctionParser]= {}# {platform_func_name: FunctionParser}
118+
119+
self._renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
120+
self._in_query_renders_map:dict[str,FunctionRender]= {}# {generic_func_name: FunctionRender}
121+
self._order_to_render:dict[str,int]= {}# {generic_func_name: int}
122+
123+
defregister_render(self,render_class:type[FunctionRender])->type[FunctionRender]:
124+
render=render_class()
125+
render.manager=self
126+
forgeneric_function_nameinrender.function_names_map:
127+
self._renders_map[generic_function_name]=render
128+
self._order_to_render[generic_function_name]=render.order_to_render
129+
ifrender.in_query_render:
130+
self._in_query_renders_map[generic_function_name]=render
131+
132+
returnrender_class
133+
134+
defregister_parser(self,parser_class:type[BaseFunctionParser])->type[BaseFunctionParser]:
135+
parser=parser_class()
136+
parser.manager=self
137+
parsers_map=self._hof_parsers_mapifisinstance(parser,HigherOrderFunctionParser)elseself._parsers_map
138+
forplatform_function_nameinparser.function_names_map:
139+
parsers_map[platform_function_name]=parser
140+
141+
ifparser.functions_group_name:
142+
parsers_map[parser.functions_group_name]=parser
143+
144+
returnparser_class
145+
146+
defget_hof_parser(self,platform_func_name:str)->HigherOrderFunctionParser:
147+
ifINIT_FUNCTIONSand (parser:=self._hof_parsers_map.get(platform_func_name)):
148+
returnparser
110149

111-
@cached_property
112-
def_inverted_names_map(self)->dict[str,str]:
113-
return {value:keyforkey,valueinself._names_map.items()}
150+
raiseNotSupportedFunctionException
114151

115-
defget_parser(self,generic_func_name:str)->HigherOrderFunctionParser:
116-
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(generic_func_name)):
152+
defget_parser(self,platform_func_name:str)->FunctionParser:
153+
ifINIT_FUNCTIONSand (parser:=self._parsers_map.get(platform_func_name)):
117154
returnparser
118155

119156
raiseNotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raiseNotSupportedFunctionException
132169

133-
defget_generic_func_name(self,platform_func_name:str)->Optional[str]:
134-
ifINIT_FUNCTIONSand (generic_func_name:=self._names_map.get(platform_func_name)):
135-
returngeneric_func_name
136-
137-
raiseNotSupportedFunctionException
138-
139-
defget_platform_func_name(self,generic_func_name:str)->Optional[str]:
140-
ifINIT_FUNCTIONS:
141-
returnself._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
deforder_to_render(self)->dict[str,int]:
145172
ifINIT_FUNCTIONS:
146173
returnself._order_to_render
147174

148175
return {}
149176

150-
@property
151-
defrender_to_prefix_functions(self)->list[str]:
152-
ifINIT_FUNCTIONS:
153-
returnself._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
classPlatformFunctions:
179+
dir_path:str=None
180+
platform_query_render:PlatformQueryRender=None
159181
manager:PlatformFunctionsManager=PlatformFunctionsManager()
182+
160183
function_delimiter="|"
161184

162-
defparse(self,query:str)->ParsedFunctions:
163-
parsed= []
164-
not_supported= []
165-
invalid= []
166-
functions=query.split(self.function_delimiter)
167-
forfuncinfunctions:
168-
split_func=func.strip().split(" ")
169-
func_name,func_body=split_func[0]," ".join(split_func[1:])
170-
try:
171-
func_parser=self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body,func))
173-
exceptNotSupportedFunctionException:
174-
not_supported.append(func)
175-
exceptInvalidFunctionSignature:
176-
invalid.append(func)
185+
def__init__(self):
186+
self.manager.platform_functions=self
187+
ifself.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
returnParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func)forfuncinnot_supported],
181-
invalid=invalid,
182-
)
191+
defparse(self,query:str)->ParsedFunctions:# noqa: ARG002
192+
returnParsedFunctions()
183193

184194
def_sort_functions_to_render(self,functions:list[Function])->list[Function]:
185195
returnsorted(functions,key=lambdafunc:self.manager.order_to_render.get(func.name,0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render=self.manager.get_render(func.name)
195205
_rendered=func_render.render(func,source_mapping)
196-
iffunc.nameinself.manager.render_to_prefix_functions:
206+
iffunc_render.render_to_prefix:
197207
rendered_prefix+=_rendered
198208
else:
199209
rendered+=self.wrap_function_with_delimiter(_rendered)

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
1819
importre
1920
fromabcimportABC,abstractmethod
2021
fromtypingimportUnion

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
fromabcimportABC,abstractmethod
2021
fromcollections.abcimportCallable
21-
fromtypingimportOptional,Union
22+
fromtypingimportClassVar,Optional,Union
2223

2324
fromapp.translator.constimportDEFAULT_VALUE_TYPE
2425
fromapp.translator.core.context_varsimportreturn_only_first_query_ctx_var
@@ -165,7 +166,14 @@ class QueryRender(ABC):
165166
is_single_line_comment:bool=False
166167
unsupported_functions_text="Unsupported functions were excluded from the result query:"
167168

168-
platform_functions:PlatformFunctions=PlatformFunctions()
169+
platform_functions:PlatformFunctions=None
170+
171+
def__init__(self):
172+
self.init_platform_functions()
173+
174+
definit_platform_functions(self)->None:
175+
self.platform_functions=PlatformFunctions()
176+
self.platform_functions.platform_query_render=self
169177

170178
defrender_not_supported_functions(self,not_supported_functions:list)->str:
171179
line_template=f"{self.comment_symbol} "ifself.comment_symbolandself.is_single_line_commentelse""
@@ -189,22 +197,23 @@ class PlatformQueryRender(QueryRender):
189197
not_token="not"
190198

191199
group_token="(%s)"
200+
query_parts_delimiter=" "
192201

193202
field_value_map=BaseQueryFieldValue(or_token=or_token)
194203

195-
query_pattern="{table} {query} {functions}"
196-
raw_log_field_pattern_map:dict=None
204+
raw_log_field_pattern_map:ClassVar[dict[str,str]]=None
197205

198206
def__init__(self):
207+
super().__init__()
199208
self.operator_map= {
200209
LogicalOperatorType.AND:f"{self.and_token} ",
201210
LogicalOperatorType.OR:f"{self.or_token} ",
202211
LogicalOperatorType.NOT:f"{self.not_token} ",
203212
}
204213

205-
defgenerate_prefix(self,log_source_signature:LogSourceSignature,functions_prefix:str="")->str:# noqa: ARG002
206-
ifstr(log_source_signature):
207-
returnf"{log_source_signature!s}{self.and_token}"
214+
defgenerate_prefix(self,log_source_signature:Optional[LogSourceSignature],functions_prefix:str="")->str:# noqa: ARG002
215+
iflog_source_signatureandstr(log_source_signature):
216+
returnf"{log_source_signature}{self.and_token}"
208217
return""
209218

210219
defgenerate_functions(self,functions:list[Function],source_mapping:SourceMapping)->RenderedFunctions:
@@ -272,6 +281,14 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
272281
query=f"{query}\n\n{query_meta_info}"
273282
returnquery
274283

284+
@staticmethod
285+
def_finalize_search_query(query:str)->str:
286+
returnquery
287+
288+
def_join_query_parts(self,prefix:str,query:str,functions:str)->str:
289+
parts=filter(lambdas:bool(s),map(str.strip, [prefix,self._finalize_search_query(query),functions]))
290+
returnself.query_parts_delimiter.join(parts)
291+
275292
deffinalize_query(
276293
self,
277294
prefix:str,
@@ -283,8 +300,7 @@ def finalize_query(
283300
*args,# noqa: ARG002
284301
**kwargs,# noqa: ARG002
285302
)->str:
286-
query=self.query_pattern.format(prefix=prefix,query=query,functions=functions).strip()
287-
303+
query=self._join_query_parts(prefix,query,functions)
288304
query=self.wrap_query_with_meta_info(meta_info=meta_info,query=query)
289305
ifnot_supported_functions:
290306
rendered_not_supported=self.render_not_supported_functions(not_supported_functions)
@@ -327,15 +343,15 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
327343

328344
defprocess_raw_log_field(self,field:str,field_type:str)->Optional[str]:
329345
ifraw_log_field_pattern:=self.raw_log_field_pattern_map.get(field_type):
330-
returnraw_log_field_pattern.pattern.format(field=field)
346+
returnraw_log_field_pattern.format(field=field)
331347

332348
defprocess_raw_log_field_prefix(self,field:str,source_mapping:SourceMapping)->Optional[list]:
333349
ifisinstance(field,list):
334-
list_of_prefix= []
350+
prefix_list= []
335351
forfinfield:
336-
ifprepared_prefix:=self.process_raw_log_field_prefix(field=f,source_mapping=source_mapping):
337-
list_of_prefix.extend(prepared_prefix)
338-
returnlist_of_prefix
352+
if_prefix_list:=self.process_raw_log_field_prefix(field=f,source_mapping=source_mapping):
353+
prefix_list.extend(_prefix_list)
354+
returnprefix_list
339355
ifraw_log_field_type:=source_mapping.raw_log_fields.get(field):
340356
return [self.process_raw_log_field(field=field,field_type=raw_log_field_type)]
341357

@@ -352,9 +368,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
352368
)
353369
ifnotmapped_fieldandself.is_strict_mapping:
354370
raiseStrictPlatformException(field_name=field.source_name,platform_name=self.details.name)
355-
iffield_prefix:=self.process_raw_log_field_prefix(field=mapped_field,source_mapping=source_mapping):
356-
defined_raw_log_fields.extend(field_prefix)
357-
return"\n".join(set(defined_raw_log_fields))
371+
ifprefix_list:=self.process_raw_log_field_prefix(field=mapped_field,source_mapping=source_mapping):
372+
forprefixinprefix_list:
373+
ifprefixnotindefined_raw_log_fields:
374+
defined_raw_log_fields.append(prefix)
375+
return"\n".join(defined_raw_log_fields)
358376

359377
def_generate_from_tokenized_query_container(self,query_container:TokenizedQueryContainer)->str:
360378
queries_map= {}
@@ -369,7 +387,7 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
369387
defined_raw_log_fields=self.generate_raw_log_fields(
370388
fields=query_container.meta_info.query_fields,source_mapping=source_mapping
371389
)
372-
prefix+=f"\n{defined_raw_log_fields}\n"
390+
prefix+=f"\n{defined_raw_log_fields}"
373391
result=self.generate_query(tokens=query_container.tokens,source_mapping=source_mapping)
374392
exceptStrictPlatformExceptionaserr:
375393
errors.append(err)

‎uncoder-core/app/translator/core/render_cti.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
fromapp.translator.core.models.iocsimportIocsChunkValue
2221
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2322

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp