Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit7ace6a1

Browse files
authored
Merge pull request#37 from UncoderIO/field-class-refactoring
field class refactoring
2 parents164b34c +d9404a0 commit7ace6a1

File tree

19 files changed

+134
-145
lines changed

19 files changed

+134
-145
lines changed

‎translator/app/translator/core/functions.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ def concat_kwargs(kwargs: Dict[str, str]) -> str:
5858

5959
@staticmethod
6060
defmap_field(field:Field,source_mapping:SourceMapping)->str:
61-
generic_field_name=field.generic_names_map[source_mapping.source_id]
61+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
6262
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
6363
ifisinstance(mapped_field,list):
6464
mapped_field=mapped_field[0]

‎translator/app/translator/core/mapping.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,7 @@ def get_suitable_source_mappings(self, *args, **kwargs) -> List[SourceMapping]:
122122

123123
defget_source_mapping(self,source_id:str)->Optional[SourceMapping]:
124124
returnself._source_mappings.get(source_id)
125+
126+
@property
127+
defdefault_mapping(self)->SourceMapping:
128+
returnself._source_mappings[DEFAULT_MAPPING_NAME]

‎translator/app/translator/core/mixins/logic.py‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,29 @@
11
fromtypingimportList,Union
22

3-
fromapp.translator.core.models.fieldimportField,Keyword
4-
fromapp.translator.core.models.identifierimportIdentifier
53
fromapp.translator.core.custom_types.tokensimportLogicalOperatorType,GroupType
4+
fromapp.translator.core.models.fieldimportFieldValue,Keyword
5+
fromapp.translator.core.models.identifierimportIdentifier
66

77

88
classANDLogicOperatorMixin:
99

1010
@staticmethod
11-
defget_missed_and_token_indices(tokens:List[Union[Field,Keyword,Identifier]])->List[int]:
11+
defget_missed_and_token_indices(tokens:List[Union[FieldValue,Keyword,Identifier]])->List[int]:
1212
missed_and_indices= []
1313
forindexinrange(len(tokens)-1):
1414
token=tokens[index]
1515
next_token=tokens[index+1]
16-
if (isinstance(token, (Field,Keyword))
17-
andnot (isinstance(next_token,Identifier)and (
18-
next_token.token_typeinLogicalOperatorType
19-
ornext_token.token_type==GroupType.R_PAREN))):
16+
if ((isinstance(token, (FieldValue,Keyword))
17+
orisinstance(token,Identifier)andtoken.token_type==GroupType.R_PAREN)
18+
andnot (isinstance(next_token,Identifier)
19+
and (next_token.token_type
20+
in (LogicalOperatorType.AND,LogicalOperatorType.OR,GroupType.R_PAREN)))):
2021
missed_and_indices.append(index+1)
2122
returnlist(reversed(missed_and_indices))
2223

23-
defadd_and_token_if_missed(self,tokens:List[Union[Field,Keyword,Identifier]])->List[Union[Field,Keyword,Identifier]]:
24+
defadd_and_token_if_missed(self,
25+
tokens:List[Union[FieldValue,Keyword,Identifier]]
26+
)->List[Union[FieldValue,Keyword,Identifier]]:
2427
indices=self.get_missed_and_token_indices(tokens=tokens)
2528
forindexinindices:
2629
tokens.insert(index,Identifier(token_type=LogicalOperatorType.AND))

‎translator/app/translator/core/models/field.py‎

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,36 @@
11
fromtypingimportUnion,Optional
22

3+
fromapp.translator.core.mappingimportSourceMapping,DEFAULT_MAPPING_NAME
34
fromapp.translator.core.models.identifierimportIdentifier
45
fromapp.translator.core.custom_types.tokensimportOperatorType
56

67

78
classField:
8-
def__init__(self,source_name:str,operator:Identifier=None,value:Union[int,str,list,tuple]=None):
9+
def__init__(self,source_name:str):
10+
self.source_name=source_name
11+
self.__generic_names_map= {}
12+
13+
defget_generic_field_name(self,source_id:str)->Optional[str]:
14+
returnself.__generic_names_map.get(source_id)
15+
16+
defset_generic_names_map(self,source_mappings:list[SourceMapping],default_mapping:SourceMapping)->None:
17+
generic_names_map= {
18+
source_mapping.source_id:source_mapping.fields_mapping.get_generic_field_name(self.source_name)
19+
forsource_mappinginsource_mappings
20+
}
21+
ifDEFAULT_MAPPING_NAMEnotingeneric_names_map:
22+
fields_mapping=default_mapping.fields_mapping
23+
generic_names_map[DEFAULT_MAPPING_NAME]=fields_mapping.get_generic_field_name(self.source_name)
24+
25+
self.__generic_names_map=generic_names_map
26+
27+
28+
classFieldValue:
29+
def__init__(self,source_name:str,operator:Identifier,value:Union[int,str,list,tuple]):
30+
self.field=Field(source_name=source_name)
931
self.operator=operator
1032
self.values= []
1133
self.__add_value(value)
12-
self.source_name=source_name# input translation field name
13-
self.generic_names_map= {}
1434

1535
@property
1636
defvalue(self):
@@ -30,31 +50,7 @@ def __add__(self, other):
3050
self.values.append(other)
3151

3252
def__repr__(self):
33-
ifself.operator:
34-
returnf"{self.source_name}{self.operator.token_type}{self.values}"
35-
36-
returnf"{self.source_name}"
37-
38-
def__eq__(self,other):
39-
ifisinstance(other,Field):
40-
returnself._hash==other._hash
41-
"""For OR operator check"""
42-
ifself.source_name==other.source_nameandself.operator==other.operator:
43-
returnTrue
44-
returnFalse
45-
46-
def__neq__(self,other):
47-
"""For AND operator check"""
48-
ifself.source_name!=other.source_name:
49-
returnTrue
50-
returnFalse
51-
52-
@property
53-
def_hash(self):
54-
returnhash(str(self))
55-
56-
def__hash__(self):
57-
returnhash(str(self))
53+
returnf"{self.field.source_name}{self.operator.token_type}{self.values}"
5854

5955

6056
classKeyword:

‎translator/app/translator/core/models/functions/base.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@
33
fromdataclassesimportdataclass,field
44
fromtypingimportList,Union
55

6-
fromapp.translator.core.models.fieldimportField,Keyword
6+
fromapp.translator.core.models.fieldimportField,FieldValue,Keyword
77
fromapp.translator.core.models.identifierimportIdentifier
88

99

1010
@dataclass
1111
classFunction:
1212
name:str=None
13-
args:List[Union[Field,Keyword,Function,Identifier]]=field(default_factory=list)
13+
args:List[Union[Field,FieldValue,Keyword,Function,Identifier]]=field(default_factory=list)
1414
as_clause:str=None
1515
by_clauses:List[Field]=field(default_factory=list)
1616

‎translator/app/translator/core/parser.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
fromapp.translator.core.functionsimportPlatformFunctions
2323
fromapp.translator.core.mappingimportBasePlatformMappings,SourceMapping
24-
fromapp.translator.core.models.fieldimportField
24+
fromapp.translator.core.models.fieldimportFieldValue
2525
fromapp.translator.core.models.functions.baseimportParsedFunctions
2626
fromapp.translator.core.models.platform_detailsimportPlatformDetails
2727
fromapp.translator.core.models.parser_outputimportSiemContainer,MetaInfoContainer
@@ -50,15 +50,15 @@ def get_tokens_and_source_mappings(self,
5050
ifnotquery:
5151
raiseTokenizerGeneralException("Can't translate empty query. Please provide more details")
5252
tokens=self.tokenizer.tokenize(query=query)
53-
field_tokens=self.tokenizer.filter_tokens(tokens,Field)
53+
field_tokens=[token.fieldfortokeninself.tokenizer.filter_tokens(tokens,FieldValue)]
5454
field_names= [field.source_nameforfieldinfield_tokens]
55-
suitable_source_mappings=self.mappings.get_suitable_source_mappings(field_names=field_names,**log_sources)
56-
self.tokenizer.set_field_generic_names_map(field_tokens,suitable_source_mappings,self.mappings)
55+
source_mappings=self.mappings.get_suitable_source_mappings(field_names=field_names,**log_sources)
56+
self.tokenizer.set_field_tokens_generic_names_map(field_tokens,source_mappings,self.mappings.default_mapping)
5757

58-
returntokens,suitable_source_mappings
58+
returntokens,source_mappings
5959

6060
defset_functions_fields_generic_names(self,
6161
functions:ParsedFunctions,
6262
source_mappings:List[SourceMapping])->None:
63-
field_tokens=self.tokenizer.filter_function_tokens(tokens=functions.functions)
64-
self.tokenizer.set_field_generic_names_map(field_tokens,source_mappings,self.mappings)
63+
field_tokens=self.tokenizer.get_field_tokens_from_func_args(args=functions.functions)
64+
self.tokenizer.set_field_tokens_generic_names_map(field_tokens,source_mappings,self.mappings.default_mapping)

‎translator/app/translator/core/render.py‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,12 @@
2727
fromapp.translator.core.exceptions.parserimportUnsupportedOperatorException
2828
fromapp.translator.core.functionsimportPlatformFunctions
2929
fromapp.translator.core.mappingimportBasePlatformMappings,SourceMapping,LogSourceSignature,DEFAULT_MAPPING_NAME
30-
fromapp.translator.core.models.fieldimportField,Keyword
30+
fromapp.translator.core.models.fieldimportField,FieldValue,Keyword
3131
fromapp.translator.core.models.functions.baseimportFunction,ParsedFunctions
32+
fromapp.translator.core.models.identifierimportIdentifier
3233
fromapp.translator.core.models.platform_detailsimportPlatformDetails
3334
fromapp.translator.core.models.parser_outputimportMetaInfoContainer
34-
fromapp.translator.core.custom_types.tokensimportLogicalOperatorType,OperatorType,GroupType
35+
fromapp.translator.core.custom_types.tokensimportLogicalOperatorType,OperatorType
3536

3637

3738
classBaseQueryFieldValue(ABC):
@@ -133,7 +134,7 @@ def generate_functions(self, functions: List[Function], source_mapping: SourceMa
133134
returnself.platform_functions.render(functions,source_mapping)ifself.platform_functionselse""
134135

135136
defmap_field(self,field:Field,source_mapping:SourceMapping)->List[str]:
136-
generic_field_name=field.generic_names_map[source_mapping.source_id]
137+
generic_field_name=field.get_generic_field_name(source_mapping.source_id)
137138
# field can be mapped to corresponding platform field name or list of platform field names
138139
mapped_field=source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
139140
ifnotmapped_fieldandself.is_strict_mapping:
@@ -145,10 +146,10 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> List[str]:
145146
returnmapped_fieldifmapped_fieldelse [generic_field_name]ifgeneric_field_nameelse [field.source_name]
146147

147148
defapply_token(self,
148-
token:Union[Field,Keyword,LogicalOperatorType,GroupType],
149+
token:Union[FieldValue,Keyword,Identifier],
149150
source_mapping:SourceMapping)->str:
150-
ifisinstance(token,(Field,Keyword)):
151-
mapped_fields=self.map_field(token,source_mapping)ifisinstance(token,Field)else [None]
151+
ifisinstance(token,FieldValue):
152+
mapped_fields=self.map_field(token.field,source_mapping)
152153
iflen(mapped_fields)>1:
153154
returnself.group_token%self.operator_map[LogicalOperatorType.OR].join([
154155
self.field_value_map.apply_field_value(field=field,operator=token.operator,value=token.value)
@@ -158,12 +159,17 @@ def apply_token(self,
158159
returnself.field_value_map.apply_field_value(field=mapped_fields[0],
159160
operator=token.operator,
160161
value=token.value)
162+
elifisinstance(token,Keyword):
163+
returnself.field_value_map.apply_field_value(field=None,
164+
operator=token.operator,
165+
value=token.value)
161166
eliftoken.token_typeinLogicalOperatorType:
162167
returnself.operator_map.get(token.token_type)
168+
163169
returntoken.token_type
164170

165171
defgenerate_query(self,
166-
query:List[Union[Field,Keyword,LogicalOperatorType,GroupType]],
172+
query:List[Union[FieldValue,Keyword,Identifier]],
167173
source_mapping:SourceMapping)->str:
168174
result_values= []
169175
fortokeninquery:
@@ -173,8 +179,7 @@ def generate_query(self,
173179
defwrap_query_with_meta_info(self,meta_info:MetaInfoContainer,query:str):
174180
ifmeta_infoand (meta_info.idormeta_info.title):
175181
query_meta_info="\n".join(
176-
self.wrap_with_comment(f"{key}{value}")
177-
forkey,valuein {"name: ":meta_info.title,"uuid: ":meta_info.id}.items()ifvalue
182+
self.wrap_with_comment(f"{key}{value}")forkey,valuein {"name: ":meta_info.title,"uuid: ":meta_info.id}.items()ifvalue
178183
)
179184
query=f"{query}\n\n{query_meta_info}"
180185
returnquery

‎translator/app/translator/core/tokenizer.py‎

Lines changed: 29 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -27,20 +27,20 @@
2727
TokenizerGeneralException,
2828
QueryParenthesesException
2929
)
30-
fromapp.translator.core.mappingimportSourceMapping,DEFAULT_MAPPING_NAME,BasePlatformMappings
31-
fromapp.translator.core.models.fieldimportField,Keyword
30+
fromapp.translator.core.mappingimportSourceMapping
31+
fromapp.translator.core.models.fieldimportField,FieldValue,Keyword
3232
fromapp.translator.core.models.functions.baseimportFunction
3333
fromapp.translator.core.models.functions.sortimportSortArg
3434
fromapp.translator.core.models.identifierimportIdentifier
3535
fromapp.translator.core.custom_types.tokensimportOperatorType,GroupType
3636
fromapp.translator.tools.utilsimportget_match_group
3737

38-
TOKEN_TYPE=Union[Field,Keyword,Identifier]
38+
TOKEN_TYPE=Union[FieldValue,Keyword,Identifier]
3939

4040

4141
classBaseTokenizer(ABC):
4242
@abstractmethod
43-
deftokenize(self,query:str)->List[Union[Field,Keyword,Identifier]]:
43+
deftokenize(self,query:str)->List[Union[FieldValue,Keyword,Identifier]]:
4444
raiseNotImplementedError()
4545

4646

@@ -180,18 +180,18 @@ def process_value_wildcard_symbols(self,
180180
returnself._clean_value(value,wildcard_symbol),op
181181

182182
@staticmethod
183-
defcreate_field(field_name:str,operator:Identifier,value:Union[str,List])->Field:
184-
returnField(operator=operator,value=value,source_name=field_name)
183+
defcreate_field_value(field_name:str,operator:Identifier,value:Union[str,List])->FieldValue:
184+
returnFieldValue(source_name=field_name,operator=operator,value=value)
185185

186-
defsearch_field_value(self,query):
186+
defsearch_field_value(self,query)->Tuple[FieldValue,str]:
187187
field_name=self.search_field(query)
188188
operator=self.search_operator(query,field_name)
189189
query,operator,value=self.search_value(query=query,operator=operator,field_name=field_name)
190190
value,operator_token=self.process_value_wildcard_symbols(value=value,
191191
operator=operator,
192192
wildcard_symbol=self.wildcard_symbol)
193-
field=self.create_field(field_name=field_name,operator=operator_token,value=value)
194-
returnfield,query
193+
field_value=self.create_field_value(field_name=field_name,operator=operator_token,value=value)
194+
returnfield_value,query
195195

196196
def_match_field_value(self,query:str,white_space_pattern:str=r"\s+")->bool:
197197
single_value_operator_group=fr"(?:{'|'.join(self.single_value_operators_map)})"
@@ -208,7 +208,7 @@ def _match_field_value(self, query: str, white_space_pattern: str = r"\s+") -> b
208208

209209
returnFalse
210210

211-
def_get_identifier(self,query:str)->Tuple[Union[Field,Keyword,Identifier],str]:
211+
def_get_identifier(self,query:str)->Tuple[Union[FieldValue,Keyword,Identifier],str]:
212212
query=query.strip("\n").strip(" ").strip("\n")
213213
ifquery.startswith(GroupType.L_PAREN):
214214
returnIdentifier(token_type=GroupType.L_PAREN),query[1:]
@@ -240,7 +240,7 @@ def _validate_parentheses(tokens):
240240
raiseQueryParenthesesException()
241241
returnTrue
242242

243-
deftokenize(self,query:str)->List[Union[Field,Keyword,Identifier]]:
243+
deftokenize(self,query:str)->List[Union[FieldValue,Keyword,Identifier]]:
244244
tokenized= []
245245
whilequery:
246246
identifier,query=self._get_identifier(query=query)
@@ -250,34 +250,28 @@ def tokenize(self, query: str) -> List[Union[Field, Keyword, Identifier]]:
250250

251251
@staticmethod
252252
deffilter_tokens(tokens:List[TOKEN_TYPE],
253-
token_type:Union[Type[Field],Type[Keyword],Type[Identifier]])->List[TOKEN_TYPE]:
253+
token_type:Union[Type[FieldValue],Type[Keyword],Type[Identifier]])->List[TOKEN_TYPE]:
254254
return [tokenfortokenintokensifisinstance(token,token_type)]
255255

256-
deffilter_function_tokens(self,
257-
tokens:List[Union[Field,Keyword,Identifier,Function,SortArg]])->List[TOKEN_TYPE]:
256+
defget_field_tokens_from_func_args(self,
257+
args:List[Union[Field,FieldValue,Keyword,Identifier,Function,SortArg]]
258+
)->List[Field]:
258259
result= []
259-
fortokenintokens:
260-
ifisinstance(token,Field):
261-
result.append(token)
262-
elifisinstance(token,Function):
263-
result.extend(self.filter_function_tokens(tokens=token.args))
264-
result.extend(self.filter_function_tokens(tokens=token.by_clauses))
265-
elifisinstance(token,SortArg):
266-
result.append(token.field)
260+
forarginargs:
261+
ifisinstance(arg,Field):
262+
result.append(arg)
263+
elifisinstance(arg,FieldValue):
264+
result.append(arg.field)
265+
elifisinstance(arg,Function):
266+
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
267+
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))
268+
elifisinstance(arg,SortArg):
269+
result.append(arg.field)
267270
returnresult
268271

269272
@staticmethod
270-
defset_field_generic_names_map(tokens:List[Field],
271-
source_mappings:List[SourceMapping],
272-
platform_mappings:BasePlatformMappings)->None:
273+
defset_field_tokens_generic_names_map(tokens:List[Field],
274+
source_mappings:List[SourceMapping],
275+
default_mapping:SourceMapping)->None:
273276
fortokenintokens:
274-
generic_names_map= {
275-
source_mapping.source_id:source_mapping.fields_mapping.get_generic_field_name(token.source_name)
276-
forsource_mappinginsource_mappings
277-
}
278-
ifDEFAULT_MAPPING_NAMEnotingeneric_names_map:
279-
default_source_mapping=platform_mappings.get_source_mapping(DEFAULT_MAPPING_NAME)
280-
fields_mapping=default_source_mapping.fields_mapping
281-
generic_names_map[DEFAULT_MAPPING_NAME]=fields_mapping.get_generic_field_name(token.source_name)
282-
283-
token.generic_names_map=generic_names_map
277+
token.set_generic_names_map(source_mappings,default_mapping)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp