Commit9eed876

authored

Merge pull request#174 from UncoderIO/gis-8171

Improve filter_tokens in core tokenizer

2 parentsd10cc06 +90da0d7 commit9eed876Copy full SHA for 9eed876

File tree

+10

-2

lines changed

+10

-2

lines changed

Lines changed: 6 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@`
`28`	`28`	`fromapp.translator.core.models.platform_detailsimportPlatformDetails`
`29`	`29`	`fromapp.translator.core.models.query_containerimportRawQueryContainer,TokenizedQueryContainer`
`30`	`30`	`fromapp.translator.core.models.query_tokens.fieldimportField`
	`31`	`+fromapp.translator.core.models.query_tokens.field_fieldimportFieldField`
`31`	`32`	`fromapp.translator.core.models.query_tokens.field_valueimportFieldValue`
`32`	`33`	`fromapp.translator.core.models.query_tokens.function_valueimportFunctionValue`
`33`	`34`	`fromapp.translator.core.tokenizerimportQueryTokenizer`
`@@ -68,6 +69,11 @@ def get_field_tokens(`
`68`	`69`	`fortokeninquery_tokens:`
`69`	`70`	`ifisinstance(token,FieldValue):`
`70`	`71`	`field_tokens.append(token.field)`
	`72`	`+elifisinstance(token,FieldField):`
	`73`	`+iftoken.field_left:`
	`74`	`+field_tokens.append(token.field_left)`
	`75`	`+iftoken.field_right:`
	`76`	`+field_tokens.append(token.field_right)`
`71`	`77`	`elifisinstance(token,FunctionValue):`
`72`	`78`	`field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function]))`
`73`	`79`

Lines changed: 4 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -338,7 +338,7 @@ def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]:`
`338`	`338`	`@staticmethod`
`339`	`339`	`deffilter_tokens(`
`340`	`340`	`tokens:list[QUERY_TOKEN_TYPE],`
`341`		`-token_type:Union[type[FieldValue],type[Field],type[Keyword],type[Identifier]],`
	`341`	`+token_type:Union[type[FieldValue],type[Field],type[FieldField],type[Keyword],type[Identifier]],`
`342`	`342`	`)->list[QUERY_TOKEN_TYPE]:`
`343`	`343`	`return [tokenfortokenintokensifisinstance(token,token_type)]`
`344`	`344`
`@@ -363,7 +363,9 @@ def get_field_tokens_from_func_args( # noqa: PLR0912`
`363`	`363`	`result.extend(self.get_field_tokens_from_func_args(args=arg.args))`
`364`	`364`	`result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))`
`365`	`365`	`result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_]))`
`366`		`-elifisinstance(arg, (JoinFunction,UnionFunction)):`
	`366`	`+elifisinstance(arg,JoinFunction):`
	`367`	`+result.extend(self.get_field_tokens_from_func_args(args=arg.condition))`
	`368`	`+elifisinstance(arg,UnionFunction):`
`367`	`369`	`continue`
`368`	`370`	`elifisinstance(arg,Function):`
`369`	`371`	`result.extend(self.get_field_tokens_from_func_args(args=arg.args))`

Comments

(0)