NotificationsYou must be signed in to change notification settings
Fork33
Star165

Commit6973aa3

authored

Merge pull request#114 from UncoderIO/gis-7328

Created base platform: aql. And fixes for qradar

2 parentsd3dba4e +87274f8 commit6973aa3Copy full SHA for 6973aa3

File tree

49 files changed

+308

-411

lines changed

uncoder-core
- app/translator
  - core
    - exceptions
      - core.py
    - mixins
      - rule.py
    - models
      - query_container.py
    - parser.py
    - render_cti.py
    - tokenizer.py
  - managers.py
  - platforms
    - arcsight
      - __init__.py
      - const.py
      - mappings
        __init__.py
        arcsight_cti.py
      - renders
        __init__.py
    - athena
      - __init__.py
    - base/aql
      - __init__.py
      - const.py
      - escape_manager.py
      - parsers
        __init__.py
      - renders
        __init__.py
        aql.py
      - tokenizer.py
    - carbonblack
      - __init__.py
    - chronicle
      - __init__.py
    - crowdstrike
      - __init__.py
    - elasticsearch
      - __init__.py
    - fireeye_helix
      - __init__.py
    - forti_siem
      - __init__.py
    - graylog
      - __init__.py
    - logpoint
      - __init__.py
    - logrhythm_axon
      - __init__.py
    - logscale
      - __init__.py
    - microsoft
      - __init__.py
    - opensearch
      - __init__.py
    - palo_alto
      - __init__.py
    - qradar
      - __init__.py
      - const.py
      - escape_manager.py
      - mapping.py
      - parsers
        qradar.py
      - renders
        qradar.py
    - qualys
      - __init__.py
    - roota
      - __init__.py
    - rsa_netwitness
      - __init__.py
    - securonix
      - __init__.py
    - sentinel_one
      - __init__.py
    - sigma
      - __init__.py
    - snowflake
      - __init__.py
    - splunk
      - __init__.py
    - sumo_logic
      - __init__.py
- requirements.txt

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+308

-411

lines changed

`‎uncoder-core/app/translator/core/exceptions/core.py‎`

Lines changed: 4 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -77,3 +77,7 @@ class InvalidYamlStructure(InvalidRuleStructure):`
`77`	`77`
`78`	`78`	`classInvalidJSONStructure(InvalidRuleStructure):`
`79`	`79`	`rule_type:str="JSON"`
	`80`	`+`
	`81`	`+`
	`82`	`+classInvalidXMLStructure(InvalidRuleStructure):`
	`83`	`+rule_type:str="XML"`

`‎uncoder-core/app/translator/core/mixins/rule.py‎`

Lines changed: 12 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,10 @@`
`1`	`1`	`importjson`
	`2`	`+fromtypingimportUnion`
`2`	`3`
	`4`	`+importxmltodict`
`3`	`5`	`importyaml`
`4`	`6`
`5`		`-fromapp.translator.core.exceptions.coreimportInvalidJSONStructure,InvalidYamlStructure`
	`7`	`+fromapp.translator.core.exceptions.coreimportInvalidJSONStructure,InvalidXMLStructure,InvalidYamlStructure`
`6`	`8`	`fromapp.translator.core.mitreimportMitreConfig`
`7`	`9`
`8`	`10`
`@@ -36,5 +38,13 @@ def parse_mitre_attack(self, tags: list[str]) -> dict[str, list]:`
`36`	`38`	`result["techniques"].append(technique)`
`37`	`39`	`eliftactic:=self.mitre_config.get_tactic(tag):`
`38`	`40`	`result["tactics"].append(tactic)`
`39`		`-`
`40`	`41`	`returnresult`
	`42`	`+`
	`43`	`+`
	`44`	`+classXMLRuleMixin:`
	`45`	`+@staticmethod`
	`46`	`+defload_rule(text:Union[str,bytes])->dict:`
	`47`	`+try:`
	`48`	`+returnxmltodict.parse(text)`
	`49`	`+exceptExceptionaserr:`
	`50`	`+raiseInvalidXMLStructure(error=str(err))fromerr`

`‎uncoder-core/app/translator/core/models/query_container.py‎`

Lines changed: 7 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,13 @@ class RawQueryContainer:`
`56`	`56`	`meta_info:MetaInfoContainer=field(default_factory=MetaInfoContainer)`
`57`	`57`
`58`	`58`
	`59`	`+@dataclass`
	`60`	`+classRawQueryDictContainer:`
	`61`	`+query:dict`
	`62`	`+language:str`
	`63`	`+meta_info:dict`
	`64`	`+`
	`65`	`+`
`59`	`66`	`@dataclass`
`60`	`67`	`classTokenizedQueryContainer:`
`61`	`68`	`tokens:list[TOKEN_TYPE]`

`‎uncoder-core/app/translator/core/parser.py‎`

Lines changed: 5 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -32,9 +32,13 @@`
`32`	`32`
`33`	`33`	`classQueryParser(ABC):`
`34`	`34`	`wrapped_with_comment_pattern:str=None`
	`35`	`+details:PlatformDetails=None`
`35`	`36`
`36`	`37`	`defremove_comments(self,text:str)->str:`
`37`		`-returnre.sub(self.wrapped_with_comment_pattern,"\n",text,flags=re.MULTILINE).strip()`
	`38`	`+ifself.wrapped_with_comment_pattern:`
	`39`	`+returnre.sub(self.wrapped_with_comment_pattern,"\n",text,flags=re.MULTILINE).strip()`
	`40`	`+`
	`41`	`+returntext`
`38`	`42`
`39`	`43`	`defparse_raw_query(self,text:str,language:str)->RawQueryContainer:`
`40`	`44`	`returnRawQueryContainer(query=text,language=language)`
`@@ -47,7 +51,6 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain`
`47`	`51`	`classPlatformQueryParser(QueryParser,ABC):`
`48`	`52`	`mappings:BasePlatformMappings=None`
`49`	`53`	`tokenizer:QueryTokenizer=None`
`50`		`-details:PlatformDetails=None`
`51`	`54`	`platform_functions:PlatformFunctions=None`
`52`	`55`
`53`	`56`	`defget_fields_tokens(self,tokens:list[Union[FieldValue,Keyword,Identifier]])->list[Field]:`

`‎uncoder-core/app/translator/core/render_cti.py‎`

Lines changed: 2 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@`
`19`	`19`
`20`	`20`
`21`	`21`	`fromapp.translator.core.models.iocsimportIocsChunkValue`
	`22`	`+fromapp.translator.core.models.platform_detailsimportPlatformDetails`
`22`	`23`
`23`	`24`
`24`	`25`	`classRenderCTI:`
`@@ -31,6 +32,7 @@ class RenderCTI:`
`31`	`32`	`final_result_for_many:str="union * \| where ({result})\n"`
`32`	`33`	`final_result_for_one:str="union * \| where {result}\n"`
`33`	`34`	`default_mapping=None`
	`35`	`+details:PlatformDetails=None`
`34`	`36`
`35`	`37`	`defcreate_field_value(self,field:str,value:str,generic_field:str)->str:# noqa: ARG002`
`36`	`38`	`returnself.field_value_template.format(key=field,value=value)`

`‎uncoder-core/app/translator/core/tokenizer.py‎`

Lines changed: 7 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,8 @@ class QueryTokenizer(BaseTokenizer):`
`52`	`52`	`single_value_operators_map:ClassVar[dict[str,str]]= {}`
`53`	`53`	`# used to generate re pattern. so the keys order is important`
`54`	`54`	`multi_value_operators_map:ClassVar[dict[str,str]]= {}`
	`55`	`+# used to generate re pattern. so the keys order is important`
	`56`	`+fields_operator_map:ClassVar[dict[str,str]]= {}`
`55`	`57`	`operators_map:ClassVar[dict[str,str]]= {}# used to generate re pattern. so the keys order is important`
`56`	`58`
`57`	`59`	`logical_operator_pattern=r"^(?P<logical_operator>and\|or\|not\|AND\|OR\|NOT)\s+"`
`@@ -73,7 +75,11 @@ class QueryTokenizer(BaseTokenizer):`
`73`	`75`	`def__init_subclass__(cls,**kwargs):`
`74`	`76`	`cls._validate_re_patterns()`
`75`	`77`	`cls.value_pattern=cls.base_value_pattern.replace("___value_pattern___",cls._value_pattern)`
`76`		`-cls.operators_map= {cls.single_value_operators_map,cls.multi_value_operators_map}`
	`78`	`+cls.operators_map= {`
	`79`	`+**cls.single_value_operators_map,`
	`80`	`+**cls.multi_value_operators_map,`
	`81`	`+**cls.fields_operator_map,`
	`82`	`+ }`
`77`	`83`	`cls.operator_pattern=rf"""(?:___field___\s(?P<operator>(?:{'\|'.join(cls.operators_map)})))\s"""`
`78`	`84`
`79`	`85`	`@classmethod`

`‎uncoder-core/app/translator/managers.py‎`

Lines changed: 43 additions & 41 deletions

Original file line number	Diff line number	Diff line change
`@@ -1,21 +1,16 @@`
`1`	`1`	`fromabcimportABC`
`2`	`2`	`fromfunctoolsimportcached_property`
	`3`	`+fromtypingimportClassVar,Union`
`3`	`4`
`4`	`5`	`fromapp.models.translationimportTranslatorPlatform`
`5`		`-fromapp.translator.core.exceptions.coreimportUnsupportedRootAParser`
	`6`	`+fromapp.translator.core.exceptions.coreimportUnsupportedPlatform,UnsupportedRootAParser`
	`7`	`+fromapp.translator.core.parserimportQueryParser`
	`8`	`+fromapp.translator.core.renderimportQueryRender`
	`9`	`+fromapp.translator.core.render_ctiimportRenderCTI`
`6`	`10`
`7`	`11`
`8`		`-classManager(ABC):`
`9`		`-platforms= {}`
`10`		`-`
`11`		`-defregister(self,cls):`
`12`		`-self.platforms[cls.details.platform_id]=cls()`
`13`		`-returncls`
`14`		`-`
`15`		`-defget(self,platform_id:str):# noqa: ANN201`
`16`		`-ifplatform:=self.platforms.get(platform_id):`
`17`		`-returnplatform`
`18`		`-raiseUnsupportedRootAParser(parser=platform_id)`
	`12`	`+classPlatformManager(ABC):`
	`13`	`+platforms:ClassVar[dict[str,Union[QueryParser,QueryRender,RenderCTI]]]= {}`
`19`	`14`
`20`	`15`	`defall_platforms(self)->list:`
`21`	`16`	`returnlist(self.platforms.keys())`
`@@ -40,54 +35,61 @@ def get_platforms_details(self) -> list[TranslatorPlatform]:`
`40`	`35`	`returnsorted(platforms,key=lambdaplatform:platform.group_name)`
`41`	`36`
`42`	`37`
`43`		`-classParserManager(Manager):`
`44`		`-platforms= {}`
`45`		`-supported_by_roota_platforms= {}`
`46`		`-main_platforms= {}`
	`38`	`+classParserManager(PlatformManager):`
	`39`	`+supported_by_roota_platforms:ClassVar[dict[str,QueryParser]]= {}`
	`40`	`+main_platforms:ClassVar[dict[str,QueryParser]]= {}`
`47`	`41`
`48`		`-defget_supported_by_roota(self,platform_id:str):# noqa: ANN201`
	`42`	`+defget(self,platform_id:str)->QueryParser:`
	`43`	`+ifplatform:=self.platforms.get(platform_id):`
	`44`	`+returnplatform`
	`45`	`+raiseUnsupportedPlatform(platform=platform_id,is_parser=True)`
	`46`	`+`
	`47`	`+defregister(self,cls:type[QueryParser])->type[QueryParser]:`
	`48`	`+self.platforms[cls.details.platform_id]=cls()`
	`49`	`+returncls`
	`50`	`+`
	`51`	`+defget_supported_by_roota(self,platform_id:str)->QueryParser:`
`49`	`52`	`ifplatform:=self.supported_by_roota_platforms.get(platform_id):`
`50`	`53`	`returnplatform`
`51`	`54`	`raiseUnsupportedRootAParser(parser=platform_id)`
`52`	`55`
`53`		`-defregister_supported_by_roota(self,cls):`
	`56`	`+defregister_supported_by_roota(self,cls:type[QueryParser])->type[QueryParser]:`
`54`	`57`	`parser=cls()`
`55`	`58`	`self.supported_by_roota_platforms[cls.details.platform_id]=parser`
`56`	`59`	`self.platforms[cls.details.platform_id]=parser`
`57`	`60`	`returncls`
`58`	`61`
`59`		`-defregister_main(self,cls):`
	`62`	`+defregister_main(self,cls:type[QueryParser])->type[QueryParser]:`
`60`	`63`	`parser=cls()`
`61`	`64`	`self.main_platforms[cls.details.platform_id]=parser`
`62`	`65`	`self.platforms[cls.details.platform_id]=parser`
`63`	`66`	`returncls`
`64`	`67`
`65`		`-@cached_property`
`66`		`-defget_platforms_details(self)->list[TranslatorPlatform]:`
`67`		`-platforms= [`
`68`		`-TranslatorPlatform(`
`69`		`-id=platform.details.platform_id,`
`70`		`-name=platform.details.name,`
`71`		`-code=platform.details.platform_id,`
`72`		`-group_name=platform.details.group_name,`
`73`		`-group_id=platform.details.group_id,`
`74`		`-platform_name=platform.details.platform_name,`
`75`		`-platform_id=platform.details.platform_id,`
`76`		`-alt_platform_name=platform.details.alt_platform_name,`
`77`		`-alt_platform=platform.details.alt_platform,`
`78`		`-first_choice=platform.details.first_choice,`
`79`		`- )`
`80`		`-forplatforminself.platforms.values()`
`81`		`- ]`
`82`		`-returnsorted(platforms,key=lambdaplatform:platform.group_name)`
`83`	`68`
	`69`	`+classRenderManager(PlatformManager):`
	`70`	`+platforms:ClassVar[dict[str,QueryRender]]= {}`
	`71`	`+`
	`72`	`+defget(self,platform_id:str)->QueryRender:`
	`73`	`+ifplatform:=self.platforms.get(platform_id):`
	`74`	`+returnplatform`
	`75`	`+raiseUnsupportedPlatform(platform=platform_id)`
	`76`	`+`
	`77`	`+defregister(self,cls:type[QueryRender])->type[QueryRender]:`
	`78`	`+self.platforms[cls.details.platform_id]=cls()`
	`79`	`+returncls`
`84`	`80`
`85`		`-classRenderManager(Manager):`
`86`		`-platforms= {}`
`87`	`81`
	`82`	`+classRenderCTIManager(PlatformManager):`
	`83`	`+platforms:ClassVar[dict[str,RenderCTI]]= {}`
`88`	`84`
`89`		`-classRenderCTIManager(Manager):`
`90`		`-platforms= {}`
	`85`	`+defget(self,platform_id:str)->RenderCTI:`
	`86`	`+ifplatform:=self.platforms.get(platform_id):`
	`87`	`+returnplatform`
	`88`	`+raiseUnsupportedPlatform(platform=platform_id)`
	`89`	`+`
	`90`	`+defregister(self,cls:type[RenderCTI])->type[RenderCTI]:`
	`91`	`+self.platforms[cls.details.platform_id]=cls()`
	`92`	`+returncls`
`91`	`93`
`92`	`94`
`93`	`95`	`parser_manager=ParserManager()`

`‎uncoder-core/app/translator/platforms/arcsight/init.py‎`

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-fromapp.translator.platforms.arcsight.renders.arcsight_ctiimportArcsightKeyword`
	`1`	`+fromapp.translator.platforms.arcsight.renders.arcsight_ctiimportArcsightKeyword# noqa: F401`

`‎uncoder-core/app/translator/platforms/arcsight/const.py‎`

Lines changed: 8 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,8 @@`
	`1`	`+ARCSIGHT_QUERY_DETAILS= {`
	`2`	`+"platform_id":"arcsight",`
	`3`	`+"name":"ArcSight Query",`
	`4`	`+"group_name":"ArcSight",`
	`5`	`+"group_id":"arcsight",`
	`6`	`+"platform_name":"Query",`
	`7`	`+"alt_platform_name":"CEF",`
	`8`	`+}`

`‎uncoder-core/app/translator/platforms/arcsight/mappings/init.py‎`

Whitespace-only changes.

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Commit6973aa3

File tree

49 files changed

Some content is hidden

49 files changed

`‎uncoder-core/app/translator/core/exceptions/core.py‎`

`‎uncoder-core/app/translator/core/mixins/rule.py‎`

`‎uncoder-core/app/translator/core/models/query_container.py‎`

`‎uncoder-core/app/translator/core/parser.py‎`

`‎uncoder-core/app/translator/core/render_cti.py‎`

`‎uncoder-core/app/translator/core/tokenizer.py‎`

`‎uncoder-core/app/translator/managers.py‎`

`‎uncoder-core/app/translator/platforms/arcsight/init.py‎`

`‎uncoder-core/app/translator/platforms/arcsight/const.py‎`

`‎uncoder-core/app/translator/platforms/arcsight/mappings/init.py‎`

0 commit comments