| # Copyright 2014 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Helper functions useful when writing scripts that integrate with GN. |
| |
| The main functions are ToGNString() and FromGNString(), to convert between |
| serialized GN veriables and Python variables. |
| |
| To use in an arbitrary Python file in the build: |
| |
| import os |
| import sys |
| |
| sys.path.append(os.path.join(os.path.dirname(__file__), |
| os.pardir, os.pardir, 'build')) |
| import gn_helpers |
| |
| Where the sequence of parameters to join is the relative path from your source |
| file to the build directory. |
| """ |
| |
| import json |
| import os |
| import re |
| import shutil |
| import sys |
| |
| |
| _CHROMIUM_ROOT= os.path.abspath( |
| os.path.join(os.path.dirname(__file__), os.pardir)) |
| |
| ARGS_GN_FILENAME='args.gn' |
| BUILD_VARS_FILENAME='build_vars.json' |
| IMPORT_RE= re.compile(r'^import\("(\S+)"\)') |
| |
| |
| classGNError(Exception): |
| pass |
| |
| |
| # Computes ASCII code of an element of encoded Python 2 str / Python 3 bytes. |
| _Ord= ordif sys.version_info.major<3elselambda c: c |
| |
| |
| def_TranslateToGnChars(s): |
| for decoded_chin s.encode('utf-8'):# str in Python 2, bytes in Python 3. |
| code=_Ord(decoded_ch)# int |
| if codein(34,36,92):# For '"', '$', or '\\'. |
| yield'\\'+ chr(code) |
| elif32<= code<127: |
| yield chr(code) |
| else: |
| yield'$0x%02X'% code |
| |
| |
| defToGNString(value, pretty=False): |
| """Returns a stringified GN equivalent of a Python value. |
| |
| Args: |
| value: The Python value to convert. |
| pretty: Whether to pretty print. If true, then non-empty lists are rendered |
| recursively with one item per line, with indents. Otherwise lists are |
| rendered without new line. |
| Returns: |
| The stringified GN equivalent to |value|. |
| |
| Raises: |
| GNError: |value| cannot be printed to GN. |
| """ |
| |
| if sys.version_info.major<3: |
| basestring_compat= basestring |
| else: |
| basestring_compat= str |
| |
| # Emits all output tokens without intervening whitespaces. |
| defGenerateTokens(v, level): |
| if isinstance(v, basestring_compat): |
| yield'"'+''.join(_TranslateToGnChars(v))+'"' |
| |
| elif isinstance(v, bool): |
| yield'true'if velse'false' |
| |
| elif isinstance(v, int): |
| yield str(v) |
| |
| elif isinstance(v, list): |
| yield'[' |
| for i, itemin enumerate(v): |
| if i>0: |
| yield',' |
| for tokinGenerateTokens(item, level+1): |
| yield tok |
| yield']' |
| |
| elif isinstance(v, dict): |
| if level>0: |
| yield'{' |
| for keyin sorted(v): |
| ifnot isinstance(key, basestring_compat): |
| raiseGNError('Dictionary key is not a string.') |
| ifnot keyor key[0].isdigit()ornot key.replace('_','').isalnum(): |
| raiseGNError('Dictionary key is not a valid GN identifier.') |
| yield key# No quotations. |
| yield'=' |
| for tokinGenerateTokens(v[key], level+1): |
| yield tok |
| if level>0: |
| yield'}' |
| |
| else:# Not supporting float: Add only when needed. |
| raiseGNError('Unsupported type when printing to GN.') |
| |
| can_start=lambda tok: tokand toknotin',}]=' |
| can_end=lambda tok: tokand toknotin',{[=' |
| |
| # Adds whitespaces, trying to keep everything (except dicts) in 1 line. |
| defPlainGlue(gen): |
| prev_tok=None |
| for i, tokin enumerate(gen): |
| if i>0: |
| if can_end(prev_tok)and can_start(tok): |
| yield'\n'# New dict item. |
| elif prev_tok=='['and tok==']': |
| yield' '# Special case for []. |
| elif tok!=',': |
| yield' ' |
| yield tok |
| prev_tok= tok |
| |
| # Adds whitespaces so non-empty lists can span multiple lines, with indent. |
| defPrettyGlue(gen): |
| prev_tok=None |
| level=0 |
| for i, tokin enumerate(gen): |
| if i>0: |
| if can_end(prev_tok)and can_start(tok): |
| yield'\n'+' '* level# New dict item. |
| elif tok=='='or prev_tokin'=': |
| yield' '# Separator before and after '=', on same line. |
| if tokin']}': |
| level-=1 |
| # Exclude '[]' and '{}' cases. |
| if int(prev_tok=='[')+ int(tok==']')==1or \ |
| int(prev_tok=='{')+ int(tok=='}')==1: |
| yield'\n'+' '* level |
| yield tok |
| if tokin'[{': |
| level+=1 |
| if tok==',': |
| yield'\n'+' '* level |
| prev_tok= tok |
| |
| token_gen=GenerateTokens(value,0) |
| ret=''.join((PrettyGlueif prettyelsePlainGlue)(token_gen)) |
| # Add terminating '\n' for dict |value| or multi-line output. |
| if isinstance(value, dict)or'\n'in ret: |
| return ret+'\n' |
| return ret |
| |
| |
| defFromGNString(input_string): |
| """Converts the input string from a GN serialized value to Python values. |
| |
| For details on supported types see GNValueParser.Parse() below. |
| |
| If your GN script did: |
| something = [ "file1", "file2" ] |
| args = [ "--values=$something" ] |
| The command line would look something like: |
| --values="[ \"file1\", \"file2\" ]" |
| Which when interpreted as a command line gives the value: |
| [ "file1", "file2" ] |
| |
| You can parse this into a Python list using GN rules with: |
| input_values = FromGNValues(options.values) |
| Although the Python 'ast' module will parse many forms of such input, it |
| will not handle GN escaping properly, nor GN booleans. You should use this |
| function instead. |
| |
| |
| A NOTE ON STRING HANDLING: |
| |
| If you just pass a string on the command line to your Python script, or use |
| string interpolation on a string variable, the strings will not be quoted: |
| str = "asdf" |
| args = [ str, "--value=$str" ] |
| Will yield the command line: |
| asdf --value=asdf |
| The unquoted asdf string will not be valid input to this function, which |
| accepts only quoted strings like GN scripts. In such cases, you can just use |
| the Python string literal directly. |
| |
| The main use cases for this is for other types, in particular lists. When |
| using string interpolation on a list (as in the top example) the embedded |
| strings will be quoted and escaped according to GN rules so the list can be |
| re-parsed to get the same result. |
| """ |
| parser=GNValueParser(input_string) |
| return parser.Parse() |
| |
| |
| defFromGNArgs(input_string): |
| """Converts a string with a bunch of gn arg assignments into a Python dict. |
| |
| Given a whitespace-separated list of |
| |
| <ident> = (integer | string | boolean | <list of the former>) |
| |
| gn assignments, this returns a Python dict, i.e.: |
| |
| FromGNArgs('foo=true\nbar=1\n') -> { 'foo': True, 'bar': 1 }. |
| |
| Only simple types and lists supported; variables, structs, calls |
| and other, more complicated things are not. |
| |
| This routine is meant to handle only the simple sorts of values that |
| arise in parsing --args. |
| """ |
| parser=GNValueParser(input_string) |
| return parser.ParseArgs() |
| |
| |
| defUnescapeGNString(value): |
| """Given a string with GN escaping, returns the unescaped string. |
| |
| Be careful not to feed with input from a Python parsing function like |
| 'ast' because it will do Python unescaping, which will be incorrect when |
| fed into the GN unescaper. |
| |
| Args: |
| value: Input string to unescape. |
| """ |
| result='' |
| i=0 |
| while i< len(value): |
| if value[i]=='\\': |
| if i< len(value)-1: |
| next_char= value[i+1] |
| if next_charin('$','"','\\'): |
| # These are the escaped characters GN supports. |
| result+= next_char |
| i+=1 |
| else: |
| # Any other backslash is a literal. |
| result+='\\' |
| else: |
| result+= value[i] |
| i+=1 |
| return result |
| |
| |
| def_IsDigitOrMinus(char): |
| return charin'-0123456789' |
| |
| |
| classGNValueParser(object): |
| """Duplicates GN parsing of values and converts to Python types. |
| |
| Normally you would use the wrapper function FromGNValue() below. |
| |
| If you expect input as a specific type, you can also call one of the Parse* |
| functions directly. All functions throw GNError on invalid input. |
| """ |
| |
| def __init__(self, string, checkout_root=_CHROMIUM_ROOT): |
| self.input= string |
| self.cur=0 |
| self.checkout_root= checkout_root |
| |
| defIsDone(self): |
| return self.cur== len(self.input) |
| |
| defReplaceImports(self): |
| """Replaces import(...) lines with the contents of the imports. |
| |
| Recurses on itself until there are no imports remaining, in the case of |
| nested imports. |
| """ |
| lines= self.input.splitlines() |
| ifnot any(line.startswith('import(')for linein lines): |
| return |
| for linein lines: |
| ifnot line.startswith('import('): |
| continue |
| regex_match= IMPORT_RE.match(line) |
| ifnot regex_match: |
| raiseGNError('Not a valid import string: %s'% line) |
| import_path= regex_match.group(1) |
| |
| if import_path.startswith("//"): |
| import_path= os.path.join(self.checkout_root, import_path[2:]) |
| elif sys.platform.startswith('win32'): |
| if import_path.startswith("/"): |
| # gn users '/C:/path/to/foo.gn', not 'C:/path/to/foo.gn' on windows |
| import_path= import_path[1:] |
| else: |
| raiseGNError('Need /-prefix for an absolute path: %s'% import_path) |
| |
| ifnot os.path.isabs(import_path): |
| raiseGNError('Unable to use relative path in import path: %s'% |
| import_path) |
| with open(import_path)as f: |
| imported_args= f.read() |
| self.input= self.input.replace(line, imported_args) |
| # Call ourselves again if we've just replaced an import() with additional |
| # imports. |
| self.ReplaceImports() |
| |
| |
| def_ConsumeWhitespace(self): |
| whilenot self.IsDone()and self.input[self.cur]in' \t\n': |
| self.cur+=1 |
| |
| defConsumeCommentAndWhitespace(self): |
| self._ConsumeWhitespace() |
| |
| # Consume each comment, line by line. |
| whilenot self.IsDone()and self.input[self.cur]=='#': |
| # Consume the rest of the comment, up until the end of the line. |
| whilenot self.IsDone()and self.input[self.cur]!='\n': |
| self.cur+=1 |
| # Move the cursor to the next line (if there is one). |
| ifnot self.IsDone(): |
| self.cur+=1 |
| |
| self._ConsumeWhitespace() |
| |
| defParse(self): |
| """Converts a string representing a printed GN value to the Python type. |
| |
| See additional usage notes on FromGNString() above. |
| |
| * GN booleans ('true', 'false') will be converted to Python booleans. |
| |
| * GN numbers ('123') will be converted to Python numbers. |
| |
| * GN strings (double-quoted as in '"asdf"') will be converted to Python |
| strings with GN escaping rules. GN string interpolation (embedded |
| variables preceded by $) are not supported and will be returned as |
| literals. |
| |
| * GN lists ('[1, "asdf", 3]') will be converted to Python lists. |
| |
| * GN scopes ('{ ... }') are not supported. |
| |
| Raises: |
| GNError: Parse fails. |
| """ |
| result= self._ParseAllowTrailing() |
| self.ConsumeCommentAndWhitespace() |
| ifnot self.IsDone(): |
| raiseGNError("Trailing input after parsing:\n "+ self.input[self.cur:]) |
| return result |
| |
| defParseArgs(self): |
| """Converts a whitespace-separated list of ident=literals to a dict. |
| |
| See additional usage notes on FromGNArgs(), above. |
| |
| Raises: |
| GNError: Parse fails. |
| """ |
| d={} |
| |
| self.ReplaceImports() |
| self.ConsumeCommentAndWhitespace() |
| |
| whilenot self.IsDone(): |
| ident= self._ParseIdent() |
| self.ConsumeCommentAndWhitespace() |
| if self.input[self.cur]!='=': |
| raiseGNError("Unexpected token: "+ self.input[self.cur:]) |
| self.cur+=1 |
| self.ConsumeCommentAndWhitespace() |
| val= self._ParseAllowTrailing() |
| self.ConsumeCommentAndWhitespace() |
| d[ident]= val |
| |
| return d |
| |
| def_ParseAllowTrailing(self): |
| """Internal version of Parse() that doesn't check for trailing stuff.""" |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError("Expected input to parse.") |
| |
| next_char= self.input[self.cur] |
| if next_char=='[': |
| return self.ParseList() |
| elif next_char=='{': |
| return self.ParseScope() |
| elif_IsDigitOrMinus(next_char): |
| return self.ParseNumber() |
| elif next_char=='"': |
| return self.ParseString() |
| elif self._ConstantFollows('true'): |
| returnTrue |
| elif self._ConstantFollows('false'): |
| returnFalse |
| else: |
| raiseGNError("Unexpected token: "+ self.input[self.cur:]) |
| |
| def_ParseIdent(self): |
| ident='' |
| |
| next_char= self.input[self.cur] |
| ifnot next_char.isalpha()andnot next_char=='_': |
| raiseGNError("Expected an identifier: "+ self.input[self.cur:]) |
| |
| ident+= next_char |
| self.cur+=1 |
| |
| next_char= self.input[self.cur] |
| while next_char.isalpha()or next_char.isdigit()or next_char=='_': |
| ident+= next_char |
| self.cur+=1 |
| next_char= self.input[self.cur] |
| |
| return ident |
| |
| defParseNumber(self): |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError('Expected number but got nothing.') |
| |
| begin= self.cur |
| |
| # The first character can include a negative sign. |
| ifnot self.IsDone()and_IsDigitOrMinus(self.input[self.cur]): |
| self.cur+=1 |
| whilenot self.IsDone()and self.input[self.cur].isdigit(): |
| self.cur+=1 |
| |
| number_string= self.input[begin:self.cur] |
| ifnot len(number_string)or number_string=='-': |
| raiseGNError('Not a valid number.') |
| return int(number_string) |
| |
| defParseString(self): |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError('Expected string but got nothing.') |
| |
| if self.input[self.cur]!='"': |
| raiseGNError('Expected string beginning in a " but got:\n '+ |
| self.input[self.cur:]) |
| self.cur+=1# Skip over quote. |
| |
| begin= self.cur |
| whilenot self.IsDone()and self.input[self.cur]!='"': |
| if self.input[self.cur]=='\\': |
| self.cur+=1# Skip over the backslash. |
| if self.IsDone(): |
| raiseGNError('String ends in a backslash in:\n '+ self.input) |
| self.cur+=1 |
| |
| if self.IsDone(): |
| raiseGNError('Unterminated string:\n '+ self.input[begin:]) |
| |
| end= self.cur |
| self.cur+=1# Consume trailing ". |
| |
| returnUnescapeGNString(self.input[begin:end]) |
| |
| defParseList(self): |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError('Expected list but got nothing.') |
| |
| # Skip over opening '['. |
| if self.input[self.cur]!='[': |
| raiseGNError('Expected [ for list but got:\n '+ self.input[self.cur:]) |
| self.cur+=1 |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError('Unterminated list:\n '+ self.input) |
| |
| list_result=[] |
| previous_had_trailing_comma=True |
| whilenot self.IsDone(): |
| if self.input[self.cur]==']': |
| self.cur+=1# Skip over ']'. |
| return list_result |
| |
| ifnot previous_had_trailing_comma: |
| raiseGNError('List items not separated by comma.') |
| |
| list_result+=[ self._ParseAllowTrailing()] |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| break |
| |
| # Consume comma if there is one. |
| previous_had_trailing_comma= self.input[self.cur]==',' |
| if previous_had_trailing_comma: |
| # Consume comma. |
| self.cur+=1 |
| self.ConsumeCommentAndWhitespace() |
| |
| raiseGNError('Unterminated list:\n '+ self.input) |
| |
| defParseScope(self): |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError('Expected scope but got nothing.') |
| |
| # Skip over opening '{'. |
| if self.input[self.cur]!='{': |
| raiseGNError('Expected { for scope but got:\n '+ self.input[self.cur:]) |
| self.cur+=1 |
| self.ConsumeCommentAndWhitespace() |
| if self.IsDone(): |
| raiseGNError('Unterminated scope:\n '+ self.input) |
| |
| scope_result={} |
| whilenot self.IsDone(): |
| if self.input[self.cur]=='}': |
| self.cur+=1 |
| return scope_result |
| |
| ident= self._ParseIdent() |
| self.ConsumeCommentAndWhitespace() |
| if self.input[self.cur]!='=': |
| raiseGNError("Unexpected token: "+ self.input[self.cur:]) |
| self.cur+=1 |
| self.ConsumeCommentAndWhitespace() |
| val= self._ParseAllowTrailing() |
| self.ConsumeCommentAndWhitespace() |
| scope_result[ident]= val |
| |
| raiseGNError('Unterminated scope:\n '+ self.input) |
| |
| def_ConstantFollows(self, constant): |
| """Checks and maybe consumes a string constant at current input location. |
| |
| Param: |
| constant: The string constant to check. |
| |
| Returns: |
| True if |constant| follows immediately at the current location in the |
| input. In this case, the string is consumed as a side effect. Otherwise, |
| returns False and the current position is unchanged. |
| """ |
| end= self.cur+ len(constant) |
| if end> len(self.input): |
| returnFalse# Not enough room. |
| if self.input[self.cur:end]== constant: |
| self.cur= end |
| returnTrue |
| returnFalse |
| |
| |
| defReadBuildVars(output_directory): |
| """Parses $output_directory/build_vars.json into a dict.""" |
| with open(os.path.join(output_directory, BUILD_VARS_FILENAME))as f: |
| return json.load(f) |
| |
| |
| defReadArgsGN(output_directory): |
| """Parses $output_directory/args.gn into a dict.""" |
| fname= os.path.join(output_directory, ARGS_GN_FILENAME) |
| ifnot os.path.exists(fname): |
| return{} |
| with open(fname)as f: |
| returnFromGNArgs(f.read()) |
| |
| |
| defCreateBuildCommand(output_directory): |
| """Returns [cmd, -C, output_directory]. |
| |
| Where |cmd| is one of: siso ninja, ninja, or autoninja. |
| """ |
| suffix='.bat'if sys.platform.startswith('win32')else'' |
| # Prefer the version on PATH, but fallback to known version if PATH doesn't |
| # have one (e.g. on bots). |
| ifnot shutil.which(f'autoninja{suffix}'): |
| third_party_prefix= os.path.join(_CHROMIUM_ROOT,'third_party') |
| ninja_prefix= os.path.join(third_party_prefix,'ninja','') |
| siso_prefix= os.path.join(third_party_prefix,'siso','cipd','') |
| # Also - bots configure reclient manually, and so do not use the "auto" |
| # wrappers. |
| ninja_cmd=[f'{ninja_prefix}ninja{suffix}'] |
| siso_cmd=[f'{siso_prefix}siso{suffix}','ninja'] |
| else: |
| ninja_cmd=[f'autoninja{suffix}'] |
| siso_cmd= list(ninja_cmd) |
| |
| if output_directoryand os.path.abspath(output_directory)!= os.path.abspath( |
| os.curdir): |
| ninja_cmd+=['-C', output_directory] |
| siso_cmd+=['-C', output_directory] |
| siso_deps= os.path.exists(os.path.join(output_directory,'.siso_deps')) |
| ninja_deps= os.path.exists(os.path.join(output_directory,'.ninja_deps')) |
| if siso_depsand ninja_deps: |
| raiseException('Found both .siso_deps and .ninja_deps in ' |
| f'{output_directory}. Not sure which build tool to use. ' |
| 'Please delete one, or better, run "gn clean".') |
| if siso_deps: |
| return siso_cmd |
| return ninja_cmd |
