diff options
Diffstat (limited to 'objectivec/DevTools/pddm.py')
| -rwxr-xr-x | objectivec/DevTools/pddm.py | 686 |
1 files changed, 686 insertions, 0 deletions
diff --git a/objectivec/DevTools/pddm.py b/objectivec/DevTools/pddm.py new file mode 100755 index 00000000..9a11fec4 --- /dev/null +++ b/objectivec/DevTools/pddm.py @@ -0,0 +1,686 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2015 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""PDDM - Poor Developers' Debug-able Macros + +A simple markup that can be added in comments of source so they can then be +expanded out into code. Most of this could be done with CPP macros, but then +developers can't really step through them in the debugger, this way they are +expanded to the same code, but you can debug them. + +Any file can be processed, but the syntax is designed around a C based compiler. +Processed lines start with "//%". There are three types of sections you can +create: Text (left alone), Macro Definitions, and Macro Expansions. There is +no order required between definitions and expansions, all definitions are read +before any expansions are processed (thus, if desired, definitions can be put +at the end of the file to keep them out of the way of the code). + +Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines +afterwards that start with "//%" are included in the definition. Multiple +macros can be defined in one block by just using a another "//%PDDM-DEFINE" +line to start the next macro. Optionally, a macro can be ended with +"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that +trailing blank lines are included in the macro. You can also end a definition +with an expansion. + +Macro Expansions are started by single lines containing +"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another +expansions. All lines in-between are replaced by the result of the expansion. +The first line of the expansion is always a blank like just for readability. + +Expansion itself is pretty simple, one macro can invoke another macro, but +you cannot nest the invoke of a macro in another macro (i.e. - can't do +"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() +within its expansion. + +When macros are expanded, the arg references can also add "$O" suffix to the +name (i.e. - "NAME$O") to specify an option to be applied. The options are: + + $S - Replace each character in the value with a space. + $l - Lowercase the first letter of the value. + $L - Lowercase the whole value. + $u - Uppercase the first letter of the value. + $U - Uppercase the whole value. + +Within a macro you can use ## to cause things to get joined together after +expansion (i.e. - "a##b" within a macro will become "ab"). + +Example: + + int foo(MyEnum x) { + switch (x) { + //%PDDM-EXPAND case(Enum_Left, 1) + //%PDDM-EXPAND case(Enum_Center, 2) + //%PDDM-EXPAND case(Enum_Right, 3) + //%PDDM-EXPAND-END + } + + //%PDDM-DEFINE case(_A, _B) + //% case _A: + //% return _B; + + A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END + can be used to avoid adding extra blank lines/returns (or make it clear when + it is desired). + + One macro can invoke another by simply using its name NAME(ARGS). You cannot + nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). + + Within a macro you can use ## to cause things to get joined together after + processing (i.e. - "a##b" within a macro will become "ab"). + + +""" + +import optparse +import os +import re +import sys + + +# Regex for macro definition. +_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)') +# Regex for macro's argument definition. +_MACRO_ARG_NAME_RE = re.compile(r'^\w+$') + +# Line inserted after each EXPAND. +_GENERATED_CODE_LINE = ( + '// This block of code is generated, do not edit it directly.' +) + + +def _MacroRefRe(macro_names): + # Takes in a list of macro names and makes a regex that will match invokes + # of those macros. + return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' % + '|'.join(macro_names)) + +def _MacroArgRefRe(macro_arg_names): + # Takes in a list of macro arg names and makes a regex that will match + # uses of those args. + return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' % + '|'.join(macro_arg_names)) + + +class PDDMError(Exception): + """Error thrown by pddm.""" + pass + + +class MacroCollection(object): + """Hold a set of macros and can resolve/expand them.""" + + def __init__(self, a_file=None): + """Initializes the collection. + + Args: + a_file: The file like stream to parse. + + Raises: + PDDMError if there are any issues. + """ + self._macros = dict() + if a_file: + self.ParseInput(a_file) + + class MacroDefinition(object): + """Holds a macro definition.""" + + def __init__(self, name, arg_names): + self._name = name + self._args = tuple(arg_names) + self._body = '' + self._needNewLine = False + + def AppendLine(self, line): + if self._needNewLine: + self._body += '\n' + self._body += line + self._needNewLine = not line.endswith('\n') + + @property + def name(self): + return self._name + + @property + def args(self): + return self._args + + @property + def body(self): + return self._body + + def ParseInput(self, a_file): + """Consumes input extracting definitions. + + Args: + a_file: The file like stream to parse. + + Raises: + PDDMError if there are any issues. + """ + input_lines = a_file.read().splitlines() + self.ParseLines(input_lines) + + def ParseLines(self, input_lines): + """Parses list of lines. + + Args: + input_lines: A list of strings of input to parse (no newlines on the + strings). + + Raises: + PDDMError if there are any issues. + """ + current_macro = None + for line in input_lines: + if line.startswith('PDDM-'): + directive = line.split(' ', 1)[0] + if directive == 'PDDM-DEFINE': + name, args = self._ParseDefineLine(line) + if self._macros.get(name): + raise PDDMError('Attempt to redefine macro: "%s"' % line) + current_macro = self.MacroDefinition(name, args) + self._macros[name] = current_macro + continue + if directive == 'PDDM-DEFINE-END': + if not current_macro: + raise PDDMError('Got DEFINE-END directive without an active macro:' + ' "%s"' % line) + current_macro = None + continue + raise PDDMError('Hit a line with an unknown directive: "%s"' % line) + + if current_macro: + current_macro.AppendLine(line) + continue + + # Allow blank lines between macro definitions. + if line.strip() == '': + continue + + raise PDDMError('Hit a line that wasn\'t a directive and no open macro' + ' definition: "%s"' % line) + + def _ParseDefineLine(self, input_line): + assert input_line.startswith('PDDM-DEFINE') + line = input_line[12:].strip() + match = _MACRO_RE.match(line) + # Must match full line + if match is None or match.group(0) != line: + raise PDDMError('Failed to parse macro definition: "%s"' % input_line) + name = match.group('name') + args_str = match.group('args').strip() + args = [] + if args_str: + for part in args_str.split(','): + arg = part.strip() + if arg == '': + raise PDDMError('Empty arg name in macro definition: "%s"' + % input_line) + if not _MACRO_ARG_NAME_RE.match(arg): + raise PDDMError('Invalid arg name "%s" in macro definition: "%s"' + % (arg, input_line)) + if arg in args: + raise PDDMError('Arg name "%s" used more than once in macro' + ' definition: "%s"' % (arg, input_line)) + args.append(arg) + return (name, tuple(args)) + + def Expand(self, macro_ref_str): + """Expands the macro reference. + + Args: + macro_ref_str: String of a macro reference (i.e. foo(a, b)). + + Returns: + The text from the expansion. + + Raises: + PDDMError if there are any issues. + """ + match = _MACRO_RE.match(macro_ref_str) + if match is None or match.group(0) != macro_ref_str: + raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str) + if match.group('name') not in self._macros: + raise PDDMError('No macro named "%s".' % match.group('name')) + return self._Expand(match, [], macro_ref_str) + + def _FormatStack(self, macro_ref_stack): + result = '' + for _, macro_ref in reversed(macro_ref_stack): + result += '\n...while expanding "%s".' % macro_ref + return result + + def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None): + if macro_ref_str is None: + macro_ref_str = macro_ref_match.group('macro_ref') + name = macro_ref_match.group('name') + for prev_name, prev_macro_ref in macro_stack: + if name == prev_name: + raise PDDMError('Found macro recusion, invoking "%s":%s' % + (macro_ref_str, self._FormatStack(macro_stack))) + macro = self._macros[name] + args_str = macro_ref_match.group('args').strip() + args = [] + if args_str or len(macro.args): + args = [x.strip() for x in args_str.split(',')] + if len(args) != len(macro.args): + raise PDDMError('Expected %d args, got: "%s".%s' % + (len(macro.args), macro_ref_str, + self._FormatStack(macro_stack))) + # Replace args usages. + result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack) + # Expand any macro invokes. + new_macro_stack = macro_stack + [(name, macro_ref_str)] + while True: + eval_result = self._EvalMacrosRefs(result, new_macro_stack) + # Consume all ## directives to glue things together. + eval_result = eval_result.replace('##', '') + if eval_result == result: + break + result = eval_result + return result + + def _ReplaceArgValues(self, + macro, arg_values, macro_ref_to_report, macro_stack): + if len(arg_values) == 0: + # Nothing to do + return macro.body + assert len(arg_values) == len(macro.args) + args = dict(zip(macro.args, arg_values)) + def _lookupArg(match): + val = args[match.group('name')] + opt = match.group('option') + if opt: + if opt == 'S': # Spaces for the length + return ' ' * len(val) + elif opt == 'l': # Lowercase first character + if val: + return val[0].lower() + val[1:] + else: + return val + elif opt == 'L': # All Lowercase + return val.lower() + elif opt == 'u': # Uppercase first character + if val: + return val[0].upper() + val[1:] + else: + return val + elif opt == 'U': # All Uppercase + return val.upper() + else: + raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s' + % (match.group('name'), match.group('option'), + macro_ref_to_report, + self._FormatStack(macro_stack))) + return val + # Let the regex do the work! + macro_arg_ref_re = _MacroArgRefRe(macro.args) + return macro_arg_ref_re.sub(_lookupArg, macro.body) + + def _EvalMacrosRefs(self, text, macro_stack): + macro_ref_re = _MacroRefRe(self._macros.keys()) + def _resolveMacro(match): + return self._Expand(match, macro_stack) + return macro_ref_re.sub(_resolveMacro, text) + + +class SourceFile(object): + """Represents a source file with PDDM directives in it.""" + + def __init__(self, a_file, import_resolver=None): + """Initializes the file reading in the file. + + Args: + a_file: The file to read in. + import_resolver: a function that given a path will return a stream for + the contents. + + Raises: + PDDMError if there are any issues. + """ + self._sections = [] + self._original_content = a_file.read() + self._import_resolver = import_resolver + self._processed_content = None + + class SectionBase(object): + + def __init__(self, first_line_num): + self._lines = [] + self._first_line_num = first_line_num + + def TryAppend(self, line, line_num): + """Try appending a line. + + Args: + line: The line to append. + line_num: The number of the line. + + Returns: + A tuple of (SUCCESS, CAN_ADD_MORE). If SUCCESS if False, the line + wasn't append. If SUCCESS is True, then CAN_ADD_MORE is True/False to + indicate if more lines can be added after this one. + """ + assert False, "sublcass should have overridden" + return (False, False) + + def HitEOF(self): + """Called when the EOF was reached for for a given section.""" + pass + + def BindMacroCollection(self, macro_collection): + """Binds the chunk to a macro collection. + + Args: + macro_collection: The collection to bind too. + """ + pass + + def Append(self, line): + self._lines.append(line) + + @property + def lines(self): + return self._lines + + @property + def num_lines_captured(self): + return len(self._lines) + + @property + def first_line_num(self): + return self._first_line_num + + @property + def first_line(self): + if not self._lines: + return '' + return self._lines[0] + + @property + def text(self): + return '\n'.join(self.lines) + '\n' + + class TextSection(SectionBase): + """Text section that is echoed out as is.""" + + def TryAppend(self, line, line_num): + if line.startswith('//%PDDM'): + return (False, False) + self.Append(line) + return (True, True) + + class ExpansionSection(SectionBase): + """Section that is the result of an macro expansion.""" + + def __init__(self, first_line_num): + SourceFile.SectionBase.__init__(self, first_line_num) + self._macro_collection = None + + def TryAppend(self, line, line_num): + if line.startswith('//%PDDM'): + directive = line.split(' ', 1)[0] + if directive == '//%PDDM-EXPAND': + self.Append(line) + return (True, True) + if directive == '//%PDDM-EXPAND-END': + assert self.num_lines_captured > 0 + return (True, False) + raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % + (directive, line_num, self.first_line)) + # Eat other lines. + return (True, True) + + def HitEOF(self): + raise PDDMError('Hit the end of the file while in "%s".' % + self.first_line) + + def BindMacroCollection(self, macro_collection): + self._macro_collection = macro_collection + + @property + def lines(self): + captured_lines = SourceFile.SectionBase.lines.fget(self) + directive_len = len('//%PDDM-EXPAND') + result = [] + for line in captured_lines: + result.append(line) + if self._macro_collection: + # Always add a blank line, seems to read better. (If need be, add an + # option to the EXPAND to indicate if this should be done.) + result.extend([_GENERATED_CODE_LINE, '']) + macro = line[directive_len:].strip() + try: + expand_result = self._macro_collection.Expand(macro) + # Since expansions are line oriented, strip trailing whitespace + # from the lines. + lines = [x.rstrip() for x in expand_result.split('\n')] + result.append('\n'.join(lines)) + except PDDMError as e: + raise PDDMError('%s\n...while expanding "%s" from the section' + ' that started:\n Line %d: %s' % + (e.message, macro, + self.first_line_num, self.first_line)) + + # Add the ending marker. + if len(captured_lines) == 1: + result.append('//%%PDDM-EXPAND-END %s' % + captured_lines[0][directive_len:].strip()) + else: + result.append('//%%PDDM-EXPAND-END (%s expansions)' % len(captured_lines)) + + return result + + class DefinitionSection(SectionBase): + """Section containing macro definitions""" + + def TryAppend(self, line, line_num): + if not line.startswith('//%'): + return (False, False) + if line.startswith('//%PDDM'): + directive = line.split(' ', 1)[0] + if directive == "//%PDDM-EXPAND": + return False, False + if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'): + raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % + (directive, line_num, self.first_line)) + self.Append(line) + return (True, True) + + def BindMacroCollection(self, macro_collection): + if macro_collection: + try: + # Parse the lines after stripping the prefix. + macro_collection.ParseLines([x[3:] for x in self.lines]) + except PDDMError as e: + raise PDDMError('%s\n...while parsing section that started:\n' + ' Line %d: %s' % + (e.message, self.first_line_num, self.first_line)) + + class ImportDefinesSection(SectionBase): + """Section containing an import of PDDM-DEFINES from an external file.""" + + def __init__(self, first_line_num, import_resolver): + SourceFile.SectionBase.__init__(self, first_line_num) + self._import_resolver = import_resolver + + def TryAppend(self, line, line_num): + if not line.startswith('//%PDDM-IMPORT-DEFINES '): + return (False, False) + assert self.num_lines_captured == 0 + self.Append(line) + return (True, False) + + def BindMacroCollection(self, macro_colletion): + if not macro_colletion: + return + if self._import_resolver is None: + raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):' + ' "%s".' % (self.first_line_num, self.first_line)) + import_name = self.first_line.split(' ', 1)[1].strip() + imported_file = self._import_resolver(import_name) + if imported_file is None: + raise PDDMError('Resolver failed to find "%s" (line %d):' + ' "%s".' % + (import_name, self.first_line_num, self.first_line)) + try: + imported_src_file = SourceFile(imported_file, self._import_resolver) + imported_src_file._ParseFile() + for section in imported_src_file._sections: + section.BindMacroCollection(macro_colletion) + except PDDMError as e: + raise PDDMError('%s\n...while importing defines:\n' + ' Line %d: %s' % + (e.message, self.first_line_num, self.first_line)) + + def _ParseFile(self): + self._sections = [] + lines = self._original_content.splitlines() + cur_section = None + for line_num, line in enumerate(lines, 1): + if not cur_section: + cur_section = self._MakeSection(line, line_num) + was_added, accept_more = cur_section.TryAppend(line, line_num) + if not was_added: + cur_section = self._MakeSection(line, line_num) + was_added, accept_more = cur_section.TryAppend(line, line_num) + assert was_added + if not accept_more: + cur_section = None + + if cur_section: + cur_section.HitEOF() + + def _MakeSection(self, line, line_num): + if not line.startswith('//%PDDM'): + section = self.TextSection(line_num) + else: + directive = line.split(' ', 1)[0] + if directive == '//%PDDM-EXPAND': + section = self.ExpansionSection(line_num) + elif directive == '//%PDDM-DEFINE': + section = self.DefinitionSection(line_num) + elif directive == '//%PDDM-IMPORT-DEFINES': + section = self.ImportDefinesSection(line_num, self._import_resolver) + else: + raise PDDMError('Unexpected line %d: "%s".' % (line_num, line)) + self._sections.append(section) + return section + + def ProcessContent(self, strip_expansion=False): + """Processes the file contents.""" + self._ParseFile() + if strip_expansion: + # Without a collection the expansions become blank, removing them. + collection = None + else: + collection = MacroCollection() + for section in self._sections: + section.BindMacroCollection(collection) + result = '' + for section in self._sections: + result += section.text + self._processed_content = result + + @property + def original_content(self): + return self._original_content + + @property + def processed_content(self): + return self._processed_content + + +def main(args): + usage = '%prog [OPTIONS] PATH ...' + description = ( + 'Processes PDDM directives in the given paths and write them back out.' + ) + parser = optparse.OptionParser(usage=usage, description=description) + parser.add_option('--dry-run', + default=False, action='store_true', + help='Don\'t write back to the file(s), just report if the' + ' contents needs an update and exit with a value of 1.') + parser.add_option('--verbose', + default=False, action='store_true', + help='Reports is a file is already current.') + parser.add_option('--collapse', + default=False, action='store_true', + help='Removes all the generated code.') + opts, extra_args = parser.parse_args(args) + + if not extra_args: + parser.error('Need atleast one file to process') + + result = 0 + for a_path in extra_args: + if not os.path.exists(a_path): + sys.stderr.write('ERROR: File not found: %s\n' % a_path) + return 100 + + def _ImportResolver(name): + # resolve based on the file being read. + a_dir = os.path.dirname(a_path) + import_path = os.path.join(a_dir, name) + if not os.path.exists(import_path): + return None + return open(import_path, 'r') + + with open(a_path, 'r') as f: + src_file = SourceFile(f, _ImportResolver) + + try: + src_file.ProcessContent(strip_expansion=opts.collapse) + except PDDMError as e: + sys.stderr.write('ERROR: %s\n...While processing "%s"\n' % + (e.message, a_path)) + return 101 + + if src_file.processed_content != src_file.original_content: + if not opts.dry_run: + print 'Updating for "%s".' % a_path + with open(a_path, 'w') as f: + f.write(src_file.processed_content) + else: + # Special result to indicate things need updating. + print 'Update needed for "%s".' % a_path + result = 1 + elif opts.verbose: + print 'No update for "%s".' % a_path + + return result + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) |