From 03f1b3ca375368a58cdb917387c241167c0a3766 Mon Sep 17 00:00:00 2001 From: Haibo Huang Date: Mon, 27 Aug 2018 15:40:48 -0700 Subject: Move files in antlr to match upstream directory structure Also update Android.bp to point to the new path. Test: m checkbuild Change-Id: I94322e3bcde0f576914f1c791f41c3091e372cdf --- .../org/antlr/codegen/templates/Python/Python.stg | 1474 ++++++++++++++++++++ 1 file changed, 1474 insertions(+) create mode 100644 tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg (limited to 'tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg') diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg b/tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg new file mode 100644 index 0000000..71c324c --- /dev/null +++ b/tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg @@ -0,0 +1,1474 @@ +/* + [The "BSD license"] + Copyright (c) 2005-2006 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** The API version of the runtime that recognizers generated by this runtime + * need. + */ +apiVersion() ::= "1" + +// System.Boolean.ToString() returns "True" and "False", but the proper C# literals are "true" and "false" +// The Java version of Boolean returns "true" and "false", so they map to themselves here. +booleanLiteral ::= [ + "True":"true", + "False":"false", + "true":"true", + "false":"false", + default:"false" +] + +/** The overall file structure of a recognizer; stores methods for rules + * and cyclic DFAs plus support code. + */ +outputFile(LEXER,PARSER,TREE_PARSER, actionScope, actions, + docComment, recognizer, + name, tokens, tokenNames, rules, cyclicDFAs, + bitsets, buildTemplate, buildAST, rewriteMode, profile, + backtracking, synpreds, memoize, numRules, + fileName, ANTLRVersion, generatedTimestamp, trace, + scopes, superClass, literals) ::= +<< +# $ANTLR + +<@imports> +import sys +from antlr3 import * + +from antlr3.tree import *<\n> + +from antlr3.compat import set, frozenset +<@end> + + + + !> + +# for convenience in actions +HIDDEN = BaseRecognizer.HIDDEN + +# token types +=}; separator="\n"> + + + + + + +def main(argv, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr): + + from antlr3.main import LexerMain + main = LexerMain()<\n> + + + from antlr3.main import ParserMain + main = ParserMain("Lexer", )<\n> + + + from antlr3.main import WalkerMain + main = WalkerMain()<\n> + + main.stdin = stdin + main.stdout = stdout + main.stderr = stderr + main.execute(argv)<\n> + + + + +if __name__ == '__main__': + main(sys.argv) + +>> + +lexer(grammar, name, tokens, scopes, rules, numRules, filterMode, + labelType="CommonToken", superClass="Lexer") ::= << + import }; separator="\n"> + +class (<@superClassName><@end>): + }> + + grammarFileName = "" + api_version = + + def __init__(self}>, input=None, state=None): + if state is None: + state = RecognizerSharedState() + super(, self).__init__(input, state) + + + + self._state.ruleMemo = {} + + + + = (, }>self, input, state)}; separator="\n"> + = }; separator="\n"> + }; separator="\n"> + self.delegates = [}; separator = ", ">] + + }; separator="\n"> + + + + + + + + + + + + + }> + + + + +>> + +/** A override of Lexer.nextToken() that backtracks over mTokens() looking + * for matches. No error can be generated upon error; just rewind, consume + * a token and then try again. backtracking needs to be set as well. + * Make rule memoization happen only at levels above 1 as we start mTokens + * at backtracking==1. + */ +filteringNextToken() ::= << +def nextToken(self): + while True: + if self.input.LA(1) == EOF: + return self.makeEOFToken() + + self._state.token = None + self._state.channel = DEFAULT_CHANNEL + self._state.tokenStartCharIndex = self.input.index() + self._state.tokenStartCharPositionInLine = self.input.charPositionInLine + self._state.tokenStartLine = self.input.line + self._state._text = None + try: + m = self.input.mark() + try: + # means we won't throw slow exception + self._state.backtracking = 1 + try: + self.mTokens() + finally: + self._state.backtracking = 0 + + except BacktrackingFailed: + # mTokens backtracks with synpred at backtracking==2 + # and we set the synpredgate to allow actions at level 1. + self.input.rewind(m) + self.input.consume() # advance one char and try again + + else: + self.emit() + return self._state.token + + except RecognitionException, re: + # shouldn't happen in backtracking mode, but... + self.reportError(re) + self.recover(re) + + +def memoize(self, input, ruleIndex, ruleStartIndex, success): + if self._state.backtracking > 1: + # is Lexer always superclass? + super(, self).memoize(input, ruleIndex, ruleStartIndex, success) + + +def alreadyParsedRule(self, input, ruleIndex): + if self._state.backtracking > 1: + return super(, self).alreadyParsedRule(input, ruleIndex) + return False + + +>> + +actionGate() ::= "self._state.backtracking == 0" + +filteringActionGate() ::= "self._state.backtracking == 1" + +/** How to generate a parser */ + +genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules, + bitsets, inputStreamType, superClass, labelType, members, + rewriteElementType, filterMode, init, ASTLabelType="Object") ::= << + +# token names +tokenNames = [ + "\", "\", "\", "\", + +]<\n> + +from import tokenNames<\n> + +}> + + import }; separator="\n"> + +}> + +class (<@superClassName><@end>): + grammarFileName = "" + api_version = + tokenNames = tokenNames + + def __init__(self}>, input, state=None, *args, **kwargs): + if state is None: + state = RecognizerSharedState() + + <@args()> + super(, self).__init__(input, state, *args, **kwargs) + + + + self._state.ruleMemo = {} + + + + }; separator="\n"> + + }> + }> + + + + = }; separator="\n"> + = (, }>self, input, state)}; separator="\n"> + = .}; separator="\n"> + }; separator="\n"> + self.delegates = [}; separator = ", ">] + + <@init><@end> + + + <@members><@end> + + + + + + + }; separator="\n"> + + }> + + + + _in_ = frozenset([};separator=", ">])<\n>}> + +>> + +delegateRule(ruleDescriptor) ::= << +def (self, ): +<\ > return self..(}; separator=", ">) + + +>> + +parser(grammar, name, scopes, tokens, tokenNames, rules, numRules, bitsets, + ASTLabelType="Object", superClass="Parser", labelType="Token", + members={}, + init={} + ) ::= << + +>> + +/** How to generate a tree parser; same as parser except the input + * stream is a different type. + */ +treeParser(grammar, name, scopes, tokens, tokenNames, globalAction, rules, + numRules, bitsets, filterMode, labelType={}, ASTLabelType="Object", + superClass={TreeRewriterTreeFilterTreeParser}, + members={}, + init={} + ) ::= << + +>> + +/** A simpler version of a rule template that is specific to the imaginary + * rules created for syntactic predicates. As they never have return values + * nor parameters etc..., just give simplest possible method. Don't do + * any of the normal memoization stuff in here either; it's a waste. + * As predicates cannot be inlined into the invoking rule, they need to + * be in a rule by themselves. + */ +synpredRule(ruleName, ruleDescriptor, block, description, nakedBlock) ::= +<< +# $ANTLR start "" +def _fragment(self, ): + + + self.traceIn("_fragment", ) + try: + + + finally: + self.traceOut("_fragment", ) + + + + +# $ANTLR end "" + + +>> + +synpred(name) ::= << +def (self): + self._state.backtracking += 1 + <@start()> + start = self.input.mark() + try: + self._fragment() + except BacktrackingFailed: + success = False + else: + success = True + self.input.rewind(start) + <@stop()> + self._state.backtracking -= 1 + return success + + +>> + +lexerSynpred(name) ::= << + +>> + +ruleMemoization(name) ::= << + +if self._state.backtracking > 0 and self.alreadyParsedRule(self.input, ): + # for cached failed rules, alreadyParsedRule will raise an exception + success = True + return + + +>> + +/** This rule has failed, exit indicating failure during backtrack */ +ruleBacktrackFailure() ::= << + +if self._state.backtracking > 0: + raise BacktrackingFailed + + +>> + +/** How to generate code for a rule. This includes any return type + * data aggregates required for multiple return values. + */ +rule(ruleName,ruleDescriptor,block,emptyRule,description,exceptions,finally,memoize) ::= << + + +# $ANTLR start "" +# : + +def (self, ): + + self.traceIn("", )<\n> + + + + + + <@preamble()> + <@body><@end> + <@postamble()> + return + +# $ANTLR end "" +>> + +ruleBody() ::= << + + +success = False<\n> + + +try: + try: + + + + <(ruleDescriptor.actions.after):execAction()> + + + + success = True<\n> + + + + <\n>}> + + + + + + except RecognitionException, re: + self.reportError(re) + self.recover(self.input, re) + <@setErrorReturnValue()> + + + + finally: + pass + + + +finally: + + self.traceOut("", )<\n> + + + + + pass +>> + +catch(decl,action) ::= << +except : + + +>> + +ruleDeclarations() ::= << + +retval = self._return() +retval.start = self.input.LT(1)<\n> + + = None +}> + + +_StartIndex = self.input.index() + +>> + +ruleScopeSetUp() ::= << +_stack.append(_scope())}; separator="\n"> +_stack.append(_scope())}; separator="\n"> +>> + +ruleScopeCleanUp() ::= << +_stack.pop()}; separator="\n"> +_stack.pop()}; separator="\n"> +>> + +ruleLabelDefs() ::= << +<[ruleDescriptor.tokenLabels,ruleDescriptor.tokenListLabels, + ruleDescriptor.wildcardTreeLabels,ruleDescriptor.wildcardTreeListLabels] + :{it | = None}; separator="\n" +> +<[ruleDescriptor.tokenListLabels,ruleDescriptor.ruleListLabels, + ruleDescriptor.wildcardTreeListLabels] + :{it | list_ = None}; separator="\n" +> +<[ruleDescriptor.ruleLabels,ruleDescriptor.ruleListLabels] + :ruleLabelDef(); separator="\n" +> + = None}; separator="\n"> +>> + +lexerRuleLabelDefs() ::= << +<[ruleDescriptor.tokenLabels, + ruleDescriptor.tokenListLabels, + ruleDescriptor.ruleLabels] + :{it | = None}; separator="\n" +> + = None}; separator="\n"> +<[ruleDescriptor.tokenListLabels, + ruleDescriptor.ruleListLabels] + :{it | list_ = None}; separator="\n" +> +>> + +ruleReturnValue() ::= <% + + + + + +retval + + + +%> + +ruleCleanUp() ::= << + + +retval.stop = self.input.LT(-1)<\n> + + +>> + +memoize() ::= << + + +if self._state.backtracking > 0: + self.memoize(self.input, , _StartIndex, success) + + + +>> + +/** How to generate a rule in the lexer; naked blocks are used for + * fragment rules. + */ +lexerRule(ruleName,nakedBlock,ruleDescriptor,block,memoize) ::= << +# $ANTLR start "" +def m(self, ): + + self.traceIn("", )<\n> + + + + + + success = False<\n> + + + try: + + + + + <\n> + + _type = + _channel = DEFAULT_CHANNEL + + + + + + + self._state.type = _type + self._state.channel = _channel + <(ruleDescriptor.actions.after):execAction()> + + + + success = True<\n> + + + + finally: + + self.traceOut("", )<\n> + + + + pass + +# $ANTLR end "" + + +>> + +/** How to generate code for the implicitly-defined lexer grammar rule + * that chooses between lexer rules. + */ +tokensRule(ruleName,nakedBlock,args,block,ruleDescriptor) ::= << +def mTokens(self): + <\n> + + +>> + +// S U B R U L E S + +/** A (...) subrule with multiple alternatives */ +block(alts,decls,decision,enclosingBlockLevel,blockLevel,decisionNumber,maxK,maxAlt,description) ::= << +# : +alt = + +<@body><@end> +>> + +blockBody() ::= << +<@predecision()> +<@decision><@end> +<@postdecision()> +<@prebranch()> +}; separator="\nel"> +<@postbranch()> +>> + +/** A rule block with multiple alternatives */ +ruleBlock(alts,decls,decision,enclosingBlockLevel,blockLevel,decisionNumber,maxK,maxAlt,description) ::= << +# : +alt = + +<@predecision()> +<@decision><@end> +<@postdecision()> +}; separator="\nel"> +>> + +ruleBlockSingleAlt(alts,decls,decision,enclosingBlockLevel,blockLevel,decisionNumber,description) ::= << +# : + +<@prealt()> + +<@postalt()> +>> + +/** A special case of a (...) subrule with a single alternative */ +blockSingleAlt(alts,decls,decision,enclosingBlockLevel,blockLevel,decisionNumber,description) ::= << +# : + +<@prealt()> + +<@postalt()> +>> + +/** A (..)+ block with 1 or more alternatives */ +positiveClosureBlock(alts,decls,decision,enclosingBlockLevel,blockLevel,decisionNumber,maxK,maxAlt,description) ::= << +# : +cnt = 0 + +<@preloop()> +<@loopBody> + +<@end> +<@postloop()> +>> + +positiveClosureBlockLoop() ::= << +while True: #loop + alt = + <@predecision()> + <@decisionBody><@end> + <@postdecision()> + }; separator="\nel"> + else: + if cnt >= 1: + break #loop + + + eee = EarlyExitException(, self.input) + <@earlyExitException()> + raise eee + + cnt += 1 +>> + +positiveClosureBlockSingleAlt ::= positiveClosureBlock + +/** A (..)* block with 1 or more alternatives */ +closureBlock(alts,decls,decision,enclosingBlockLevel,blockLevel,decisionNumber,maxK,maxAlt,description) ::= << +# : + +<@preloop()> +<@loopBody> + +<@end> +<@postloop()> +>> + +closureBlockLoop() ::= << +while True: #loop + alt = + <@predecision()> + <@decisionBody><@end> + <@postdecision()> + }; separator="\nel"> + else: + break #loop +>> + +closureBlockSingleAlt ::= closureBlock + +/** Optional blocks (x)? are translated to (x|) by before code generation + * so we can just use the normal block template + */ +optionalBlock ::= block + +optionalBlockSingleAlt ::= block + +/** A case in a switch that jumps to an alternative given the alternative + * number. A DFA predicts the alternative and then a simple switch + * does the jump to the code that actually matches that alternative. + */ +altSwitchCase(altNum,alt) ::= << +if alt == : + <@prealt()> + +>> + +/** An alternative is just a list of elements; at outermost level */ +alt(elements,altNum,description,autoAST,outerAlt, treeLevel,rew) ::= << +# : +pass +<@declarations()> + + +<@cleanup()> +>> + +/** What to emit when there is no rewrite. For auto build + * mode, does nothing. + */ +noRewrite(rewriteBlockLevel, treeLevel) ::= "" + +// E L E M E N T S + +/** Dump the elements one per line */ +element(e) ::= << +<@prematch()> +<\n> +>> + +/** match a token optionally with a label in front */ +tokenRef(token,label,elementIndex,terminalOptions) ::= << +