import unittest from mako import ast, exceptions, pyparser, util, compat from test import eq_, requires_python_2, requires_python_3, \ requires_python_26_or_greater exception_kwargs = { 'source': '', 'lineno': 0, 'pos': 0, 'filename': ''} class AstParseTest(unittest.TestCase): def test_locate_identifiers(self): """test the location of identifiers in a python code string""" code = """ a = 10 b = 5 c = x * 5 + a + b + q (g,h,i) = (1,2,3) [u,k,j] = [4,5,6] foo.hoho.lala.bar = 7 + gah.blah + u + blah for lar in (1,2,3): gh = 5 x = 12 ("hello world, ", a, b) ("Another expr", c) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_( parsed.declared_identifiers, set(['a', 'b', 'c', 'g', 'h', 'i', 'u', 'k', 'j', 'gh', 'lar', 'x']) ) eq_( parsed.undeclared_identifiers, set(['x', 'q', 'foo', 'gah', 'blah']) ) parsed = ast.PythonCode("x + 5 * (y-z)", **exception_kwargs) assert parsed.undeclared_identifiers == set(['x', 'y', 'z']) assert parsed.declared_identifiers == set() def test_locate_identifiers_2(self): code = """ import foobar from lala import hoho, yaya import bleep as foo result = [] data = get_data() for x in data: result.append(x+7) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['get_data'])) eq_( parsed.declared_identifiers, set(['result', 'data', 'x', 'hoho', 'foobar', 'foo', 'yaya']) ) def test_locate_identifiers_3(self): """test that combination assignment/expressions of the same identifier log the ident as 'undeclared'""" code = """ x = x + 5 for y in range(1, y): ("hi",) [z for z in range(1, z)] (q for q in range (1, q)) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_( parsed.undeclared_identifiers, set(['x', 'y', 'z', 'q', 'range']) ) def test_locate_identifiers_4(self): code = """ x = 5 (y, ) def mydef(mydefarg): print("mda is", mydefarg) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['y'])) eq_(parsed.declared_identifiers, set(['mydef', 'x'])) def test_locate_identifiers_5(self): code = """ try: print(x) except: print(y) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['x', 'y'])) def test_locate_identifiers_6(self): code = """ def foo(): return bar() """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['bar'])) code = """ def lala(x, y): return x, y, z print(x) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['z', 'x'])) eq_(parsed.declared_identifiers, set(['lala'])) code = """ def lala(x, y): def hoho(): def bar(): z = 7 print(z) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['z'])) eq_(parsed.declared_identifiers, set(['lala'])) def test_locate_identifiers_7(self): code = """ import foo.bar """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['foo'])) eq_(parsed.undeclared_identifiers, set()) def test_locate_identifiers_8(self): code = """ class Hi(object): foo = 7 def hoho(self): x = 5 """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['Hi'])) eq_(parsed.undeclared_identifiers, set()) def test_locate_identifiers_9(self): code = """ ",".join([t for t in ("a", "b", "c")]) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['t'])) eq_(parsed.undeclared_identifiers, set(['t'])) code = """ [(val, name) for val, name in x] """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['val', 'name'])) eq_(parsed.undeclared_identifiers, set(['val', 'name', 'x'])) def test_locate_identifiers_10(self): code = """ lambda q: q + 5 """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set()) eq_(parsed.undeclared_identifiers, set()) def test_locate_identifiers_11(self): code = """ def x(q): return q + 5 """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['x'])) eq_(parsed.undeclared_identifiers, set()) def test_locate_identifiers_12(self): code = """ def foo(): s = 1 def bar(): t = s """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['foo'])) eq_(parsed.undeclared_identifiers, set()) def test_locate_identifiers_13(self): code = """ def foo(): class Bat(object): pass Bat """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['foo'])) eq_(parsed.undeclared_identifiers, set()) def test_locate_identifiers_14(self): code = """ def foo(): class Bat(object): pass Bat print(Bat) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['foo'])) eq_(parsed.undeclared_identifiers, set(['Bat'])) @requires_python_2 def test_locate_identifiers_15(self): code = """ def t1((x,y)): return x+5, y+4 t2 = lambda (x,y):(x+5, y+4) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.declared_identifiers, set(['t1', 't2'])) eq_(parsed.undeclared_identifiers, set()) @requires_python_26_or_greater def test_locate_identifiers_16(self): code = """ try: print(x) except Exception as e: print(y) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['x', 'y', 'Exception'])) @requires_python_26_or_greater def test_locate_identifiers_17(self): code = """ try: print(x) except (Foo, Bar) as e: print(y) """ parsed = ast.PythonCode(code, **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['x', 'y', 'Foo', 'Bar'])) def test_no_global_imports(self): code = """ from foo import * import x as bar """ self.assertRaises(exceptions.CompileException, ast.PythonCode, code, **exception_kwargs) def test_python_fragment(self): parsed = ast.PythonFragment("for x in foo:", **exception_kwargs) eq_(parsed.declared_identifiers, set(['x'])) eq_(parsed.undeclared_identifiers, set(['foo'])) parsed = ast.PythonFragment("try:", **exception_kwargs) if compat.py3k: parsed = ast.PythonFragment( "except MyException as e:", **exception_kwargs) else: parsed = ast.PythonFragment( "except MyException, e:", **exception_kwargs) eq_(parsed.declared_identifiers, set(['e'])) eq_(parsed.undeclared_identifiers, set(['MyException'])) def test_argument_list(self): parsed = ast.ArgumentList("3, 5, 'hi', x+5, " "context.get('lala')", **exception_kwargs) eq_(parsed.undeclared_identifiers, set(['x', 'context'])) eq_([x for x in parsed.args], ["3", "5", "'hi'", "(x + 5)", "context.get('lala')"]) parsed = ast.ArgumentList("h", **exception_kwargs) eq_(parsed.args, ["h"]) def test_function_decl(self): """test getting the arguments from a function""" code = "def foo(a, b, c=None, d='hi', e=x, f=y+7):pass" parsed = ast.FunctionDecl(code, **exception_kwargs) eq_(parsed.funcname, 'foo') eq_(parsed.argnames, ['a', 'b', 'c', 'd', 'e', 'f']) eq_(parsed.kwargnames, []) def test_function_decl_2(self): """test getting the arguments from a function""" code = "def foo(a, b, c=None, *args, **kwargs):pass" parsed = ast.FunctionDecl(code, **exception_kwargs) eq_(parsed.funcname, 'foo') eq_(parsed.argnames, ['a', 'b', 'c', 'args']) eq_(parsed.kwargnames, ['kwargs']) @requires_python_3 def test_function_decl_3(self): """test getting the arguments from a fancy py3k function""" code = "def foo(a, b, *c, d, e, **f):pass" parsed = ast.FunctionDecl(code, **exception_kwargs) eq_(parsed.funcname, 'foo') eq_(parsed.argnames, ['a', 'b', 'c']) eq_(parsed.kwargnames, ['d', 'e', 'f']) def test_expr_generate(self): """test the round trip of expressions to AST back to python source""" x = 1 y = 2 class F(object): def bar(self, a,b): return a + b def lala(arg): return "blah" + arg local_dict = dict(x=x, y=y, foo=F(), lala=lala) code = "str((x+7*y) / foo.bar(5,6)) + lala('ho')" astnode = pyparser.parse(code) newcode = pyparser.ExpressionGenerator(astnode).value() eq_(eval(code, local_dict), eval(newcode, local_dict)) a = ["one", "two", "three"] hoho = {'somevalue': "asdf"} g = [1, 2, 3, 4, 5] local_dict = dict(a=a, hoho=hoho, g=g) code = "a[2] + hoho['somevalue'] + "\ "repr(g[3:5]) + repr(g[3:]) + repr(g[:5])" astnode = pyparser.parse(code) newcode = pyparser.ExpressionGenerator(astnode).value() eq_(eval(code, local_dict), eval(newcode, local_dict)) local_dict = {'f': lambda: 9, 'x': 7} code = "x+f()" astnode = pyparser.parse(code) newcode = pyparser.ExpressionGenerator(astnode).value() eq_(eval(code, local_dict), eval(newcode, local_dict)) for code in ["repr({'x':7,'y':18})", "repr([])", "repr({})", "repr([{3:[]}])", "repr({'x':37*2 + len([6,7,8])})", "repr([1, 2, {}, {'x':'7'}])", "repr({'x':-1})", "repr(((1,2,3), (4,5,6)))", "repr(1 and 2 and 3 and 4)", "repr(True and False or 55)", "repr(lambda x, y: (x + y))", "repr(lambda *arg, **kw: arg, kw)", "repr(1 & 2 | 3)", "repr(3//5)", "repr(3^5)", "repr([q.endswith('e') for q in " "['one', 'two', 'three']])", "repr([x for x in (5,6,7) if x == 6])", "repr(not False)"]: local_dict = {} astnode = pyparser.parse(code) newcode = pyparser.ExpressionGenerator(astnode).value() if "lambda" in code: eq_(code, newcode) else: eq_(eval(code, local_dict), eval(newcode, local_dict) )