diff options
Diffstat (limited to 'tests/test_c_lexer.py')
| -rw-r--r-- | tests/test_c_lexer.py | 39 |
1 files changed, 36 insertions, 3 deletions
diff --git a/tests/test_c_lexer.py b/tests/test_c_lexer.py index 11c7b26..d63d6fd 100644 --- a/tests/test_c_lexer.py +++ b/tests/test_c_lexer.py @@ -77,6 +77,10 @@ class TestCLexerNoErrors(unittest.TestCase): self.assertTokensTypes('0xf7', ['INT_CONST_HEX']) self.assertTokensTypes('0b110', ['INT_CONST_BIN']) self.assertTokensTypes('0x01202AAbbf7Ul', ['INT_CONST_HEX']) + self.assertTokensTypes("'12'", ['INT_CONST_CHAR']) + self.assertTokensTypes("'123'", ['INT_CONST_CHAR']) + self.assertTokensTypes("'1AB4'", ['INT_CONST_CHAR']) + self.assertTokensTypes(r"'1A\n4'", ['INT_CONST_CHAR']) # no 0 before x, so ID catches it self.assertTokensTypes('xf7', ['ID']) @@ -116,6 +120,7 @@ class TestCLexerNoErrors(unittest.TestCase): self.assertTokensTypes(r"""'\t'""", ['CHAR_CONST']) self.assertTokensTypes(r"""'\''""", ['CHAR_CONST']) self.assertTokensTypes(r"""'\?'""", ['CHAR_CONST']) + self.assertTokensTypes(r"""'\0'""", ['CHAR_CONST']) self.assertTokensTypes(r"""'\012'""", ['CHAR_CONST']) self.assertTokensTypes(r"""'\x2f'""", ['CHAR_CONST']) self.assertTokensTypes(r"""'\x2f12'""", ['CHAR_CONST']) @@ -149,6 +154,24 @@ class TestCLexerNoErrors(unittest.TestCase): self.assertTokensTypes( '"\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123"', ['STRING_LITERAL']) + # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line + # directives with Windows paths as filenames (..\..\dir\file) + self.assertTokensTypes( + r'"\x"', + ['STRING_LITERAL']) + self.assertTokensTypes( + r'"\a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z\A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z"', + ['STRING_LITERAL']) + self.assertTokensTypes( + r'"C:\x\fa\x1e\xited"', + ['STRING_LITERAL']) + # The lexer is permissive and allows decimal escapes (not just octal) + self.assertTokensTypes( + '"jx\9"', + ['STRING_LITERAL']) + self.assertTokensTypes( + '"fo\9999999"', + ['STRING_LITERAL']) def test_mess(self): self.assertTokensTypes( @@ -428,14 +451,24 @@ class TestCLexerErrors(unittest.TestCase): def test_char_constants(self): self.assertLexerError("'", ERR_UNMATCHED_QUOTE) self.assertLexerError("'b\n", ERR_UNMATCHED_QUOTE) - - self.assertLexerError("'jx'", ERR_INVALID_CCONST) + self.assertLexerError("'\\xaa\n'", ERR_UNMATCHED_QUOTE) + + self.assertLexerError(r"'123\12a'", ERR_INVALID_CCONST) + self.assertLexerError(r"'123\xabg'", ERR_INVALID_CCONST) + self.assertLexerError("''", ERR_INVALID_CCONST) + self.assertLexerError("'abcjx'", ERR_INVALID_CCONST) self.assertLexerError(r"'\*'", ERR_INVALID_CCONST) def test_string_literals(self): - self.assertLexerError(r'"jx\9"', ERR_STRING_ESCAPE) + self.assertLexerError(r'"jx\`"', ERR_STRING_ESCAPE) self.assertLexerError(r'"hekllo\* on ix"', ERR_STRING_ESCAPE) self.assertLexerError(r'L"hekllo\* on ix"', ERR_STRING_ESCAPE) + # Should not suffer from slow backtracking + self.assertLexerError(r'"\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\`\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123"', ERR_STRING_ESCAPE) + self.assertLexerError(r'"\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\x23\`\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23\xf1\x23"', ERR_STRING_ESCAPE) + # Should not suffer from slow backtracking when there's no end quote + self.assertLexerError(r'"\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\`\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\123\12\123456', ERR_ILLEGAL_CHAR) + self.assertLexerError(r'"\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\`\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x23\x2\x23456', ERR_ILLEGAL_CHAR) def test_preprocessor(self): self.assertLexerError('#line "ka"', ERR_FILENAME_BEFORE_LINE) |