Issue #3729: Reevaluate tokens in IllegalTokenText

5 files changed, 64 insertions, 73 deletions

diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheck.java b/src/main/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheck.java
index 5960490b8..f6a6b1bae 100644
--- a/src/main/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheck.java
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheck.java
@@ -23,8 +23,8 @@ import java.util.regex.Pattern;
 
 import com.puppycrawl.tools.checkstyle.api.AbstractCheck;
 import com.puppycrawl.tools.checkstyle.api.DetailAST;
+import com.puppycrawl.tools.checkstyle.api.TokenTypes;
 import com.puppycrawl.tools.checkstyle.utils.CommonUtils;
-import com.puppycrawl.tools.checkstyle.utils.TokenUtils;
 
 /**
  * <p>
@@ -82,7 +82,16 @@ public class IllegalTokenTextCheck
 
     @Override
     public int[] getAcceptableTokens() {
-        return TokenUtils.getAllTokenIds();
+        return new int[] {
+            TokenTypes.NUM_DOUBLE,
+            TokenTypes.NUM_FLOAT,
+            TokenTypes.NUM_INT,
+            TokenTypes.NUM_LONG,
+            TokenTypes.IDENT,
+            TokenTypes.COMMENT_CONTENT,
+            TokenTypes.STRING_LITERAL,
+            TokenTypes.CHAR_LITERAL,
+        };
     }
 
     @Override
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/utils/TokenUtils.java b/src/main/java/com/puppycrawl/tools/checkstyle/utils/TokenUtils.java
index 85d3ae043..aad1d5659 100644
--- a/src/main/java/com/puppycrawl/tools/checkstyle/utils/TokenUtils.java
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/utils/TokenUtils.java
@@ -103,6 +103,14 @@ public final class TokenUtils {
     }
 
     /**
+     * Get total number of TokenTypes.
+     * @return total number of TokenTypes.
+     */
+    public static int getTokenTypesTotalNumber() {
+        return TOKEN_IDS.length;
+    }
+
+    /**
      * Get all token IDs that are available in TokenTypes.
      * @return array of token IDs
      */
diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java
index bc4843a23..afb50559f 100644
--- a/src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java
+++ b/src/test/java/com/puppycrawl/tools/checkstyle/checks/coding/IllegalTokenTextCheckTest.java
@@ -23,12 +23,16 @@ import static com.puppycrawl.tools.checkstyle.checks.coding.IllegalTokenTextChec
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 
 import org.junit.Assert;
 import org.junit.Test;
 
 import com.puppycrawl.tools.checkstyle.BaseCheckTestSupport;
 import com.puppycrawl.tools.checkstyle.DefaultConfiguration;
+import com.puppycrawl.tools.checkstyle.api.TokenTypes;
+import com.puppycrawl.tools.checkstyle.utils.TokenUtils;
 
 public class IllegalTokenTextCheckTest
     extends BaseCheckTestSupport {
@@ -122,4 +126,29 @@ public class IllegalTokenTextCheckTest
         verify(checkConfig, getPath("InputIllegalTokens.java"), expected);
     }
 
+    @Test
+    public void testAcceptableTokensMakeSense() {
+        final int expectedTokenTypesTotalNumber = 169;
+        Assert.assertEquals("Total number of TokenTypes has changed, acceptable tokens in"
+                + " IllegalTokenTextCheck need to be reconsidered.",
+            expectedTokenTypesTotalNumber, TokenUtils.getTokenTypesTotalNumber());
+
+        final IllegalTokenTextCheck check = new IllegalTokenTextCheck();
+        final int[] allowedTokens = check.getAcceptableTokens();
+        final List<Integer> tokenTypesWithMutableText = Arrays.asList(
+            TokenTypes.NUM_DOUBLE,
+            TokenTypes.NUM_FLOAT,
+            TokenTypes.NUM_INT,
+            TokenTypes.NUM_LONG,
+            TokenTypes.IDENT,
+            TokenTypes.COMMENT_CONTENT,
+            TokenTypes.STRING_LITERAL,
+            TokenTypes.CHAR_LITERAL
+        );
+        for (int tokenType : allowedTokens) {
+            Assert.assertTrue(TokenUtils.getTokenName(tokenType) + " should not be allowed"
+                + " in this check as its text is a constant (IllegalTokenCheck should be used for"
+                + " such cases).", tokenTypesWithMutableText.contains(tokenType));
+        }
+    }
 }
diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/internal/AllChecksTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/internal/AllChecksTest.java
index cafb4e61b..4ef7b73a5 100644
--- a/src/test/java/com/puppycrawl/tools/checkstyle/internal/AllChecksTest.java
+++ b/src/test/java/com/puppycrawl/tools/checkstyle/internal/AllChecksTest.java
@@ -119,43 +119,9 @@ public class AllChecksTest extends BaseCheckTestSupport {
                         Collectors.toSet()));
         // we have no need to block specific token text
         CHECKSTYLE_TOKENS_IN_CONFIG_TO_IGNORE.put("IllegalTokenText",
-                Stream.of("LITERAL_SUPER", "LITERAL_ASSERT", "ENUM_CONSTANT_DEF",
-                        "TYPE_PARAMETERS", "TYPE_UPPER_BOUNDS", "NUM_DOUBLE", "LITERAL_SWITCH",
-                        "ANNOTATIONS", "LITERAL_SHORT", "LITERAL_PROTECTED", "FOR_CONDITION",
-                        "FOR_INIT", "LITERAL_LONG", "MINUS", "OBJBLOCK", "LITERAL_NULL",
-                        "ANNOTATION", "LITERAL_TRUE", "COMMENT_CONTENT", "LITERAL_CHAR",
-                        "PARAMETER_DEF", "POST_DEC", "ANNOTATION_FIELD_DEF", "BLOCK_COMMENT_END",
-                        "TYPE", "LITERAL_INT", "BSR", "ENUM", "LABELED_STAT",
-                        "ANNOTATION_MEMBER_VALUE_PAIR", "TYPECAST", "LITERAL_SYNCHRONIZED",
-                        "PLUS_ASSIGN", "DOT", "LPAREN", "LITERAL_IF", "LITERAL_CATCH", "BAND",
-                        "INTERFACE_DEF", "LOR", "BNOT", "METHOD_CALL", "AT", "ELLIPSIS",
-                        "ARRAY_INIT", "FOR_EACH_CLAUSE", "LITERAL_THROWS", "CHAR_LITERAL",
-                        "CASE_GROUP", "POST_INC", "SEMI", "LITERAL_FINALLY", "ASSIGN",
-                        "RESOURCE_SPECIFICATION", "STATIC_IMPORT", "GENERIC_START", "IMPORT", "SL",
-                        "VARIABLE_DEF", "LITERAL_DOUBLE", "RCURLY", "RESOURCE", "SR", "COMMA",
-                        "BAND_ASSIGN", "METHOD_DEF", "LITERAL_VOID", "NUM_LONG",
-                        "LITERAL_TRANSIENT", "LITERAL_THIS", "LCURLY", "MINUS_ASSIGN",
-                        "TYPE_LOWER_BOUNDS", "TYPE_ARGUMENT", "LITERAL_CLASS", "INSTANCE_INIT",
-                        "DIV", "LITERAL_VOLATILE", "STAR", "UNARY_MINUS", "FOR_ITERATOR",
-                        "NOT_EQUAL", "LE", "LITERAL_INTERFACE", "LITERAL_FLOAT",
-                        "LITERAL_INSTANCEOF", "BOR_ASSIGN", "LT", "SL_ASSIGN", "ELIST",
-                        "ANNOTATION_ARRAY_INIT", "MODIFIERS", "LITERAL_BREAK", "EXTENDS_CLAUSE",
-                        "TYPE_PARAMETER", "LITERAL_DEFAULT", "STATIC_INIT", "BSR_ASSIGN",
-                        "TYPE_EXTENSION_AND", "BOR", "LITERAL_PRIVATE", "LITERAL_THROW",
-                        "LITERAL_BYTE", "BXOR", "WILDCARD_TYPE", "FINAL", "PARAMETERS", "RPAREN",
-                        "SR_ASSIGN", "UNARY_PLUS", "EMPTY_STAT", "LITERAL_STATIC",
-                        "LITERAL_CONTINUE", "STAR_ASSIGN", "LAMBDA", "RBRACK", "BXOR_ASSIGN",
-                        "CTOR_CALL", "LITERAL_FALSE", "DO_WHILE", "LITERAL_PUBLIC",
-                        "LITERAL_WHILE", "PLUS", "INC", "CTOR_DEF", "GENERIC_END", "DIV_ASSIGN",
-                        "SLIST", "LNOT", "LAND", "LITERAL_ELSE", "ABSTRACT", "STRICTFP",
-                        "QUESTION", "LITERAL_NEW", "LITERAL_RETURN", "SINGLE_LINE_COMMENT",
-                        "INDEX_OP", "EXPR", "BLOCK_COMMENT_BEGIN", "PACKAGE_DEF",
-                        "IMPLEMENTS_CLAUSE", "NUM_FLOAT", "LITERAL_DO", "EOF", "GE", "RESOURCES",
-                        "MOD", "DEC", "EQUAL", "LITERAL_BOOLEAN", "CLASS_DEF", "COLON",
-                        "LITERAL_TRY", "ENUM_DEF", "GT", "NUM_INT", "ANNOTATION_DEF",
-                        "LITERAL_NATIVE", "METHOD_REF", "TYPE_ARGUMENTS", "DOUBLE_COLON", "IDENT",
-                        "MOD_ASSIGN", "LITERAL_FOR", "SUPER_CTOR_CALL", "STRING_LITERAL",
-                        "ARRAY_DECLARATOR", "LITERAL_CASE").collect(Collectors.toSet()));
+                Stream.of("NUM_DOUBLE", "NUM_FLOAT", "NUM_INT", "NUM_LONG", "IDENT",
+                    "COMMENT_CONTENT", "STRING_LITERAL", "CHAR_LITERAL")
+                    .collect(Collectors.toSet()));
         // we do not use this check as it is deprecated
         CHECKSTYLE_TOKENS_IN_CONFIG_TO_IGNORE.put("WriteTag",
                 Stream.of("ENUM_CONSTANT_DEF", "METHOD_DEF", "CTOR_DEF", "ANNOTATION_FIELD_DEF")
@@ -234,37 +200,8 @@ public class AllChecksTest extends BaseCheckTestSupport {
                 .collect(Collectors.toSet()));
         GOOGLE_TOKENS_IN_CONFIG_TO_IGNORE.put("IllegalTokenText", Stream.of(
                 // all other java tokens and text are allowed
-                "LITERAL_SUPER", "LITERAL_ASSERT", "ENUM_CONSTANT_DEF", "TYPE_PARAMETERS",
-                "TYPE_UPPER_BOUNDS", "NUM_DOUBLE", "LITERAL_SWITCH", "ANNOTATIONS",
-                "LITERAL_SHORT", "LITERAL_PROTECTED", "FOR_CONDITION", "FOR_INIT", "LITERAL_LONG",
-                "MINUS", "OBJBLOCK", "LITERAL_NULL", "ANNOTATION", "LITERAL_TRUE",
-                "COMMENT_CONTENT", "LITERAL_CHAR", "PARAMETER_DEF", "POST_DEC",
-                "ANNOTATION_FIELD_DEF", "BLOCK_COMMENT_END", "TYPE", "LITERAL_INT", "BSR", "ENUM",
-                "LABELED_STAT", "ANNOTATION_MEMBER_VALUE_PAIR", "TYPECAST", "LITERAL_SYNCHRONIZED",
-                "PLUS_ASSIGN", "DOT", "LPAREN", "LITERAL_IF", "LITERAL_CATCH", "BAND",
-                "INTERFACE_DEF", "LOR", "BNOT", "METHOD_CALL", "AT", "ELLIPSIS", "ARRAY_INIT",
-                "FOR_EACH_CLAUSE", "LITERAL_THROWS", "CASE_GROUP", "POST_INC", "SEMI",
-                "LITERAL_FINALLY", "ASSIGN", "RESOURCE_SPECIFICATION", "STATIC_IMPORT",
-                "GENERIC_START", "IMPORT", "SL", "VARIABLE_DEF", "LITERAL_DOUBLE", "RCURLY",
-                "RESOURCE", "SR", "COMMA", "BAND_ASSIGN", "METHOD_DEF", "LITERAL_VOID",
-                "NUM_LONG", "LITERAL_TRANSIENT", "LITERAL_THIS", "LCURLY", "MINUS_ASSIGN",
-                "TYPE_LOWER_BOUNDS", "TYPE_ARGUMENT", "LITERAL_CLASS", "INSTANCE_INIT", "DIV",
-                "LITERAL_VOLATILE", "STAR", "UNARY_MINUS", "FOR_ITERATOR", "NOT_EQUAL", "LE",
-                "LITERAL_INTERFACE", "LITERAL_FLOAT", "LITERAL_INSTANCEOF", "BOR_ASSIGN", "LT",
-                "SL_ASSIGN", "ELIST", "ANNOTATION_ARRAY_INIT", "MODIFIERS", "LITERAL_BREAK",
-                "EXTENDS_CLAUSE", "TYPE_PARAMETER", "LITERAL_DEFAULT", "STATIC_INIT", "BSR_ASSIGN",
-                "TYPE_EXTENSION_AND", "BOR", "LITERAL_PRIVATE", "LITERAL_THROW", "LITERAL_BYTE",
-                "BXOR", "WILDCARD_TYPE", "FINAL", "PARAMETERS", "RPAREN", "SR_ASSIGN",
-                "UNARY_PLUS", "EMPTY_STAT", "LITERAL_STATIC", "LITERAL_CONTINUE", "STAR_ASSIGN",
-                "LAMBDA", "RBRACK", "BXOR_ASSIGN", "CTOR_CALL", "LITERAL_FALSE", "DO_WHILE",
-                "LITERAL_PUBLIC", "LITERAL_WHILE", "PLUS", "INC", "CTOR_DEF", "GENERIC_END",
-                "DIV_ASSIGN", "SLIST", "LNOT", "LAND", "LITERAL_ELSE", "ABSTRACT", "STRICTFP",
-                "QUESTION", "LITERAL_NEW", "LITERAL_RETURN", "SINGLE_LINE_COMMENT", "INDEX_OP",
-                "EXPR", "BLOCK_COMMENT_BEGIN", "PACKAGE_DEF", "IMPLEMENTS_CLAUSE", "NUM_FLOAT",
-                "LITERAL_DO", "EOF", "GE", "RESOURCES", "MOD", "DEC", "EQUAL", "LITERAL_BOOLEAN",
-                "CLASS_DEF", "COLON", "LITERAL_TRY", "ENUM_DEF", "GT", "NUM_INT", "ANNOTATION_DEF",
-                "LITERAL_NATIVE", "METHOD_REF", "TYPE_ARGUMENTS", "DOUBLE_COLON", "IDENT",
-                "MOD_ASSIGN", "LITERAL_FOR", "SUPER_CTOR_CALL", "ARRAY_DECLARATOR", "LITERAL_CASE")
+                "NUM_DOUBLE", "NUM_FLOAT", "NUM_INT", "NUM_LONG", "IDENT",
+                "COMMENT_CONTENT", "STRING_LITERAL", "CHAR_LITERAL")
                 .collect(Collectors.toSet()));
         GOOGLE_TOKENS_IN_CONFIG_TO_IGNORE.put("OperatorWrap", Stream.of(
                 // specifically allowed via '4.5.1 Where to break' because the following are
diff --git a/src/xdocs/config_coding.xml b/src/xdocs/config_coding.xml
index 5e49c090e..d02afcf8a 100644
--- a/src/xdocs/config_coding.xml
+++ b/src/xdocs/config_coding.xml
@@ -1654,7 +1654,8 @@ class SomeClass
     <section name="IllegalTokenText">
       <subsection name="Description">
         <p>
-          Checks for illegal token text.
+          Checks specified tokens text for matching an illegal pattern from
+          <code>format</code> property. By default no tokens are specified.
         </p>
       </subsection>
 
@@ -1688,8 +1689,15 @@ class SomeClass
           <tr>
             <td>tokens</td>
             <td>tokens to check</td>
-            <td>subset of tokens <a
-            href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html">TokenTypes</a>.
+            <td>subset of tokens
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_DOUBLE">NUM_DOUBLE</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_FLOAT">NUM_FLOAT</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_INT">NUM_INT</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#NUM_LONG">NUM_LONG</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#IDENT">IDENT</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#COMMENT_CONTENT">COMMENT_CONTENT</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#STRING_LITERAL">STRING_LITERAL</a>,
+                <a href="apidocs/com/puppycrawl/tools/checkstyle/api/TokenTypes.html#CHAR_LITERAL">CHAR_LITERAL</a>.
             </td>
             <td>empty</td>
           </tr>