path: root/src/proguard/optimize/evaluation/SimpleEnumUseChecker.java

/*
 * ProGuard -- shrinking, optimization, obfuscation, and preverification
 *             of Java bytecode.
 *
 * Copyright (c) 2002-2014 Eric Lafortune (eric@graphics.cornell.edu)
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package proguard.optimize.evaluation;

import proguard.classfile.*;
import proguard.classfile.attribute.*;
import proguard.classfile.attribute.visitor.*;
import proguard.classfile.constant.ClassConstant;
import proguard.classfile.constant.visitor.ConstantVisitor;
import proguard.classfile.instruction.*;
import proguard.classfile.instruction.visitor.InstructionVisitor;
import proguard.classfile.util.*;
import proguard.classfile.visitor.*;
import proguard.evaluation.*;
import proguard.evaluation.value.*;
import proguard.optimize.info.SimpleEnumMarker;

/**
 * This ClassVisitor marks enums that can't be simplified due to the way they
 * are used in the classes that it visits.
 *
 * @see SimpleEnumMarker
 * @author Eric Lafortune
 */
public class SimpleEnumUseChecker
extends      SimplifiedVisitor
implements   ClassVisitor,
             MemberVisitor,
             AttributeVisitor,
             InstructionVisitor,
             ConstantVisitor,
             ParameterVisitor
{
    //*
    private static final boolean DEBUG = false;
    /*/
    private static       boolean DEBUG = System.getProperty("enum") != null;
    //*/

    private final PartialEvaluator       partialEvaluator;
    private final MemberVisitor          methodCodeChecker           = new AllAttributeVisitor(this);
    private final ConstantVisitor        invokedMethodChecker        = new ReferencedMemberVisitor(this);
    private final ConstantVisitor        parameterChecker            = new ReferencedMemberVisitor(new AllParameterVisitor(this));
    private final ClassVisitor           complexEnumMarker           = new SimpleEnumMarker(false);
    private final ReferencedClassVisitor referencedComplexEnumMarker = new ReferencedClassVisitor(complexEnumMarker);


    // Fields acting as parameters and return values for the visitor methods.
    private int invocationOffset;


    /**
     * Creates a new SimpleEnumUseSimplifier.
     */
    public SimpleEnumUseChecker()
    {
        this(new PartialEvaluator());
    }


    /**
     * Creates a new SimpleEnumUseChecker.
     * @param partialEvaluator the partial evaluator that will execute the code
     *                         and provide information about the results.
     */
    public SimpleEnumUseChecker(PartialEvaluator partialEvaluator)
    {
        this.partialEvaluator = partialEvaluator;
    }


    // Implementations for ClassVisitor.

    public void visitProgramClass(ProgramClass programClass)
    {
        if ((programClass.getAccessFlags() & ClassConstants.ACC_ANNOTATTION) != 0)
        {
            // Unmark the simple enum classes in annotations.
            programClass.methodsAccept(referencedComplexEnumMarker);
        }
        else
        {
            // Unmark the simple enum classes that are used in a complex way.
            programClass.methodsAccept(methodCodeChecker);
        }
    }


    // Implementations for AttributeVisitor.

    public void visitAnyAttribute(Clazz clazz, Attribute attribute) {}


    public void visitCodeAttribute(Clazz clazz, Method method, CodeAttribute codeAttribute)
    {
        // Evaluate the method.
        partialEvaluator.visitCodeAttribute(clazz, method, codeAttribute);

        int codeLength = codeAttribute.u4codeLength;

        // Check all traced instructions.
        for (int offset = 0; offset < codeLength; offset++)
        {
            if (partialEvaluator.isTraced(offset))
            {
                Instruction instruction = InstructionFactory.create(codeAttribute.code,
                                                                    offset);

                instruction.accept(clazz, method, codeAttribute, offset, this);

                // Check generalized stacks and variables at branch targets.
                if (partialEvaluator.isBranchOrExceptionTarget(offset))
                {
                    checkMixedStackEntriesBefore(offset);

                    checkMixedVariablesBefore(offset);
                }
            }
        }
    }


    // Implementations for InstructionVisitor.

    public void visitSimpleInstruction(Clazz clazz, Method method, CodeAttribute codeAttribute, int offset, SimpleInstruction simpleInstruction)
    {
        switch (simpleInstruction.opcode)
        {
            case InstructionConstants.OP_AASTORE:
            {
                // Check if the instruction is storing a simple enum in a
                // more general array.
                if (!isPoppingSimpleEnumType(offset, 2))
                {
                    if (DEBUG)
                    {
                        if (isPoppingSimpleEnumType(offset))
                        {
                            System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] stores enum ["+
                                               partialEvaluator.getStackBefore(offset).getTop(0).referenceValue().getType()+"] in more general array ["+
                                               partialEvaluator.getStackBefore(offset).getTop(2).referenceValue().getType()+"]");
                        }
                    }

                    markPoppedComplexEnumType(offset);
                }
                break;
            }
            case InstructionConstants.OP_ARETURN:
            {
                // Check if the instruction is returning a simple enum as a
                // more general type.
                if (!isReturningSimpleEnumType(clazz, method))
                {
                    if (DEBUG)
                    {
                        if (isPoppingSimpleEnumType(offset))
                        {
                            System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] returns enum [" +
                                               partialEvaluator.getStackBefore(offset).getTop(0).referenceValue().getType()+"] as more general type");
                        }
                    }

                    markPoppedComplexEnumType(offset);
                }
                break;
            }
            case InstructionConstants.OP_MONITORENTER:
            case InstructionConstants.OP_MONITOREXIT:
            {
                // Make sure the popped type is not a simple enum type.
                if (DEBUG)
                {
                    if (isPoppingSimpleEnumType(offset))
                    {
                        System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] uses enum ["+
                                           partialEvaluator.getStackBefore(offset).getTop(0).referenceValue().getType()+"] as monitor");
                    }
                }

                markPoppedComplexEnumType(offset);

                break;
            }
        }
    }


    public void visitVariableInstruction(Clazz clazz, Method method, CodeAttribute codeAttribute, int offset, VariableInstruction variableInstruction)
    {
    }


    public void visitConstantInstruction(Clazz clazz, Method method, CodeAttribute codeAttribute, int offset, ConstantInstruction constantInstruction)
    {
        switch (constantInstruction.opcode)
        {
            case InstructionConstants.OP_PUTSTATIC:
            case InstructionConstants.OP_PUTFIELD:
            {
                // Check if the instruction is generalizing a simple enum to a
                // different type.
                invocationOffset = offset;
                clazz.constantPoolEntryAccept(constantInstruction.constantIndex,
                                              parameterChecker);
                break;
            }
            case InstructionConstants.OP_INVOKEVIRTUAL:
            {
                // Check if the instruction is calling a simple enum.
                String invokedMethodName =
                    clazz.getRefName(constantInstruction.constantIndex);
                String invokedMethodType =
                    clazz.getRefType(constantInstruction.constantIndex);
                int stackEntryIndex =
                    ClassUtil.internalMethodParameterSize(invokedMethodType);
                if (isPoppingSimpleEnumType(offset, stackEntryIndex) &&
                    !isSupportedMethod(invokedMethodName,
                                       invokedMethodType))
                {
                    if (DEBUG)
                    {
                        System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] calls ["+partialEvaluator.getStackBefore(offset).getTop(stackEntryIndex).referenceValue().getType()+"."+invokedMethodName+"]");
                    }

                    markPoppedComplexEnumType(offset, stackEntryIndex);
                }

                // Check if any of the parameters is generalizing a simple
                // enum to a different type.
                invocationOffset = offset;
                clazz.constantPoolEntryAccept(constantInstruction.constantIndex,
                                              parameterChecker);
                break;
            }
            case InstructionConstants.OP_INVOKESPECIAL:
            case InstructionConstants.OP_INVOKESTATIC:
            case InstructionConstants.OP_INVOKEINTERFACE:
            {
                // Check if it is calling a method that we can't simplify.
                clazz.constantPoolEntryAccept(constantInstruction.constantIndex,
                                              invokedMethodChecker);

                // Check if any of the parameters is generalizing a simple
                // enum to a different type.
                invocationOffset = offset;
                clazz.constantPoolEntryAccept(constantInstruction.constantIndex,
                                              parameterChecker);
                break;
            }
            case InstructionConstants.OP_CHECKCAST:
            case InstructionConstants.OP_INSTANCEOF:
            {
                // Check if the instruction is popping a different type.
                if (!isPoppingExpectedType(offset,
                                           clazz,
                                           constantInstruction.constantIndex))
                {
                    if (DEBUG)
                    {
                        if (isPoppingSimpleEnumType(offset))
                        {
                            System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] is casting or checking ["+
                                               partialEvaluator.getStackBefore(offset).getTop(0).referenceValue().getType()+"] as ["+
                                               clazz.getClassName(constantInstruction.constantIndex)+"]");
                        }
                    }

                    // Make sure the popped type is not a simple enum type.
                    markPoppedComplexEnumType(offset);

                    // Make sure the checked type is not a simple enum type.
                    // We're somewhat arbitrarily skipping casts in static
                    // methods of simple enum classes, because they do occur
                    // in values() and valueOf(String), without obstructing
                    // simplification.
                    if (!isSimpleEnum(clazz)                                       ||
                        (method.getAccessFlags() & ClassConstants.ACC_STATIC) == 0 ||
                        constantInstruction.opcode != InstructionConstants.OP_CHECKCAST)
                    {
                        if (DEBUG)
                        {
                            if (isSimpleEnum(((ClassConstant)((ProgramClass)clazz).getConstant(constantInstruction.constantIndex)).referencedClass))
                            {
                                System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] is casting or checking ["+
                                                   partialEvaluator.getStackBefore(offset).getTop(0).referenceValue().getType()+"] as ["+
                                                   clazz.getClassName(constantInstruction.constantIndex)+"]");
                            }
                        }

                        markConstantComplexEnumType(clazz, constantInstruction.constantIndex);
                    }
                }
                break;
            }
        }
    }


    public void visitBranchInstruction(Clazz clazz, Method method, CodeAttribute codeAttribute, int offset, BranchInstruction branchInstruction)
    {
        switch (branchInstruction.opcode)
        {
            case InstructionConstants.OP_IFACMPEQ:
            case InstructionConstants.OP_IFACMPNE:
            {
                // Check if the instruction is comparing different types.
                if (!isPoppingIdenticalTypes(offset, 0, 1))
                {
                    if (DEBUG)
                    {
                        if (isPoppingSimpleEnumType(offset, 0))
                        {
                            System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] compares ["+partialEvaluator.getStackBefore(offset).getTop(0).referenceValue().getType()+"] to plain type");
                        }

                        if (isPoppingSimpleEnumType(offset, 1))
                        {
                            System.out.println("SimpleEnumUseChecker: ["+clazz.getName()+"."+method.getName(clazz)+method.getDescriptor(clazz)+"] compares ["+partialEvaluator.getStackBefore(offset).getTop(1).referenceValue().getType()+"] to plain type");
                        }
                    }

                    // Make sure the first popped type is not a simple enum type.
                    markPoppedComplexEnumType(offset, 0);

                    // Make sure the second popped type is not a simple enum type.
                    markPoppedComplexEnumType(offset, 1);
                }
                break;
            }
        }
    }


    public void visitAnySwitchInstruction(Clazz clazz, Method method, CodeAttribute codeAttribute, int offset, SwitchInstruction switchInstruction)
    {
    }


    // Implementations for MemberVisitor.

    public void visitLibraryMethod(LibraryClass libraryClass, LibraryMethod libraryMethod) {}


    public void visitProgramMethod(ProgramClass programClass, ProgramMethod programMethod)
    {
        if (isSimpleEnum(programClass) &&
            isUnsupportedMethod(programMethod.getName(programClass),
                                programMethod.getDescriptor(programClass)))
        {
            if (DEBUG)
            {
                System.out.println("SimpleEnumUseChecker: invocation of ["+programClass.getName()+"."+programMethod.getName(programClass)+programMethod.getDescriptor(programClass)+"]");
            }

            complexEnumMarker.visitProgramClass(programClass);
        }
    }


    // Implementations for ParameterVisitor.

    public void visitParameter(Clazz clazz, Member member, int parameterIndex, int parameterCount, int parameterOffset, int parameterSize, String parameterType, Clazz referencedClass)
    {
        // Check if the parameter is passing a simple enum as a more general
        // type.
        int stackEntryIndex = parameterSize - parameterOffset - 1;
        if (ClassUtil.isInternalClassType(parameterType) &&
            !isPoppingExpectedType(invocationOffset, stackEntryIndex,
                                   ClassUtil.isInternalArrayType(parameterType) ?
                                       parameterType :
                                       ClassUtil.internalClassNameFromClassType(parameterType)))
        {
            if (DEBUG)
            {
                ReferenceValue poppedValue =
                    partialEvaluator.getStackBefore(invocationOffset).getTop(stackEntryIndex).referenceValue();
                if (isSimpleEnumType(poppedValue))
                {
                    System.out.println("SimpleEnumUseChecker: ["+poppedValue.getType()+"] "+
                                       (member instanceof Field ?
                                            ("is stored as more general type ["+parameterType+"] in field ["+clazz.getName()+"."+member.getName(clazz)+"]") :
                                            ("is passed as more general argument #"+parameterIndex+" ["+parameterType+"] to ["+clazz.getName()+"."+member.getName(clazz)+"]")));
                }
            }

            // Make sure the popped type is not a simple enum type.
            markPoppedComplexEnumType(invocationOffset, stackEntryIndex);
        }
    }


    // Small utility methods.

    /**
     * Returns whether the specified enum method is supported for simple enums.
     */
    private boolean isSupportedMethod(String name, String type)
    {
        return
            name.equals(ClassConstants.METHOD_NAME_ORDINAL) &&
            type.equals(ClassConstants.METHOD_TYPE_ORDINAL) ||

            name.equals(ClassConstants.METHOD_NAME_CLONE) &&
            type.equals(ClassConstants.METHOD_TYPE_CLONE);
    }


    /**
     * Returns whether the specified enum method is unsupported for simple enums.
     */
    private boolean isUnsupportedMethod(String name, String type)
    {
        return
            name.equals(ClassConstants.METHOD_NAME_VALUEOF);
    }


    /**
     * Unmarks simple enum classes that are mixed with incompatible reference
     * types in the stack before the given instruction offset.
     */
    private void checkMixedStackEntriesBefore(int offset)
    {
        TracedStack stackBefore = partialEvaluator.getStackBefore(offset);

        // Check all stack entries.
        int stackSize = stackBefore.size();

        for (int stackEntryIndex = 0; stackEntryIndex < stackSize; stackEntryIndex++)
        {
            // Check reference entries.
            Value stackEntry = stackBefore.getBottom(stackEntryIndex);
            if (stackEntry.computationalType() == Value.TYPE_REFERENCE)
            {
                // Check reference entries with multiple producers.
                InstructionOffsetValue producerOffsets =
                    stackBefore.getBottomActualProducerValue(stackEntryIndex).instructionOffsetValue();

                int producerCount = producerOffsets.instructionOffsetCount();
                if (producerCount > 1)
                {
                    // Is the consumed stack entry not a simple enum?
                    ReferenceValue consumedStackEntry =
                        stackEntry.referenceValue();

                    if (!isSimpleEnumType(consumedStackEntry))
                    {
                        // Check all producers.
                        for (int producerIndex = 0; producerIndex < producerCount; producerIndex++)
                        {
                            int producerOffset =
                                producerOffsets.instructionOffset(producerIndex);

                            if (producerOffset >= 0)
                            {
                                if (DEBUG)
                                {
                                    ReferenceValue producedValue =
                                        partialEvaluator.getStackAfter(producerOffset).getTop(0).referenceValue();
                                    if (isSimpleEnumType(producedValue))
                                    {
                                        System.out.println("SimpleEnumUseChecker: ["+producedValue.getType()+"] mixed with general type on stack");
                                    }
                                }

                                // Make sure the produced stack entry isn't a
                                // simple enum either.
                                markPushedComplexEnumType(producerOffset);
                            }
                        }
                    }
                }
            }
        }
    }


    /**
     * Unmarks simple enum classes that are mixed with incompatible reference
     * types in the variables before the given instruction offset.
     */
    private void checkMixedVariablesBefore(int offset)
    {
        TracedVariables variablesBefore =
            partialEvaluator.getVariablesBefore(offset);

        // Check all variables.
        int variablesSize = variablesBefore.size();

        for (int variableIndex = 0; variableIndex < variablesSize; variableIndex++)
        {
            // Check reference variables.
            Value variable = variablesBefore.getValue(variableIndex);
            if (variable != null &&
                variable.computationalType() == Value.TYPE_REFERENCE)
            {
                // Check reference variables with multiple producers.
                InstructionOffsetValue producerOffsets =
                    variablesBefore.getProducerValue(variableIndex).instructionOffsetValue();

                int producerCount = producerOffsets.instructionOffsetCount();
                if (producerCount > 1)
                {
                    // Is the consumed variable not a simple enum?
                    ReferenceValue consumedVariable =
                        variable.referenceValue();

                    if (!isSimpleEnumType(consumedVariable))
                    {
                        // Check all producers.
                        for (int producerIndex = 0; producerIndex < producerCount; producerIndex++)
                        {
                            int producerOffset =
                                producerOffsets.instructionOffset(producerIndex);

                            if (producerOffset >= 0)
                            {
                                if (DEBUG)
                                {
                                    ReferenceValue producedValue =
                                        partialEvaluator.getVariablesAfter(producerOffset).getValue(variableIndex).referenceValue();
                                    if (isSimpleEnumType(producedValue))
                                    {
                                        System.out.println("SimpleEnumUseChecker: ["+producedValue.getType()+"] mixed with general type in variables");
                                    }
                                }

                                // Make sure the stored variable entry isn't a
                                // simple enum either.
                                markStoredComplexEnumType(producerOffset, variableIndex);
                            }
                        }
                    }
                }
            }
        }
    }


    /**
     * Returns whether the instruction at the given offset is popping two
     * identical reference types.
     */
    private boolean isPoppingIdenticalTypes(int offset,
                                            int stackEntryIndex1,
                                            int stackEntryIndex2)
    {
        TracedStack stackBefore = partialEvaluator.getStackBefore(offset);

        String type1 =
            stackBefore.getTop(stackEntryIndex1).referenceValue().getType();
        String type2 =
            stackBefore.getTop(stackEntryIndex2).referenceValue().getType();

        return type1 == null ? type2 == null : type1.equals(type2);
    }


    /**
     * Returns whether the instruction at the given offset is popping exactly
     * the reference type of the specified class constant.
     */
    private boolean isPoppingExpectedType(int   offset,
                                          Clazz clazz,
                                          int   constantIndex)
    {
        return isPoppingExpectedType(offset, 0, clazz, constantIndex);
    }


    /**
     * Returns whether the instruction at the given offset is popping exactly
     * the reference type of the specified class constant.
     */
    private boolean isPoppingExpectedType(int   offset,
                                          int   stackEntryIndex,
                                          Clazz clazz,
                                          int   constantIndex)
    {
        return isPoppingExpectedType(offset,
                                     stackEntryIndex,
                                     clazz.getClassName(constantIndex));
    }


    /**
     * Returns whether the instruction at the given offset is popping exactly
     * the given reference type.
     */
    private boolean isPoppingExpectedType(int    offset,
                                          int    stackEntryIndex,
                                          String expectedType)
    {
        TracedStack stackBefore = partialEvaluator.getStackBefore(offset);

        String poppedType =
            stackBefore.getTop(stackEntryIndex).referenceValue().getType();

        return expectedType.equals(poppedType);
    }


    /**
     * Returns whether the given method is returning a simple enum type.
     * This includes simple enum arrays.
     */
    private boolean isReturningSimpleEnumType(Clazz clazz, Method method)
    {
        String descriptor = method.getDescriptor(clazz);
        String returnType = ClassUtil.internalMethodReturnType(descriptor);

        if (ClassUtil.isInternalClassType(returnType))
        {
            Clazz[] referencedClasses =
                ((ProgramMethod)method).referencedClasses;

            if (referencedClasses != null)
            {
                int returnedClassIndex =
                    new DescriptorClassEnumeration(descriptor).classCount() - 1;

                Clazz returnedClass = referencedClasses[returnedClassIndex];

                return isSimpleEnum(returnedClass);
            }
        }

        return false;
    }


    /**
     * Returns whether the instruction at the given offset is popping a type
     * with a simple enum class. This includes simple enum arrays.
     */
    private boolean isPoppingSimpleEnumType(int offset)
    {
        return isPoppingSimpleEnumType(offset, 0);
    }


    /**
     * Returns whether the instruction at the given offset is popping a type
     * with a simple enum class. This includes simple enum arrays.
     */
    private boolean isPoppingSimpleEnumType(int offset, int stackEntryIndex)
    {
        ReferenceValue referenceValue =
            partialEvaluator.getStackBefore(offset).getTop(stackEntryIndex).referenceValue();

        return isSimpleEnumType(referenceValue);
    }


    /**
     * Returns whether the given value is a simple enum type. This includes
     * simple enum arrays.
     */
    private boolean isSimpleEnumType(ReferenceValue referenceValue)
    {
        return isSimpleEnum(referenceValue.getReferencedClass());
    }


    /**
     * Returns whether the given class is not null and a simple enum class.
     */
    private boolean isSimpleEnum(Clazz clazz)
    {
        return clazz != null &&
               SimpleEnumMarker.isSimpleEnum(clazz);
    }


    /**
     * Marks the enum class of the popped type as complex.
     */
    private void markConstantComplexEnumType(Clazz clazz, int constantIndex)
    {
        clazz.constantPoolEntryAccept(constantIndex,
                                      referencedComplexEnumMarker);
    }


    /**
     * Marks the enum class of the popped type as complex.
     */
    private void markPoppedComplexEnumType(int offset)
    {
        markPoppedComplexEnumType(offset, 0);
    }


    /**
     * Marks the enum class of the specified popped type as complex.
     */
    private void markPoppedComplexEnumType(int offset, int stackEntryIndex)
    {
        ReferenceValue referenceValue =
            partialEvaluator.getStackBefore(offset).getTop(stackEntryIndex).referenceValue();

        markComplexEnumType(referenceValue);
    }


    /**
     * Marks the enum class of the specified pushed type as complex.
     */
    private void markPushedComplexEnumType(int offset)
    {
        ReferenceValue referenceValue =
            partialEvaluator.getStackAfter(offset).getTop(0).referenceValue();

        markComplexEnumType(referenceValue);
    }


    /**
     * Marks the enum class of the specified stored type as complex.
     */
    private void markStoredComplexEnumType(int offset, int variableIndex)
    {
        ReferenceValue referenceValue =
            partialEvaluator.getVariablesAfter(offset).getValue(variableIndex).referenceValue();

        markComplexEnumType(referenceValue);
    }


    /**
     * Marks the enum class of the specified value as complex.
     */
    private void markComplexEnumType(ReferenceValue referenceValue)
    {
        Clazz clazz = referenceValue.getReferencedClass();
        if (clazz != null)
        {
            clazz.accept(complexEnumMarker);
        }
    }
}