diff options
author | Justin Klaassen <justinklaassen@google.com> | 2018-04-03 23:21:57 -0400 |
---|---|---|
committer | Justin Klaassen <justinklaassen@google.com> | 2018-04-03 23:21:57 -0400 |
commit | 4d01eeaffaa720e4458a118baa137a11614f00f7 (patch) | |
tree | 66751893566986236788e3c796a7cc5e90d05f52 /android/view | |
parent | a192cc2a132cb0ee8588e2df755563ec7008c179 (diff) | |
download | android-28-4d01eeaffaa720e4458a118baa137a11614f00f7.tar.gz |
Import Android SDK Platform P [4697573]
/google/data/ro/projects/android/fetch_artifact \
--bid 4697573 \
--target sdk_phone_armv7-win_sdk \
sdk-repo-linux-sources-4697573.zip
AndroidVersion.ApiLevel has been modified to appear as 28
Change-Id: If80578c3c657366cc9cf75f8db13d46e2dd4e077
Diffstat (limited to 'android/view')
77 files changed, 8077 insertions, 2784 deletions
diff --git a/android/view/AttachInfo_Accessor.java b/android/view/AttachInfo_Accessor.java index 4445a223..60c13c09 100644 --- a/android/view/AttachInfo_Accessor.java +++ b/android/view/AttachInfo_Accessor.java @@ -16,13 +16,12 @@ package android.view; -import com.android.layoutlib.bridge.android.BridgeWindow; -import com.android.layoutlib.bridge.android.BridgeWindowSession; - import android.content.Context; import android.os.Handler; import android.view.View.AttachInfo; +import com.android.layoutlib.bridge.util.ReflectionUtils; + /** * Class allowing access to package-protected methods/fields. */ @@ -33,8 +32,9 @@ public class AttachInfo_Accessor { WindowManager wm = (WindowManager)context.getSystemService(Context.WINDOW_SERVICE); Display display = wm.getDefaultDisplay(); ViewRootImpl root = new ViewRootImpl(context, display); - AttachInfo info = new AttachInfo(new BridgeWindowSession(), new BridgeWindow(), - display, root, new Handler(), null, context); + AttachInfo info = new AttachInfo(ReflectionUtils.createProxy(IWindowSession.class), + ReflectionUtils.createProxy(IWindow.class), display, root, new Handler(), null, + context); info.mHasWindowFocus = true; info.mWindowVisibility = View.VISIBLE; info.mInTouchMode = false; // this is so that we can display selections. diff --git a/android/view/BridgeInflater.java b/android/view/BridgeInflater.java index 58d8c527..84fd0edd 100644 --- a/android/view/BridgeInflater.java +++ b/android/view/BridgeInflater.java @@ -31,6 +31,8 @@ import com.android.layoutlib.bridge.android.support.RecyclerViewUtil; import com.android.layoutlib.bridge.impl.ParserFactory; import com.android.layoutlib.bridge.util.ReflectionUtils; import com.android.resources.ResourceType; +import com.android.tools.layoutlib.annotations.NotNull; +import com.android.tools.layoutlib.annotations.Nullable; import com.android.util.Pair; import org.xmlpull.v1.XmlPullParser; @@ -45,25 +47,13 @@ import android.widget.ImageView; import android.widget.NumberPicker; import java.io.File; -import java.util.Arrays; -import java.util.Collections; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; - -import static com.android.SdkConstants.AUTO_COMPLETE_TEXT_VIEW; -import static com.android.SdkConstants.BUTTON; -import static com.android.SdkConstants.CHECKED_TEXT_VIEW; -import static com.android.SdkConstants.CHECK_BOX; -import static com.android.SdkConstants.EDIT_TEXT; -import static com.android.SdkConstants.IMAGE_BUTTON; -import static com.android.SdkConstants.IMAGE_VIEW; -import static com.android.SdkConstants.MULTI_AUTO_COMPLETE_TEXT_VIEW; -import static com.android.SdkConstants.RADIO_BUTTON; -import static com.android.SdkConstants.SEEK_BAR; -import static com.android.SdkConstants.SPINNER; -import static com.android.SdkConstants.TEXT_VIEW; +import java.util.function.BiFunction; + import static com.android.layoutlib.bridge.android.BridgeContext.getBaseContext; /** @@ -72,21 +62,7 @@ import static com.android.layoutlib.bridge.android.BridgeContext.getBaseContext; public final class BridgeInflater extends LayoutInflater { private final LayoutlibCallback mLayoutlibCallback; - /** - * If true, the inflater will try to replace the framework widgets with the AppCompat versions. - * Ideally, this should be based on the activity being an AppCompat activity but since that is - * not trivial to check from layoutlib, we currently base the decision on the current theme - * being an AppCompat theme. - */ - private boolean mLoadAppCompatViews; - /** - * This set contains the framework views that have an AppCompat version but failed to load. - * This might happen because not all widgets are contained in all versions of the support - * library. - * This will help us to avoid trying to load the AppCompat version multiple times if it - * doesn't exist. - */ - private Set<String> mFailedAppCompatViews = new HashSet<>(); + private boolean mIsInMerge = false; private ResourceReference mResourceReference; private Map<View, String> mOpenDrawerLayouts; @@ -94,15 +70,6 @@ public final class BridgeInflater extends LayoutInflater { // Keep in sync with the same value in LayoutInflater. private static final int[] ATTRS_THEME = new int[] {com.android.internal.R.attr.theme }; - private static final String APPCOMPAT_WIDGET_PREFIX = "android.support.v7.widget.AppCompat"; - /** List of platform widgets that have an AppCompat version */ - private static final Set<String> APPCOMPAT_VIEWS = Collections.unmodifiableSet( - new HashSet<>( - Arrays.asList(TEXT_VIEW, IMAGE_VIEW, BUTTON, EDIT_TEXT, SPINNER, - IMAGE_BUTTON, CHECK_BOX, RADIO_BUTTON, CHECKED_TEXT_VIEW, - AUTO_COMPLETE_TEXT_VIEW, MULTI_AUTO_COMPLETE_TEXT_VIEW, "RatingBar", - SEEK_BAR))); - /** * List of class prefixes which are tried first by default. * <p/> @@ -113,6 +80,7 @@ public final class BridgeInflater extends LayoutInflater { "android.webkit.", "android.app." }; + private BiFunction<String, AttributeSet, View> mCustomInflater; public static String[] getClassPrefixList() { return sClassPrefixList; @@ -121,13 +89,9 @@ public final class BridgeInflater extends LayoutInflater { private BridgeInflater(LayoutInflater original, Context newContext) { super(original, newContext); newContext = getBaseContext(newContext); - if (newContext instanceof BridgeContext) { - mLayoutlibCallback = ((BridgeContext) newContext).getLayoutlibCallback(); - mLoadAppCompatViews = ((BridgeContext) newContext).isAppCompatTheme(); - } else { - mLayoutlibCallback = null; - mLoadAppCompatViews = false; - } + mLayoutlibCallback = (newContext instanceof BridgeContext) ? + ((BridgeContext) newContext).getLayoutlibCallback() : + null; } /** @@ -140,26 +104,14 @@ public final class BridgeInflater extends LayoutInflater { super(context); mLayoutlibCallback = layoutlibCallback; mConstructorArgs[0] = context; - mLoadAppCompatViews = context.isAppCompatTheme(); } @Override public View onCreateView(String name, AttributeSet attrs) throws ClassNotFoundException { - View view = null; - - try { - if (mLoadAppCompatViews - && APPCOMPAT_VIEWS.contains(name) - && !mFailedAppCompatViews.contains(name)) { - // We are using an AppCompat theme so try to load the appcompat views - view = loadCustomView(APPCOMPAT_WIDGET_PREFIX + name, attrs, true); + View view = createViewFromCustomInflater(name, attrs); - if (view == null) { - mFailedAppCompatViews.add(name); // Do not try this one anymore - } - } - - if (view == null) { + if (view == null) { + try { // First try to find a class using the default Android prefixes for (String prefix : sClassPrefixList) { try { @@ -181,19 +133,19 @@ public final class BridgeInflater extends LayoutInflater { } catch (ClassNotFoundException e) { // Ignore. We'll try again using the custom view loader below. } - } - // Finally try again using the custom view loader - if (view == null) { - view = loadCustomView(name, attrs); + // Finally try again using the custom view loader + if (view == null) { + view = loadCustomView(name, attrs); + } + } catch (InflateException e) { + // Don't catch the InflateException below as that results in hiding the real cause. + throw e; + } catch (Exception e) { + // Wrap the real exception in a ClassNotFoundException, so that the calling method + // can deal with it. + throw new ClassNotFoundException("onCreateView", e); } - } catch (InflateException e) { - // Don't catch the InflateException below as that results in hiding the real cause. - throw e; - } catch (Exception e) { - // Wrap the real exception in a ClassNotFoundException, so that the calling method - // can deal with it. - throw new ClassNotFoundException("onCreateView", e); } setupViewInContext(view, attrs); @@ -201,6 +153,110 @@ public final class BridgeInflater extends LayoutInflater { return view; } + /** + * Finds the createView method in the given customInflaterClass. Since createView is + * currently package protected, it will show in the declared class so we iterate up the + * hierarchy and return the first instance we find. + * The returned method will be accessible. + */ + @NotNull + private static Method getCreateViewMethod(Class<?> customInflaterClass) throws NoSuchMethodException { + Class<?> current = customInflaterClass; + do { + try { + Method method = current.getDeclaredMethod("createView", View.class, String.class, + Context.class, AttributeSet.class, boolean.class, boolean.class, + boolean.class, boolean.class); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignore) { + } + current = current.getSuperclass(); + } while (current != null && current != Object.class); + + throw new NoSuchMethodException(); + } + + /** + * Finds the custom inflater class. If it's defined in the theme, we'll use that one (if the + * class does not exist, null is returned). + * If {@code viewInflaterClass} is not defined in the theme, we'll try to instantiate + * {@code android.support.v7.app.AppCompatViewInflater} + */ + @Nullable + private static Class<?> findCustomInflater(@NotNull BridgeContext bc, + @NotNull LayoutlibCallback layoutlibCallback) { + ResourceValue value = bc.getRenderResources().findItemInTheme("viewInflaterClass", false); + String inflaterName = value != null ? value.getValue() : null; + + if (inflaterName != null) { + try { + return layoutlibCallback.findClass(inflaterName); + } catch (ClassNotFoundException ignore) { + } + + // viewInflaterClass was defined but we couldn't find the class + } else if (bc.isAppCompatTheme()) { + // Older versions of AppCompat do not define the viewInflaterClass so try to get it + // manually + try { + return layoutlibCallback.findClass("android.support.v7.app.AppCompatViewInflater"); + } catch (ClassNotFoundException ignore) { + } + } + + return null; + } + + /** + * Checks if there is a custom inflater and, when present, tries to instantiate the view + * using it. + */ + @Nullable + private View createViewFromCustomInflater(@NotNull String name, @NotNull AttributeSet attrs) { + if (mCustomInflater == null) { + Context context = getContext(); + context = getBaseContext(context); + if (context instanceof BridgeContext) { + BridgeContext bc = (BridgeContext) context; + Class<?> inflaterClass = findCustomInflater(bc, mLayoutlibCallback); + + if (inflaterClass != null) { + try { + Constructor<?> constructor = inflaterClass.getDeclaredConstructor(); + constructor.setAccessible(true); + Object inflater = constructor.newInstance(); + Method method = getCreateViewMethod(inflaterClass); + Context finalContext = context; + mCustomInflater = (viewName, attributeSet) -> { + try { + return (View) method.invoke(inflater, null, viewName, finalContext, + attributeSet, + false, + false /*readAndroidTheme*/, // No need after L + true /*readAppTheme*/, + true /*wrapContext*/); + } catch (IllegalAccessException | InvocationTargetException e) { + assert false : "Call to createView failed"; + } + return null; + }; + } catch (InvocationTargetException | IllegalAccessException | + NoSuchMethodException | InstantiationException ignore) { + } + } + } + + if (mCustomInflater == null) { + // There is no custom inflater. We'll create a nop custom inflater to avoid the + // penalty of trying to instantiate again + mCustomInflater = (s, attributeSet) -> null; + } + } + + return mCustomInflater.apply(name, attrs); + } + @Override public View createViewFromTag(View parent, String name, Context context, AttributeSet attrs, boolean ignoreThemeAttr) { diff --git a/android/view/Display.java b/android/view/Display.java index 5bd7446d..31cfebcc 100644 --- a/android/view/Display.java +++ b/android/view/Display.java @@ -267,21 +267,21 @@ public final class Display { * * @see #getState */ - public static final int STATE_UNKNOWN = 0; + public static final int STATE_UNKNOWN = ViewProtoEnums.DISPLAY_STATE_UNKNOWN; // 0 /** * Display state: The display is off. * * @see #getState */ - public static final int STATE_OFF = 1; + public static final int STATE_OFF = ViewProtoEnums.DISPLAY_STATE_OFF; // 1 /** * Display state: The display is on. * * @see #getState */ - public static final int STATE_ON = 2; + public static final int STATE_ON = ViewProtoEnums.DISPLAY_STATE_ON; // 2 /** * Display state: The display is dozing in a low power state; it is still @@ -291,7 +291,7 @@ public final class Display { * @see #getState * @see android.os.PowerManager#isInteractive */ - public static final int STATE_DOZE = 3; + public static final int STATE_DOZE = ViewProtoEnums.DISPLAY_STATE_DOZE; // 3 /** * Display state: The display is dozing in a suspended low power state; it is still @@ -303,7 +303,7 @@ public final class Display { * @see #getState * @see android.os.PowerManager#isInteractive */ - public static final int STATE_DOZE_SUSPEND = 4; + public static final int STATE_DOZE_SUSPEND = ViewProtoEnums.DISPLAY_STATE_DOZE_SUSPEND; // 4 /** * Display state: The display is on and optimized for VR mode. @@ -311,7 +311,7 @@ public final class Display { * @see #getState * @see android.os.PowerManager#isInteractive */ - public static final int STATE_VR = 5; + public static final int STATE_VR = ViewProtoEnums.DISPLAY_STATE_VR; // 5 /** * Display state: The display is in a suspended full power state; it is still @@ -323,7 +323,7 @@ public final class Display { * @see #getState * @see android.os.PowerManager#isInteractive */ - public static final int STATE_ON_SUSPEND = 6; + public static final int STATE_ON_SUSPEND = ViewProtoEnums.DISPLAY_STATE_ON_SUSPEND; // 6 /* The color mode constants defined below must be kept in sync with the ones in * system/core/include/system/graphics-base.h */ diff --git a/android/view/DisplayCutout.java b/android/view/DisplayCutout.java index a61c8c1d..66a9c6c0 100644 --- a/android/view/DisplayCutout.java +++ b/android/view/DisplayCutout.java @@ -18,15 +18,12 @@ package android.view; import static android.view.DisplayCutoutProto.BOUNDS; import static android.view.DisplayCutoutProto.INSETS; -import static android.view.Surface.ROTATION_0; -import static android.view.Surface.ROTATION_180; -import static android.view.Surface.ROTATION_270; -import static android.view.Surface.ROTATION_90; + +import static com.android.internal.annotations.VisibleForTesting.Visibility.PRIVATE; import android.content.res.Resources; import android.graphics.Matrix; import android.graphics.Path; -import android.graphics.Point; import android.graphics.Rect; import android.graphics.RectF; import android.graphics.Region; @@ -38,19 +35,33 @@ import android.util.PathParser; import android.util.proto.ProtoOutputStream; import com.android.internal.R; +import com.android.internal.annotations.GuardedBy; import com.android.internal.annotations.VisibleForTesting; +import java.util.ArrayList; import java.util.List; /** - * Represents a part of the display that is not functional for displaying content. + * Represents the area of the display that is not functional for displaying content. * * <p>{@code DisplayCutout} is immutable. */ public final class DisplayCutout { private static final String TAG = "DisplayCutout"; + private static final String BOTTOM_MARKER = "@bottom"; private static final String DP_MARKER = "@dp"; + private static final String RIGHT_MARKER = "@right"; + + /** + * Category for overlays that allow emulating a display cutout on devices that don't have + * one. + * + * @see android.content.om.IOverlayManager + * @hide + */ + public static final String EMULATION_OVERLAY_CATEGORY = + "com.android.internal.display_cutout_emulation"; private static final Rect ZERO_RECT = new Rect(); private static final Region EMPTY_REGION = new Region(); @@ -60,7 +71,19 @@ public final class DisplayCutout { * * @hide */ - public static final DisplayCutout NO_CUTOUT = new DisplayCutout(ZERO_RECT, EMPTY_REGION); + public static final DisplayCutout NO_CUTOUT = new DisplayCutout(ZERO_RECT, EMPTY_REGION, + false /* copyArguments */); + + + private static final Object CACHE_LOCK = new Object(); + @GuardedBy("CACHE_LOCK") + private static String sCachedSpec; + @GuardedBy("CACHE_LOCK") + private static int sCachedDisplayWidth; + @GuardedBy("CACHE_LOCK") + private static float sCachedDensity; + @GuardedBy("CACHE_LOCK") + private static DisplayCutout sCachedCutout; private final Rect mSafeInsets; private final Region mBounds; @@ -68,18 +91,34 @@ public final class DisplayCutout { /** * Creates a DisplayCutout instance. * - * NOTE: the Rects passed into this instance are not copied and MUST remain unchanged. + * @param safeInsets the insets from each edge which avoid the display cutout as returned by + * {@link #getSafeInsetTop()} etc. + * @param boundingRects the bounding rects of the display cutouts as returned by + * {@link #getBoundingRects()} ()}. + */ + // TODO(b/73953958): @VisibleForTesting(visibility = PRIVATE) + public DisplayCutout(Rect safeInsets, List<Rect> boundingRects) { + this(safeInsets != null ? new Rect(safeInsets) : ZERO_RECT, + boundingRectsToRegion(boundingRects), + true /* copyArguments */); + } + + /** + * Creates a DisplayCutout instance. * - * @hide + * @param copyArguments if true, create a copy of the arguments. If false, the passed arguments + * are not copied and MUST remain unchanged forever. */ - @VisibleForTesting - public DisplayCutout(Rect safeInsets, Region bounds) { - mSafeInsets = safeInsets != null ? safeInsets : ZERO_RECT; - mBounds = bounds != null ? bounds : Region.obtain(); + private DisplayCutout(Rect safeInsets, Region bounds, boolean copyArguments) { + mSafeInsets = safeInsets == null ? ZERO_RECT : + (copyArguments ? new Rect(safeInsets) : safeInsets); + mBounds = bounds == null ? Region.obtain() : + (copyArguments ? Region.obtain(bounds) : bounds); } /** - * Returns true if there is no cutout or it is outside of the content view. + * Returns true if the safe insets are empty (and therefore the current view does not + * overlap with the cutout or cutout area). * * @hide */ @@ -87,28 +126,37 @@ public final class DisplayCutout { return mSafeInsets.equals(ZERO_RECT); } - /** Returns the inset from the top which avoids the display cutout. */ + /** + * Returns true if there is no cutout, i.e. the bounds are empty. + * + * @hide + */ + public boolean isBoundsEmpty() { + return mBounds.isEmpty(); + } + + /** Returns the inset from the top which avoids the display cutout in pixels. */ public int getSafeInsetTop() { return mSafeInsets.top; } - /** Returns the inset from the bottom which avoids the display cutout. */ + /** Returns the inset from the bottom which avoids the display cutout in pixels. */ public int getSafeInsetBottom() { return mSafeInsets.bottom; } - /** Returns the inset from the left which avoids the display cutout. */ + /** Returns the inset from the left which avoids the display cutout in pixels. */ public int getSafeInsetLeft() { return mSafeInsets.left; } - /** Returns the inset from the right which avoids the display cutout. */ + /** Returns the inset from the right which avoids the display cutout in pixels. */ public int getSafeInsetRight() { return mSafeInsets.right; } /** - * Returns the safe insets in a rect. + * Returns the safe insets in a rect in pixel units. * * @return a rect which is set to the safe insets. * @hide @@ -120,23 +168,60 @@ public final class DisplayCutout { /** * Returns the bounding region of the cutout. * + * <p> + * <strong>Note:</strong> There may be more than one cutout, in which case the returned + * {@code Region} will be non-contiguous and its bounding rect will be meaningless without + * intersecting it first. + * + * Example: + * <pre> + * // Getting the bounding rectangle of the top display cutout + * Region bounds = displayCutout.getBounds(); + * bounds.op(0, 0, Integer.MAX_VALUE, displayCutout.getSafeInsetTop(), Region.Op.INTERSECT); + * Rect topDisplayCutout = bounds.getBoundingRect(); + * </pre> + * * @return the bounding region of the cutout. Coordinates are relative - * to the top-left corner of the content view. + * to the top-left corner of the content view and in pixel units. + * @hide */ public Region getBounds() { return Region.obtain(mBounds); } /** - * Returns the bounding rect of the cutout. + * Returns a list of {@code Rect}s, each of which is the bounding rectangle for a non-functional + * area on the display. * - * @return the bounding rect of the cutout. Coordinates are relative - * to the top-left corner of the content view. - * @hide + * There will be at most one non-functional area per short edge of the device, and none on + * the long edges. + * + * @return a list of bounding {@code Rect}s, one for each display cutout area. */ - public Rect getBoundingRect() { - // TODO(roosa): Inline. - return mBounds.getBounds(); + public List<Rect> getBoundingRects() { + List<Rect> result = new ArrayList<>(); + Region bounds = Region.obtain(); + // top + bounds.set(mBounds); + bounds.op(0, 0, Integer.MAX_VALUE, getSafeInsetTop(), Region.Op.INTERSECT); + if (!bounds.isEmpty()) { + result.add(bounds.getBounds()); + } + // left + bounds.set(mBounds); + bounds.op(0, 0, getSafeInsetLeft(), Integer.MAX_VALUE, Region.Op.INTERSECT); + if (!bounds.isEmpty()) { + result.add(bounds.getBounds()); + } + // right & bottom + bounds.set(mBounds); + bounds.op(getSafeInsetLeft() + 1, getSafeInsetTop() + 1, + Integer.MAX_VALUE, Integer.MAX_VALUE, Region.Op.INTERSECT); + if (!bounds.isEmpty()) { + result.add(bounds.getBounds()); + } + bounds.recycle(); + return result; } @Override @@ -162,7 +247,7 @@ public final class DisplayCutout { @Override public String toString() { return "DisplayCutout{insets=" + mSafeInsets - + " boundingRect=" + getBoundingRect() + + " boundingRect=" + mBounds.getBounds() + "}"; } @@ -207,74 +292,19 @@ public final class DisplayCutout { } bounds.translate(-insetLeft, -insetTop); - - return new DisplayCutout(safeInsets, bounds); + return new DisplayCutout(safeInsets, bounds, false /* copyArguments */); } /** - * Calculates the safe insets relative to the given reference frame. + * Returns a copy of this instance with the safe insets replaced with the parameter. + * + * @param safeInsets the new safe insets in pixels + * @return a copy of this instance with the safe insets replaced with the argument. * - * @return a copy of this instance with the safe insets calculated * @hide */ - public DisplayCutout calculateRelativeTo(Rect frame) { - if (mBounds.isEmpty() || !Rect.intersects(frame, mBounds.getBounds())) { - return NO_CUTOUT; - } - - return DisplayCutout.calculateRelativeTo(frame, Region.obtain(mBounds)); - } - - private static DisplayCutout calculateRelativeTo(Rect frame, Region bounds) { - Rect boundingRect = bounds.getBounds(); - Rect safeRect = new Rect(); - - int bestArea = 0; - int bestVariant = 0; - for (int variant = ROTATION_0; variant <= ROTATION_270; variant++) { - int area = calculateInsetVariantArea(frame, boundingRect, variant, safeRect); - if (bestArea < area) { - bestArea = area; - bestVariant = variant; - } - } - calculateInsetVariantArea(frame, boundingRect, bestVariant, safeRect); - if (safeRect.isEmpty()) { - // The entire frame overlaps with the cutout. - safeRect.set(0, frame.height(), 0, 0); - } else { - // Convert safeRect to insets relative to frame. We're reusing the rect here to avoid - // an allocation. - safeRect.set( - Math.max(0, safeRect.left - frame.left), - Math.max(0, safeRect.top - frame.top), - Math.max(0, frame.right - safeRect.right), - Math.max(0, frame.bottom - safeRect.bottom)); - } - - bounds.translate(-frame.left, -frame.top); - - return new DisplayCutout(safeRect, bounds); - } - - private static int calculateInsetVariantArea(Rect frame, Rect boundingRect, int variant, - Rect outSafeRect) { - switch (variant) { - case ROTATION_0: - outSafeRect.set(frame.left, frame.top, frame.right, boundingRect.top); - break; - case ROTATION_90: - outSafeRect.set(frame.left, frame.top, boundingRect.left, frame.bottom); - break; - case ROTATION_180: - outSafeRect.set(frame.left, boundingRect.bottom, frame.right, frame.bottom); - break; - case ROTATION_270: - outSafeRect.set(boundingRect.right, frame.top, frame.right, frame.bottom); - break; - } - - return outSafeRect.isEmpty() ? 0 : outSafeRect.width() * outSafeRect.height(); + public DisplayCutout replaceSafeInsets(Rect safeInsets) { + return new DisplayCutout(new Rect(safeInsets), mBounds, false /* copyArguments */); } private static int atLeastZero(int value) { @@ -283,21 +313,17 @@ public final class DisplayCutout { /** - * Creates an instance from a bounding polygon. + * Creates an instance from a bounding rect. * * @hide */ - public static DisplayCutout fromBoundingPolygon(List<Point> points) { + public static DisplayCutout fromBoundingRect(int left, int top, int right, int bottom) { Path path = new Path(); path.reset(); - for (int i = 0; i < points.size(); i++) { - Point point = points.get(i); - if (i == 0) { - path.moveTo(point.x, point.y); - } else { - path.lineTo(point.x, point.y); - } - } + path.moveTo(left, top); + path.lineTo(left, bottom); + path.lineTo(right, bottom); + path.lineTo(right, top); path.close(); return fromBounds(path); } @@ -317,7 +343,7 @@ public final class DisplayCutout { Region bounds = new Region(); bounds.setPath(path, clipRegion); clipRegion.recycle(); - return new DisplayCutout(ZERO_RECT, bounds); + return new DisplayCutout(ZERO_RECT, bounds, false /* copyArguments */); } /** @@ -325,18 +351,49 @@ public final class DisplayCutout { * * @hide */ - public static DisplayCutout fromResources(Resources res, int displayWidth) { - String spec = res.getString(R.string.config_mainBuiltInDisplayCutout); + public static DisplayCutout fromResources(Resources res, int displayWidth, int displayHeight) { + return fromSpec(res.getString(R.string.config_mainBuiltInDisplayCutout), + displayWidth, displayHeight, res.getDisplayMetrics().density); + } + + /** + * Creates an instance according to the supplied {@link android.util.PathParser.PathData} spec. + * + * @hide + */ + @VisibleForTesting(visibility = PRIVATE) + public static DisplayCutout fromSpec(String spec, int displayWidth, int displayHeight, + float density) { if (TextUtils.isEmpty(spec)) { return null; } + synchronized (CACHE_LOCK) { + if (spec.equals(sCachedSpec) && sCachedDisplayWidth == displayWidth + && sCachedDensity == density) { + return sCachedCutout; + } + } spec = spec.trim(); + final float offsetX; + if (spec.endsWith(RIGHT_MARKER)) { + offsetX = displayWidth; + spec = spec.substring(0, spec.length() - RIGHT_MARKER.length()).trim(); + } else { + offsetX = displayWidth / 2f; + } final boolean inDp = spec.endsWith(DP_MARKER); if (inDp) { spec = spec.substring(0, spec.length() - DP_MARKER.length()); } - Path p; + String bottomSpec = null; + if (spec.contains(BOTTOM_MARKER)) { + String[] splits = spec.split(BOTTOM_MARKER, 2); + spec = splits[0].trim(); + bottomSpec = splits[1].trim(); + } + + final Path p; try { p = PathParser.createPathFromPathData(spec); } catch (Throwable e) { @@ -346,12 +403,43 @@ public final class DisplayCutout { final Matrix m = new Matrix(); if (inDp) { - final float dpToPx = res.getDisplayMetrics().density; - m.postScale(dpToPx, dpToPx); + m.postScale(density, density); } - m.postTranslate(displayWidth / 2f, 0); + m.postTranslate(offsetX, 0); p.transform(m); - return fromBounds(p); + + if (bottomSpec != null) { + final Path bottomPath; + try { + bottomPath = PathParser.createPathFromPathData(bottomSpec); + } catch (Throwable e) { + Log.wtf(TAG, "Could not inflate bottom cutout: ", e); + return null; + } + // Keep top transform + m.postTranslate(0, displayHeight); + bottomPath.transform(m); + p.addPath(bottomPath); + } + + final DisplayCutout result = fromBounds(p); + synchronized (CACHE_LOCK) { + sCachedSpec = spec; + sCachedDisplayWidth = displayWidth; + sCachedDensity = density; + sCachedCutout = result; + } + return result; + } + + private static Region boundingRectsToRegion(List<Rect> rects) { + Region result = Region.obtain(); + if (rects != null) { + for (Rect r : rects) { + result.op(r, Region.Op.UNION); + } + } + return result; } /** @@ -439,7 +527,7 @@ public final class DisplayCutout { Rect safeInsets = in.readTypedObject(Rect.CREATOR); Region bounds = in.readTypedObject(Region.CREATOR); - return new DisplayCutout(safeInsets, bounds); + return new DisplayCutout(safeInsets, bounds, false /* copyArguments */); } public DisplayCutout get() { diff --git a/android/view/DisplayInfo.java b/android/view/DisplayInfo.java index 37e9815c..913e5924 100644 --- a/android/view/DisplayInfo.java +++ b/android/view/DisplayInfo.java @@ -20,6 +20,7 @@ import static android.view.DisplayInfoProto.APP_HEIGHT; import static android.view.DisplayInfoProto.APP_WIDTH; import static android.view.DisplayInfoProto.LOGICAL_HEIGHT; import static android.view.DisplayInfoProto.LOGICAL_WIDTH; +import static android.view.DisplayInfoProto.NAME; import android.content.res.CompatibilityInfo; import android.content.res.Configuration; @@ -30,9 +31,8 @@ import android.util.ArraySet; import android.util.DisplayMetrics; import android.util.proto.ProtoOutputStream; -import libcore.util.Objects; - import java.util.Arrays; +import java.util.Objects; /** * Describes the characteristics of a particular logical display. @@ -294,8 +294,8 @@ public final class DisplayInfo implements Parcelable { && layerStack == other.layerStack && flags == other.flags && type == other.type - && Objects.equal(address, other.address) - && Objects.equal(uniqueId, other.uniqueId) + && Objects.equals(address, other.address) + && Objects.equals(uniqueId, other.uniqueId) && appWidth == other.appWidth && appHeight == other.appHeight && smallestNominalAppWidth == other.smallestNominalAppWidth @@ -308,13 +308,13 @@ public final class DisplayInfo implements Parcelable { && overscanTop == other.overscanTop && overscanRight == other.overscanRight && overscanBottom == other.overscanBottom - && Objects.equal(displayCutout, other.displayCutout) + && Objects.equals(displayCutout, other.displayCutout) && rotation == other.rotation && modeId == other.modeId && defaultModeId == other.defaultModeId && colorMode == other.colorMode && Arrays.equals(supportedColorModes, other.supportedColorModes) - && Objects.equal(hdrCapabilities, other.hdrCapabilities) + && Objects.equals(hdrCapabilities, other.hdrCapabilities) && logicalDensityDpi == other.logicalDensityDpi && physicalXDpi == other.physicalXDpi && physicalYDpi == other.physicalYDpi @@ -322,7 +322,7 @@ public final class DisplayInfo implements Parcelable { && presentationDeadlineNanos == other.presentationDeadlineNanos && state == other.state && ownerUid == other.ownerUid - && Objects.equal(ownerPackageName, other.ownerPackageName) + && Objects.equals(ownerPackageName, other.ownerPackageName) && removeMode == other.removeMode; } @@ -685,6 +685,7 @@ public final class DisplayInfo implements Parcelable { protoOutputStream.write(LOGICAL_HEIGHT, logicalHeight); protoOutputStream.write(APP_WIDTH, appWidth); protoOutputStream.write(APP_HEIGHT, appHeight); + protoOutputStream.write(NAME, name); protoOutputStream.end(token); } diff --git a/android/view/DisplayListCanvas.java b/android/view/DisplayListCanvas.java index 8f9ae0e3..671532c9 100644 --- a/android/view/DisplayListCanvas.java +++ b/android/view/DisplayListCanvas.java @@ -198,8 +198,8 @@ public final class DisplayListCanvas extends RecordingCanvas { * * @param layer The layer to composite on this canvas */ - void drawHardwareLayer(HardwareLayer layer) { - nDrawLayer(mNativeCanvasWrapper, layer.getLayerHandle()); + void drawTextureLayer(TextureLayer layer) { + nDrawTextureLayer(mNativeCanvasWrapper, layer.getLayerHandle()); } /////////////////////////////////////////////////////////////////////////// @@ -257,7 +257,7 @@ public final class DisplayListCanvas extends RecordingCanvas { @CriticalNative private static native void nDrawRenderNode(long renderer, long renderNode); @CriticalNative - private static native void nDrawLayer(long renderer, long layer); + private static native void nDrawTextureLayer(long renderer, long layer); @CriticalNative private static native void nDrawCircle(long renderer, long propCx, long propCy, long propRadius, long propPaint); diff --git a/android/view/IWindowManagerImpl.java b/android/view/IWindowManagerImpl.java index 93e6c0b9..f62aa6c1 100644 --- a/android/view/IWindowManagerImpl.java +++ b/android/view/IWindowManagerImpl.java @@ -346,7 +346,7 @@ public class IWindowManagerImpl implements IWindowManager { } @Override - public void setScreenCaptureDisabled(int userId, boolean disabled) { + public void refreshScreenCaptureDisabled(int userId) { // TODO Auto-generated method stub } @@ -387,6 +387,15 @@ public class IWindowManagerImpl implements IWindowManager { } @Override + public void setShelfHeight(boolean visible, int shelfHeight) { + // TODO Auto-generated method stub + } + + @Override + public void setNavBarVirtualKeyHapticFeedbackEnabled(boolean enabled) { + } + + @Override public boolean stopViewServer() throws RemoteException { // TODO Auto-generated method stub return false; @@ -526,14 +535,6 @@ public class IWindowManagerImpl implements IWindowManager { } @Override - public void enableSurfaceTrace(ParcelFileDescriptor fd) throws RemoteException { - } - - @Override - public void disableSurfaceTrace() throws RemoteException { - } - - @Override public Region getCurrentImeTouchRegion() throws RemoteException { return null; } @@ -561,4 +562,12 @@ public class IWindowManagerImpl implements IWindowManager { public boolean isWindowTraceEnabled() throws RemoteException { return false; } + + @Override + public void requestUserActivityNotification() throws RemoteException { + } + + @Override + public void dontOverrideDisplayInfo(int displayId) throws RemoteException { + } } diff --git a/android/view/MenuInflater_Delegate.java b/android/view/MenuInflater_Delegate.java index 977a2a72..d16d8519 100644 --- a/android/view/MenuInflater_Delegate.java +++ b/android/view/MenuInflater_Delegate.java @@ -56,7 +56,10 @@ public class MenuInflater_Delegate { } } - if (menuItem == null || !menuItem.getClass().getName().startsWith("android.support.")) { + String menuItemName = menuItem != null ? menuItem.getClass().getName() : null; + if (menuItemName == null || + !menuItemName.startsWith("android.support.") || + !menuItemName.startsWith("androidx.")) { // This means that Bridge did not take over the instantiation of some object properly. // This is most likely a bug in the LayoutLib code. // We suppress this error for AppCompat menus since we do not support them in the menu diff --git a/android/view/NotificationHeaderView.java b/android/view/NotificationHeaderView.java index fbba8abf..7a103647 100644 --- a/android/view/NotificationHeaderView.java +++ b/android/view/NotificationHeaderView.java @@ -17,6 +17,7 @@ package android.view; import android.annotation.Nullable; +import android.app.AppOpsManager; import android.app.Notification; import android.content.Context; import android.content.res.Resources; @@ -25,6 +26,7 @@ import android.graphics.Canvas; import android.graphics.Outline; import android.graphics.Rect; import android.graphics.drawable.Drawable; +import android.util.ArraySet; import android.util.AttributeSet; import android.widget.ImageView; import android.widget.RemoteViews; @@ -49,10 +51,15 @@ public class NotificationHeaderView extends ViewGroup { private View mHeaderText; private View mSecondaryHeaderText; private OnClickListener mExpandClickListener; + private OnClickListener mAppOpsListener; private HeaderTouchListener mTouchListener = new HeaderTouchListener(); private ImageView mExpandButton; private CachingIconView mIcon; private View mProfileBadge; + private View mOverlayIcon; + private View mCameraIcon; + private View mMicIcon; + private View mAppOps; private int mIconColor; private int mOriginalNotificationColor; private boolean mExpanded; @@ -108,6 +115,10 @@ public class NotificationHeaderView extends ViewGroup { mExpandButton = findViewById(com.android.internal.R.id.expand_button); mIcon = findViewById(com.android.internal.R.id.icon); mProfileBadge = findViewById(com.android.internal.R.id.profile_badge); + mCameraIcon = findViewById(com.android.internal.R.id.camera); + mMicIcon = findViewById(com.android.internal.R.id.mic); + mOverlayIcon = findViewById(com.android.internal.R.id.overlay); + mAppOps = findViewById(com.android.internal.R.id.app_ops); } @Override @@ -198,6 +209,11 @@ public class NotificationHeaderView extends ViewGroup { layoutRight = end - paddingEnd; end = layoutLeft = layoutRight - child.getMeasuredWidth(); } + if (child == mAppOps) { + int paddingEnd = mContentEndMargin; + layoutRight = end - paddingEnd; + end = layoutLeft = layoutRight - child.getMeasuredWidth(); + } if (getLayoutDirection() == LAYOUT_DIRECTION_RTL) { int ltrLeft = layoutLeft; layoutLeft = getWidth() - layoutRight; @@ -252,15 +268,26 @@ public class NotificationHeaderView extends ViewGroup { } private void updateTouchListener() { - if (mExpandClickListener != null) { - mTouchListener.bindTouchRects(); + if (mExpandClickListener == null && mAppOpsListener == null) { + setOnTouchListener(null); + return; } + setOnTouchListener(mTouchListener); + mTouchListener.bindTouchRects(); + } + + /** + * Sets onclick listener for app ops icons. + */ + public void setAppOpsOnClickListener(OnClickListener l) { + mAppOpsListener = l; + mAppOps.setOnClickListener(mAppOpsListener); + updateTouchListener(); } @Override public void setOnClickListener(@Nullable OnClickListener l) { mExpandClickListener = l; - setOnTouchListener(mExpandClickListener != null ? mTouchListener : null); mExpandButton.setOnClickListener(mExpandClickListener); updateTouchListener(); } @@ -289,6 +316,22 @@ public class NotificationHeaderView extends ViewGroup { updateExpandButton(); } + /** + * Shows or hides 'app op in use' icons based on app usage. + */ + public void showAppOpsIcons(ArraySet<Integer> appOps) { + if (mOverlayIcon == null || mCameraIcon == null || mMicIcon == null || appOps == null) { + return; + } + + mOverlayIcon.setVisibility(appOps.contains(AppOpsManager.OP_SYSTEM_ALERT_WINDOW) + ? View.VISIBLE : View.GONE); + mCameraIcon.setVisibility(appOps.contains(AppOpsManager.OP_CAMERA) + ? View.VISIBLE : View.GONE); + mMicIcon.setVisibility(appOps.contains(AppOpsManager.OP_RECORD_AUDIO) + ? View.VISIBLE : View.GONE); + } + private void updateExpandButton() { int drawableId; int contentDescriptionId; @@ -335,6 +378,7 @@ public class NotificationHeaderView extends ViewGroup { private final ArrayList<Rect> mTouchRects = new ArrayList<>(); private Rect mExpandButtonRect; + private Rect mAppOpsRect; private int mTouchSlop; private boolean mTrackGesture; private float mDownX; @@ -347,6 +391,7 @@ public class NotificationHeaderView extends ViewGroup { mTouchRects.clear(); addRectAroundView(mIcon); mExpandButtonRect = addRectAroundView(mExpandButton); + mAppOpsRect = addRectAroundView(mAppOps); addWidthRect(); mTouchSlop = ViewConfiguration.get(getContext()).getScaledTouchSlop(); } @@ -368,16 +413,18 @@ public class NotificationHeaderView extends ViewGroup { private Rect getRectAroundView(View view) { float size = 48 * getResources().getDisplayMetrics().density; + float width = Math.max(size, view.getWidth()); + float height = Math.max(size, view.getHeight()); final Rect r = new Rect(); if (view.getVisibility() == GONE) { view = getFirstChildNotGone(); - r.left = (int) (view.getLeft() - size / 2.0f); + r.left = (int) (view.getLeft() - width / 2.0f); } else { - r.left = (int) ((view.getLeft() + view.getRight()) / 2.0f - size / 2.0f); + r.left = (int) ((view.getLeft() + view.getRight()) / 2.0f - width / 2.0f); } - r.top = (int) ((view.getTop() + view.getBottom()) / 2.0f - size / 2.0f); - r.bottom = (int) (r.top + size); - r.right = (int) (r.left + size); + r.top = (int) ((view.getTop() + view.getBottom()) / 2.0f - height / 2.0f); + r.bottom = (int) (r.top + height); + r.right = (int) (r.left + width); return r; } @@ -405,6 +452,11 @@ public class NotificationHeaderView extends ViewGroup { break; case MotionEvent.ACTION_UP: if (mTrackGesture) { + if (mAppOps.isVisibleToUser() && (mAppOpsRect.contains((int) x, (int) y) + || mAppOpsRect.contains((int) mDownX, (int) mDownY))) { + mAppOps.performClick(); + return true; + } mExpandButton.performClick(); } break; diff --git a/android/view/PixelCopy.java b/android/view/PixelCopy.java index a14609f3..2797a4da 100644 --- a/android/view/PixelCopy.java +++ b/android/view/PixelCopy.java @@ -263,8 +263,16 @@ public final class PixelCopy { "Only able to copy windows with decor views"); } Surface surface = null; - if (source.peekDecorView().getViewRootImpl() != null) { - surface = source.peekDecorView().getViewRootImpl().mSurface; + final ViewRootImpl root = source.peekDecorView().getViewRootImpl(); + if (root != null) { + surface = root.mSurface; + final Rect surfaceInsets = root.mWindowAttributes.surfaceInsets; + if (srcRect == null) { + srcRect = new Rect(surfaceInsets.left, surfaceInsets.top, + root.mWidth + surfaceInsets.left, root.mHeight + surfaceInsets.top); + } else { + srcRect.offset(surfaceInsets.left, surfaceInsets.top); + } } if (surface == null || !surface.isValid()) { throw new IllegalArgumentException( diff --git a/android/view/PointerIcon.java b/android/view/PointerIcon.java index 3fd46963..8cb46b70 100644 --- a/android/view/PointerIcon.java +++ b/android/view/PointerIcon.java @@ -23,6 +23,10 @@ import android.content.res.Resources; import android.content.res.TypedArray; import android.content.res.XmlResourceParser; import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.RectF; import android.graphics.drawable.AnimationDrawable; import android.graphics.drawable.BitmapDrawable; import android.graphics.drawable.Drawable; @@ -396,6 +400,33 @@ public final class PointerIcon implements Parcelable { return true; } + /** + * Get the Bitmap from the Drawable. + * + * If the Bitmap needed to be scaled up to account for density, BitmapDrawable + * handles this at draw time. But this class doesn't actually draw the Bitmap; + * it is just a holder for native code to access its SkBitmap. So this needs to + * get a version that is scaled to account for density. + */ + private Bitmap getBitmapFromDrawable(BitmapDrawable bitmapDrawable) { + Bitmap bitmap = bitmapDrawable.getBitmap(); + final int scaledWidth = bitmapDrawable.getIntrinsicWidth(); + final int scaledHeight = bitmapDrawable.getIntrinsicHeight(); + if (scaledWidth == bitmap.getWidth() && scaledHeight == bitmap.getHeight()) { + return bitmap; + } + + Rect src = new Rect(0, 0, bitmap.getWidth(), bitmap.getHeight()); + RectF dst = new RectF(0, 0, scaledWidth, scaledHeight); + + Bitmap scaled = Bitmap.createBitmap(scaledWidth, scaledHeight, bitmap.getConfig()); + Canvas canvas = new Canvas(scaled); + Paint paint = new Paint(); + paint.setFilterBitmap(true); + canvas.drawBitmap(bitmap, src, dst, paint); + return scaled; + } + private void loadResource(Context context, Resources resources, @XmlRes int resourceId) { final XmlResourceParser parser = resources.getXml(resourceId); final int bitmapRes; @@ -452,7 +483,8 @@ public final class PointerIcon implements Parcelable { + "is different. All frames should have the exact same size and " + "share the same hotspot."); } - mBitmapFrames[i - 1] = ((BitmapDrawable)drawableFrame).getBitmap(); + BitmapDrawable bitmapDrawableFrame = (BitmapDrawable) drawableFrame; + mBitmapFrames[i - 1] = getBitmapFromDrawable(bitmapDrawableFrame); } } } @@ -461,7 +493,8 @@ public final class PointerIcon implements Parcelable { + "refer to a bitmap drawable."); } - final Bitmap bitmap = ((BitmapDrawable) drawable).getBitmap(); + BitmapDrawable bitmapDrawable = (BitmapDrawable) drawable; + final Bitmap bitmap = getBitmapFromDrawable(bitmapDrawable); validateHotSpot(bitmap, hotSpotX, hotSpotY); // Set the properties now that we have successfully loaded the icon. mBitmap = bitmap; diff --git a/android/view/RecordingCanvas.java b/android/view/RecordingCanvas.java index fbb862be..f7a41ffa 100644 --- a/android/view/RecordingCanvas.java +++ b/android/view/RecordingCanvas.java @@ -34,7 +34,7 @@ import android.graphics.Rect; import android.graphics.RectF; import android.graphics.TemporaryBuffer; import android.text.GraphicsOperations; -import android.text.MeasuredText; +import android.text.PrecomputedText; import android.text.SpannableString; import android.text.SpannedString; import android.text.TextUtils; @@ -474,8 +474,7 @@ public class RecordingCanvas extends Canvas { } nDrawTextRun(mNativeCanvasWrapper, text, index, count, contextIndex, contextCount, - x, y, isRtl, paint.getNativeInstance(), 0 /* measured text */, - 0 /* measured text offset */); + x, y, isRtl, paint.getNativeInstance(), 0 /* measured text */); } @Override @@ -506,19 +505,16 @@ public class RecordingCanvas extends Canvas { char[] buf = TemporaryBuffer.obtain(contextLen); TextUtils.getChars(text, contextStart, contextEnd, buf, 0); long measuredTextPtr = 0; - int measuredTextOffset = 0; - if (text instanceof MeasuredText) { - MeasuredText mt = (MeasuredText) text; + if (text instanceof PrecomputedText) { + PrecomputedText mt = (PrecomputedText) text; int paraIndex = mt.findParaIndex(start); if (end <= mt.getParagraphEnd(paraIndex)) { // Only support if the target is in the same paragraph. measuredTextPtr = mt.getMeasuredParagraph(paraIndex).getNativePtr(); - measuredTextOffset = start - mt.getParagraphStart(paraIndex); } } nDrawTextRun(mNativeCanvasWrapper, buf, start - contextStart, len, - 0, contextLen, x, y, isRtl, paint.getNativeInstance(), - measuredTextPtr, measuredTextOffset); + 0, contextLen, x, y, isRtl, paint.getNativeInstance(), measuredTextPtr); TemporaryBuffer.recycle(buf); } } @@ -641,7 +637,7 @@ public class RecordingCanvas extends Canvas { @FastNative private static native void nDrawTextRun(long nativeCanvas, char[] text, int start, int count, int contextStart, int contextCount, float x, float y, boolean isRtl, long nativePaint, - long nativeMeasuredText, int measuredTextOffset); + long nativePrecomputedText); @FastNative private static native void nDrawTextOnPath(long nativeCanvas, char[] text, int index, int count, diff --git a/android/view/RemoteAnimationAdapter.java b/android/view/RemoteAnimationAdapter.java index d597e597..a864e550 100644 --- a/android/view/RemoteAnimationAdapter.java +++ b/android/view/RemoteAnimationAdapter.java @@ -52,6 +52,9 @@ public class RemoteAnimationAdapter implements Parcelable { private final long mDuration; private final long mStatusBarTransitionDelay; + /** @see #getCallingPid */ + private int mCallingPid; + /** * @param runner The interface that gets notified when we actually need to start the animation. * @param duration The duration of the animation. @@ -83,6 +86,20 @@ public class RemoteAnimationAdapter implements Parcelable { return mStatusBarTransitionDelay; } + /** + * To be called by system_server to keep track which pid is running this animation. + */ + public void setCallingPid(int pid) { + mCallingPid = pid; + } + + /** + * @return The pid of the process running the animation. + */ + public int getCallingPid() { + return mCallingPid; + } + @Override public int describeContents() { return 0; diff --git a/android/view/RemoteAnimationDefinition.java b/android/view/RemoteAnimationDefinition.java index 381f6926..d2240e1f 100644 --- a/android/view/RemoteAnimationDefinition.java +++ b/android/view/RemoteAnimationDefinition.java @@ -16,10 +16,14 @@ package android.view; +import static android.app.WindowConfiguration.ACTIVITY_TYPE_UNDEFINED; + import android.annotation.Nullable; +import android.app.WindowConfiguration; +import android.app.WindowConfiguration.ActivityType; import android.os.Parcel; import android.os.Parcelable; -import android.util.ArrayMap; +import android.util.ArraySet; import android.util.SparseArray; import android.view.WindowManager.TransitionType; @@ -30,7 +34,7 @@ import android.view.WindowManager.TransitionType; */ public class RemoteAnimationDefinition implements Parcelable { - private final SparseArray<RemoteAnimationAdapter> mTransitionAnimationMap; + private final SparseArray<RemoteAnimationAdapterEntry> mTransitionAnimationMap; public RemoteAnimationDefinition() { mTransitionAnimationMap = new SparseArray<>(); @@ -40,34 +44,80 @@ public class RemoteAnimationDefinition implements Parcelable { * Registers a remote animation for a specific transition. * * @param transition The transition type. Must be one of WindowManager.TRANSIT_* values. + * @param activityTypeFilter The remote animation only runs if an activity with type of this + * parameter is involved in the transition. + * @param adapter The adapter that described how to run the remote animation. + */ + public void addRemoteAnimation(@TransitionType int transition, + @ActivityType int activityTypeFilter, RemoteAnimationAdapter adapter) { + mTransitionAnimationMap.put(transition, + new RemoteAnimationAdapterEntry(adapter, activityTypeFilter)); + } + + /** + * Registers a remote animation for a specific transition without defining an activity type + * filter. + * + * @param transition The transition type. Must be one of WindowManager.TRANSIT_* values. * @param adapter The adapter that described how to run the remote animation. */ public void addRemoteAnimation(@TransitionType int transition, RemoteAnimationAdapter adapter) { - mTransitionAnimationMap.put(transition, adapter); + addRemoteAnimation(transition, ACTIVITY_TYPE_UNDEFINED, adapter); } /** * Checks whether a remote animation for specific transition is defined. * * @param transition The transition type. Must be one of WindowManager.TRANSIT_* values. + * @param activityTypes The set of activity types of activities that are involved in the + * transition. Will be used for filtering. * @return Whether this definition has defined a remote animation for the specified transition. */ - public boolean hasTransition(@TransitionType int transition) { - return mTransitionAnimationMap.get(transition) != null; + public boolean hasTransition(@TransitionType int transition, ArraySet<Integer> activityTypes) { + return getAdapter(transition, activityTypes) != null; } /** * Retrieves the remote animation for a specific transition. * * @param transition The transition type. Must be one of WindowManager.TRANSIT_* values. + * @param activityTypes The set of activity types of activities that are involved in the + * transition. Will be used for filtering. * @return The remote animation adapter for the specified transition. */ - public @Nullable RemoteAnimationAdapter getAdapter(@TransitionType int transition) { - return mTransitionAnimationMap.get(transition); + public @Nullable RemoteAnimationAdapter getAdapter(@TransitionType int transition, + ArraySet<Integer> activityTypes) { + final RemoteAnimationAdapterEntry entry = mTransitionAnimationMap.get(transition); + if (entry == null) { + return null; + } + if (entry.activityTypeFilter == ACTIVITY_TYPE_UNDEFINED + || activityTypes.contains(entry.activityTypeFilter)) { + return entry.adapter; + } else { + return null; + } } public RemoteAnimationDefinition(Parcel in) { - mTransitionAnimationMap = in.readSparseArray(null /* loader */); + final int size = in.readInt(); + mTransitionAnimationMap = new SparseArray<>(size); + for (int i = 0; i < size; i++) { + final int transition = in.readInt(); + final RemoteAnimationAdapterEntry entry = in.readTypedObject( + RemoteAnimationAdapterEntry.CREATOR); + mTransitionAnimationMap.put(transition, entry); + } + } + + /** + * To be called by system_server to keep track which pid is running the remote animations inside + * this definition. + */ + public void setCallingPid(int pid) { + for (int i = mTransitionAnimationMap.size() - 1; i >= 0; i--) { + mTransitionAnimationMap.valueAt(i).adapter.setCallingPid(pid); + } } @Override @@ -77,7 +127,12 @@ public class RemoteAnimationDefinition implements Parcelable { @Override public void writeToParcel(Parcel dest, int flags) { - dest.writeSparseArray((SparseArray) mTransitionAnimationMap); + final int size = mTransitionAnimationMap.size(); + dest.writeInt(size); + for (int i = 0; i < size; i++) { + dest.writeInt(mTransitionAnimationMap.keyAt(i)); + dest.writeTypedObject(mTransitionAnimationMap.valueAt(i), flags); + } } public static final Creator<RemoteAnimationDefinition> CREATOR = @@ -90,4 +145,50 @@ public class RemoteAnimationDefinition implements Parcelable { return new RemoteAnimationDefinition[size]; } }; + + private static class RemoteAnimationAdapterEntry implements Parcelable { + + final RemoteAnimationAdapter adapter; + + /** + * Only run the transition if one of the activities matches the filter. + * {@link WindowConfiguration.ACTIVITY_TYPE_UNDEFINED} means no filter + */ + @ActivityType final int activityTypeFilter; + + RemoteAnimationAdapterEntry(RemoteAnimationAdapter adapter, int activityTypeFilter) { + this.adapter = adapter; + this.activityTypeFilter = activityTypeFilter; + } + + private RemoteAnimationAdapterEntry(Parcel in) { + adapter = in.readParcelable(RemoteAnimationAdapter.class.getClassLoader()); + activityTypeFilter = in.readInt(); + } + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeParcelable(adapter, flags); + dest.writeInt(activityTypeFilter); + } + + @Override + public int describeContents() { + return 0; + } + + private static final Creator<RemoteAnimationAdapterEntry> CREATOR + = new Creator<RemoteAnimationAdapterEntry>() { + + @Override + public RemoteAnimationAdapterEntry createFromParcel(Parcel in) { + return new RemoteAnimationAdapterEntry(in); + } + + @Override + public RemoteAnimationAdapterEntry[] newArray(int size) { + return new RemoteAnimationAdapterEntry[size]; + } + }; + } } diff --git a/android/view/RemoteAnimationTarget.java b/android/view/RemoteAnimationTarget.java index c28c3894..5b2cc817 100644 --- a/android/view/RemoteAnimationTarget.java +++ b/android/view/RemoteAnimationTarget.java @@ -16,13 +16,26 @@ package android.view; +import static android.app.RemoteAnimationTargetProto.CLIP_RECT; +import static android.app.RemoteAnimationTargetProto.CONTENT_INSETS; +import static android.app.RemoteAnimationTargetProto.IS_TRANSLUCENT; +import static android.app.RemoteAnimationTargetProto.LEASH; +import static android.app.RemoteAnimationTargetProto.MODE; +import static android.app.RemoteAnimationTargetProto.POSITION; +import static android.app.RemoteAnimationTargetProto.PREFIX_ORDER_INDEX; +import static android.app.RemoteAnimationTargetProto.SOURCE_CONTAINER_BOUNDS; +import static android.app.RemoteAnimationTargetProto.TASK_ID; +import static android.app.RemoteAnimationTargetProto.WINDOW_CONFIGURATION; + import android.annotation.IntDef; import android.app.WindowConfiguration; import android.graphics.Point; import android.graphics.Rect; import android.os.Parcel; import android.os.Parcelable; +import android.util.proto.ProtoOutputStream; +import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -79,6 +92,11 @@ public class RemoteAnimationTarget implements Parcelable { public final Rect clipRect; /** + * The insets of the main app window. + */ + public final Rect contentInsets; + + /** * The index of the element in the tree in prefix order. This should be used for z-layering * to preserve original z-layer order in the hierarchy tree assuming no "boosting" needs to * happen. @@ -104,18 +122,25 @@ public class RemoteAnimationTarget implements Parcelable { */ public final WindowConfiguration windowConfiguration; + /** + * Whether the task is not presented in Recents UI. + */ + public boolean isNotInRecents; + public RemoteAnimationTarget(int taskId, int mode, SurfaceControl leash, boolean isTranslucent, - Rect clipRect, int prefixOrderIndex, Point position, Rect sourceContainerBounds, - WindowConfiguration windowConfig) { + Rect clipRect, Rect contentInsets, int prefixOrderIndex, Point position, + Rect sourceContainerBounds, WindowConfiguration windowConfig, boolean isNotInRecents) { this.mode = mode; this.taskId = taskId; this.leash = leash; this.isTranslucent = isTranslucent; this.clipRect = new Rect(clipRect); + this.contentInsets = new Rect(contentInsets); this.prefixOrderIndex = prefixOrderIndex; this.position = new Point(position); this.sourceContainerBounds = new Rect(sourceContainerBounds); this.windowConfiguration = windowConfig; + this.isNotInRecents = isNotInRecents; } public RemoteAnimationTarget(Parcel in) { @@ -124,10 +149,12 @@ public class RemoteAnimationTarget implements Parcelable { leash = in.readParcelable(null); isTranslucent = in.readBoolean(); clipRect = in.readParcelable(null); + contentInsets = in.readParcelable(null); prefixOrderIndex = in.readInt(); position = in.readParcelable(null); sourceContainerBounds = in.readParcelable(null); windowConfiguration = in.readParcelable(null); + isNotInRecents = in.readBoolean(); } @Override @@ -142,10 +169,41 @@ public class RemoteAnimationTarget implements Parcelable { dest.writeParcelable(leash, 0 /* flags */); dest.writeBoolean(isTranslucent); dest.writeParcelable(clipRect, 0 /* flags */); + dest.writeParcelable(contentInsets, 0 /* flags */); dest.writeInt(prefixOrderIndex); dest.writeParcelable(position, 0 /* flags */); dest.writeParcelable(sourceContainerBounds, 0 /* flags */); dest.writeParcelable(windowConfiguration, 0 /* flags */); + dest.writeBoolean(isNotInRecents); + } + + public void dump(PrintWriter pw, String prefix) { + pw.print(prefix); pw.print("mode="); pw.print(mode); + pw.print(" taskId="); pw.print(taskId); + pw.print(" isTranslucent="); pw.print(isTranslucent); + pw.print(" clipRect="); clipRect.printShortString(pw); + pw.print(" contentInsets="); contentInsets.printShortString(pw); + pw.print(" prefixOrderIndex="); pw.print(prefixOrderIndex); + pw.print(" position="); position.printShortString(pw); + pw.print(" sourceContainerBounds="); sourceContainerBounds.printShortString(pw); + pw.println(); + pw.print(prefix); pw.print("windowConfiguration="); pw.println(windowConfiguration); + pw.print(prefix); pw.print("leash="); pw.println(leash); + } + + public void writeToProto(ProtoOutputStream proto, long fieldId) { + final long token = proto.start(fieldId); + proto.write(TASK_ID, taskId); + proto.write(MODE, mode); + leash.writeToProto(proto, LEASH); + proto.write(IS_TRANSLUCENT, isTranslucent); + clipRect.writeToProto(proto, CLIP_RECT); + contentInsets.writeToProto(proto, CONTENT_INSETS); + proto.write(PREFIX_ORDER_INDEX, prefixOrderIndex); + position.writeToProto(proto, POSITION); + sourceContainerBounds.writeToProto(proto, SOURCE_CONTAINER_BOUNDS); + windowConfiguration.writeToProto(proto, WINDOW_CONFIGURATION); + proto.end(token); } public static final Creator<RemoteAnimationTarget> CREATOR diff --git a/android/view/RenderNode.java b/android/view/RenderNode.java index 50701518..7c25fac3 100644 --- a/android/view/RenderNode.java +++ b/android/view/RenderNode.java @@ -353,9 +353,24 @@ public class RenderNode { return nHasShadow(mNativeRenderNode); } - /** setShadowColor */ - public boolean setShadowColor(int color) { - return nSetShadowColor(mNativeRenderNode, color); + /** setSpotShadowColor */ + public boolean setSpotShadowColor(int color) { + return nSetSpotShadowColor(mNativeRenderNode, color); + } + + /** setAmbientShadowColor */ + public boolean setAmbientShadowColor(int color) { + return nSetAmbientShadowColor(mNativeRenderNode, color); + } + + /** getSpotShadowColor */ + public int getSpotShadowColor() { + return nGetSpotShadowColor(mNativeRenderNode); + } + + /** getAmbientShadowColor */ + public int getAmbientShadowColor() { + return nGetAmbientShadowColor(mNativeRenderNode); } /** @@ -672,6 +687,11 @@ public class RenderNode { return nIsPivotExplicitlySet(mNativeRenderNode); } + /** lint */ + public boolean resetPivot() { + return nResetPivot(mNativeRenderNode); + } + /** * Sets the camera distance for the display list. Refer to * {@link View#setCameraDistance(float)} for more information on how to @@ -888,6 +908,8 @@ public class RenderNode { @CriticalNative private static native boolean nSetPivotX(long renderNode, float pivotX); @CriticalNative + private static native boolean nResetPivot(long renderNode); + @CriticalNative private static native boolean nSetLayerType(long renderNode, int layerType); @CriticalNative private static native boolean nSetLayerPaint(long renderNode, long paint); @@ -915,7 +937,13 @@ public class RenderNode { @CriticalNative private static native boolean nHasShadow(long renderNode); @CriticalNative - private static native boolean nSetShadowColor(long renderNode, int color); + private static native boolean nSetSpotShadowColor(long renderNode, int color); + @CriticalNative + private static native boolean nSetAmbientShadowColor(long renderNode, int color); + @CriticalNative + private static native int nGetSpotShadowColor(long renderNode); + @CriticalNative + private static native int nGetAmbientShadowColor(long renderNode); @CriticalNative private static native boolean nSetClipToOutline(long renderNode, boolean clipToOutline); @CriticalNative diff --git a/android/view/RenderNodeAnimator.java b/android/view/RenderNodeAnimator.java index c4a71601..d26a2f64 100644 --- a/android/view/RenderNodeAnimator.java +++ b/android/view/RenderNodeAnimator.java @@ -158,7 +158,7 @@ public class RenderNodeAnimator extends Animator { } private void applyInterpolator() { - if (mInterpolator == null) return; + if (mInterpolator == null || mNativePtr == null) return; long ni; if (isNativeInterpolator(mInterpolator)) { diff --git a/android/view/Surface.java b/android/view/Surface.java index 8830c90a..df81a311 100644 --- a/android/view/Surface.java +++ b/android/view/Surface.java @@ -250,6 +250,18 @@ public class Surface implements Parcelable { } /** + * Destroys the HwuiContext without completely + * releasing the Surface. + * @hide + */ + public void hwuiDestroy() { + if (mHwuiContext != null) { + mHwuiContext.destroy(); + mHwuiContext = null; + } + } + + /** * Returns true if this object holds a valid surface. * * @return True if it holds a physical surface, so lockCanvas() will succeed. @@ -396,7 +408,44 @@ public class Surface implements Parcelable { synchronized (mLock) { checkNotReleasedLocked(); if (mHwuiContext == null) { - mHwuiContext = new HwuiContext(); + mHwuiContext = new HwuiContext(false); + } + return mHwuiContext.lockCanvas( + nativeGetWidth(mNativeObject), + nativeGetHeight(mNativeObject)); + } + } + + /** + * Gets a {@link Canvas} for drawing into this surface that supports wide color gamut. + * + * After drawing into the provided {@link Canvas}, the caller must + * invoke {@link #unlockCanvasAndPost} to post the new contents to the surface. + * + * Unlike {@link #lockCanvas(Rect)} and {@link #lockHardwareCanvas()}, + * this will return a hardware-accelerated canvas that supports wide color gamut. + * See the <a href="{@docRoot}guide/topics/graphics/hardware-accel.html#unsupported"> + * unsupported drawing operations</a> for a list of what is and isn't + * supported in a hardware-accelerated canvas. It is also required to + * fully cover the surface every time {@link #lockHardwareCanvas()} is + * called as the buffer is not preserved between frames. Partial updates + * are not supported. + * + * @return A canvas for drawing into the surface. + * + * @throws IllegalStateException If the canvas cannot be locked. + * + * @hide + */ + public Canvas lockHardwareWideColorGamutCanvas() { + synchronized (mLock) { + checkNotReleasedLocked(); + if (mHwuiContext != null && !mHwuiContext.isWideColorGamut()) { + mHwuiContext.destroy(); + mHwuiContext = null; + } + if (mHwuiContext == null) { + mHwuiContext = new HwuiContext(true); } return mHwuiContext.lockCanvas( nativeGetWidth(mNativeObject), @@ -829,11 +878,14 @@ public class Surface implements Parcelable { private final RenderNode mRenderNode; private long mHwuiRenderer; private DisplayListCanvas mCanvas; + private final boolean mIsWideColorGamut; - HwuiContext() { + HwuiContext(boolean isWideColorGamut) { mRenderNode = RenderNode.create("HwuiCanvas", null); mRenderNode.setClipToBounds(false); - mHwuiRenderer = nHwuiCreate(mRenderNode.mNativeRenderNode, mNativeObject); + mIsWideColorGamut = isWideColorGamut; + mHwuiRenderer = nHwuiCreate(mRenderNode.mNativeRenderNode, mNativeObject, + isWideColorGamut); } Canvas lockCanvas(int width, int height) { @@ -864,9 +916,13 @@ public class Surface implements Parcelable { mHwuiRenderer = 0; } } + + boolean isWideColorGamut() { + return mIsWideColorGamut; + } } - private static native long nHwuiCreate(long rootNode, long surface); + private static native long nHwuiCreate(long rootNode, long surface, boolean isWideColorGamut); private static native void nHwuiSetSurface(long renderer, long surface); private static native void nHwuiDraw(long renderer); private static native void nHwuiDestroy(long renderer); diff --git a/android/view/SurfaceControl.java b/android/view/SurfaceControl.java index bd7f8e54..d4610a56 100644 --- a/android/view/SurfaceControl.java +++ b/android/view/SurfaceControl.java @@ -152,6 +152,7 @@ public class SurfaceControl implements Parcelable { private static native void nativeSeverChildren(long transactionObj, long nativeObject); private static native void nativeSetOverrideScalingMode(long transactionObj, long nativeObject, int scalingMode); + private static native void nativeDestroy(long transactionObj, long nativeObject); private static native IBinder nativeGetHandle(long nativeObject); private static native boolean nativeGetTransformToDisplayInverse(long nativeObject); @@ -352,8 +353,8 @@ public class SurfaceControl implements Parcelable { private int mFormat = PixelFormat.OPAQUE; private String mName; private SurfaceControl mParent; - private int mWindowType; - private int mOwnerUid; + private int mWindowType = -1; + private int mOwnerUid = -1; /** * Begin building a SurfaceControl with a given {@link SurfaceSession}. @@ -565,7 +566,7 @@ public class SurfaceControl implements Parcelable { */ private SurfaceControl(SurfaceSession session, String name, int w, int h, int format, int flags, SurfaceControl parent, int windowType, int ownerUid) - throws OutOfResourcesException { + throws OutOfResourcesException, IllegalArgumentException { if (session == null) { throw new IllegalArgumentException("session must not be null"); } @@ -763,18 +764,14 @@ public class SurfaceControl implements Parcelable { } public void deferTransactionUntil(IBinder handle, long frame) { - if (frame > 0) { - synchronized(SurfaceControl.class) { - sGlobalTransaction.deferTransactionUntil(this, handle, frame); - } + synchronized(SurfaceControl.class) { + sGlobalTransaction.deferTransactionUntil(this, handle, frame); } } public void deferTransactionUntil(Surface barrier, long frame) { - if (frame > 0) { - synchronized(SurfaceControl.class) { - sGlobalTransaction.deferTransactionUntilSurface(this, barrier, frame); - } + synchronized(SurfaceControl.class) { + sGlobalTransaction.deferTransactionUntilSurface(this, barrier, frame); } } @@ -1479,6 +1476,9 @@ public class SurfaceControl implements Parcelable { public Transaction deferTransactionUntil(SurfaceControl sc, IBinder handle, long frameNumber) { + if (frameNumber < 0) { + return this; + } sc.checkNotReleased(); nativeDeferTransactionUntil(mNativeObject, sc.mNativeObject, handle, frameNumber); return this; @@ -1486,6 +1486,9 @@ public class SurfaceControl implements Parcelable { public Transaction deferTransactionUntilSurface(SurfaceControl sc, Surface barrierSurface, long frameNumber) { + if (frameNumber < 0) { + return this; + } sc.checkNotReleased(); nativeDeferTransactionUntilSurface(mNativeObject, sc.mNativeObject, barrierSurface.mNativeObject, frameNumber); @@ -1570,6 +1573,16 @@ public class SurfaceControl implements Parcelable { return this; } + /** + * Same as {@link #destroy()} except this is invoked in a transaction instead of + * immediately. + */ + public Transaction destroy(SurfaceControl sc) { + sc.checkNotReleased(); + nativeDestroy(mNativeObject, sc.mNativeObject); + return this; + } + public Transaction setDisplaySurface(IBinder displayToken, Surface surface) { if (displayToken == null) { throw new IllegalArgumentException("displayToken must not be null"); diff --git a/android/view/HardwareLayer.java b/android/view/TextureLayer.java index 7af10201..35a886fa 100644 --- a/android/view/HardwareLayer.java +++ b/android/view/TextureLayer.java @@ -25,19 +25,17 @@ import android.graphics.SurfaceTexture; import com.android.internal.util.VirtualRefBasePtr; /** - * A hardware layer can be used to render graphics operations into a hardware - * friendly buffer. For instance, with an OpenGL backend a hardware layer - * would use a Frame Buffer Object (FBO.) The hardware layer can be used as - * a drawing cache when a complex set of graphics operations needs to be - * drawn several times. + * TextureLayer represents a SurfaceTexture that will be composited by RenderThread into the + * frame when drawn in a HW accelerated Canvas. This is backed by a DeferredLayerUpdater on + * the native side. * * @hide */ -final class HardwareLayer { +final class TextureLayer { private ThreadedRenderer mRenderer; private VirtualRefBasePtr mFinalizer; - private HardwareLayer(ThreadedRenderer renderer, long deferredUpdater) { + private TextureLayer(ThreadedRenderer renderer, long deferredUpdater) { if (renderer == null || deferredUpdater == 0) { throw new IllegalArgumentException("Either hardware renderer: " + renderer + " or deferredUpdater: " + deferredUpdater + " is invalid"); @@ -141,11 +139,12 @@ final class HardwareLayer { mRenderer.pushLayerUpdate(this); } - static HardwareLayer adoptTextureLayer(ThreadedRenderer renderer, long layer) { - return new HardwareLayer(renderer, layer); + static TextureLayer adoptTextureLayer(ThreadedRenderer renderer, long layer) { + return new TextureLayer(renderer, layer); } - private static native boolean nPrepare(long layerUpdater, int width, int height, boolean isOpaque); + private static native boolean nPrepare(long layerUpdater, int width, int height, + boolean isOpaque); private static native void nSetLayerPaint(long layerUpdater, long paint); private static native void nSetTransform(long layerUpdater, long matrix); private static native void nSetSurfaceTexture(long layerUpdater, SurfaceTexture surface); diff --git a/android/view/TextureView.java b/android/view/TextureView.java index 25dce998..37179404 100644 --- a/android/view/TextureView.java +++ b/android/view/TextureView.java @@ -106,7 +106,7 @@ import android.util.Log; public class TextureView extends View { private static final String LOG_TAG = "TextureView"; - private HardwareLayer mLayer; + private TextureLayer mLayer; private SurfaceTexture mSurface; private SurfaceTextureListener mListener; private boolean mHadSurface; @@ -336,13 +336,13 @@ public class TextureView extends View { if (canvas.isHardwareAccelerated()) { DisplayListCanvas displayListCanvas = (DisplayListCanvas) canvas; - HardwareLayer layer = getHardwareLayer(); + TextureLayer layer = getTextureLayer(); if (layer != null) { applyUpdate(); applyTransformMatrix(); mLayer.setLayerPaint(mLayerPaint); // ensure layer paint is up to date - displayListCanvas.drawHardwareLayer(layer); + displayListCanvas.drawTextureLayer(layer); } } } @@ -369,7 +369,7 @@ public class TextureView extends View { } } - HardwareLayer getHardwareLayer() { + TextureLayer getTextureLayer() { if (mLayer == null) { if (mAttachInfo == null || mAttachInfo.mThreadedRenderer == null) { return null; @@ -602,7 +602,7 @@ public class TextureView extends View { // the layer here thanks to the validate() call at the beginning of // this method if (mLayer == null && mUpdateSurface) { - getHardwareLayer(); + getTextureLayer(); } if (mLayer != null) { diff --git a/android/view/ThreadedRenderer.java b/android/view/ThreadedRenderer.java index 8b730f28..5eb7e9cb 100644 --- a/android/view/ThreadedRenderer.java +++ b/android/view/ThreadedRenderer.java @@ -34,6 +34,7 @@ import android.os.Trace; import android.util.Log; import android.view.Surface.OutOfResourcesException; import android.view.View.AttachInfo; +import android.view.animation.AnimationUtils; import com.android.internal.R; import com.android.internal.util.VirtualRefBasePtr; @@ -166,18 +167,6 @@ public final class ThreadedRenderer { public static final String OVERDRAW_PROPERTY_SHOW = "show"; /** - * Defines the rendering pipeline to be used by the ThreadedRenderer. - * - * Possible values: - * "opengl", will use the existing OpenGL renderer - * "skiagl", will use Skia's OpenGL renderer - * "skiavk", will use Skia's Vulkan renderer - * - * @hide - */ - public static final String DEBUG_RENDERER_PROPERTY = "debug.hwui.renderer"; - - /** * Turn on to debug non-rectangular clip operations. * * Possible values: @@ -343,6 +332,7 @@ public final class ThreadedRenderer { private static final int FLAG_DUMP_FRAMESTATS = 1 << 0; private static final int FLAG_DUMP_RESET = 1 << 1; + private static final int FLAG_DUMP_ALL = FLAG_DUMP_FRAMESTATS; @IntDef(flag = true, prefix = { "FLAG_DUMP_" }, value = { FLAG_DUMP_FRAMESTATS, @@ -648,7 +638,10 @@ public final class ThreadedRenderer { */ void dumpGfxInfo(PrintWriter pw, FileDescriptor fd, String[] args) { pw.flush(); - int flags = 0; + // If there's no arguments, eg 'dumpsys gfxinfo', then dump everything. + // If there's a targetted package, eg 'dumpsys gfxinfo com.android.systemui', then only + // dump the summary information + int flags = (args == null || args.length == 0) ? FLAG_DUMP_ALL : 0; for (int i = 0; i < args.length; i++) { switch (args[i]) { case "framestats": @@ -657,6 +650,9 @@ public final class ThreadedRenderer { case "reset": flags |= FLAG_DUMP_RESET; break; + case "-a": // magic option passed when dumping a bugreport. + flags = FLAG_DUMP_ALL; + break; } } nDumpProfileInfo(mNativeProxy, fd, flags); @@ -838,9 +834,9 @@ public final class ThreadedRenderer { * * @return A hardware layer */ - HardwareLayer createTextureLayer() { + TextureLayer createTextureLayer() { long layer = nCreateTextureLayer(mNativeProxy); - return HardwareLayer.adoptTextureLayer(this, layer); + return TextureLayer.adoptTextureLayer(this, layer); } @@ -849,7 +845,7 @@ public final class ThreadedRenderer { } - boolean copyLayerInto(final HardwareLayer layer, final Bitmap bitmap) { + boolean copyLayerInto(final TextureLayer layer, final Bitmap bitmap) { return nCopyLayerInto(mNativeProxy, layer.getDeferredLayerUpdater(), bitmap); } @@ -860,7 +856,7 @@ public final class ThreadedRenderer { * * @param layer The hardware layer that needs an update */ - void pushLayerUpdate(HardwareLayer layer) { + void pushLayerUpdate(TextureLayer layer) { nPushLayerUpdate(mNativeProxy, layer.getDeferredLayerUpdater()); } @@ -868,7 +864,7 @@ public final class ThreadedRenderer { * Tells the HardwareRenderer that the layer is destroyed. The renderer * should remove the layer from any update queues. */ - void onLayerDestroyed(HardwareLayer layer) { + void onLayerDestroyed(TextureLayer layer) { nCancelLayerUpdate(mNativeProxy, layer.getDeferredLayerUpdater()); } @@ -938,6 +934,20 @@ public final class ThreadedRenderer { nSetHighContrastText(highContrastText); } + /** + * If set RenderThread will avoid doing any IPC using instead a fake vsync & DisplayInfo source + */ + public static void setIsolatedProcess(boolean isIsolated) { + nSetIsolatedProcess(isIsolated); + } + + /** + * If set extra graphics debugging abilities will be enabled such as dumping skp + */ + public static void setDebuggingEnabled(boolean enable) { + nSetDebuggingEnabled(enable); + } + @Override protected void finalize() throws Throwable { try { @@ -948,6 +958,107 @@ public final class ThreadedRenderer { } } + /** + * Basic synchronous renderer. Currently only used to render the Magnifier, so use with care. + * TODO: deduplicate against ThreadedRenderer. + * + * @hide + */ + public static class SimpleRenderer { + private final RenderNode mRootNode; + private long mNativeProxy; + private final float mLightY, mLightZ; + private Surface mSurface; + private final FrameInfo mFrameInfo = new FrameInfo(); + + public SimpleRenderer(final Context context, final String name, final Surface surface) { + final TypedArray a = context.obtainStyledAttributes(null, R.styleable.Lighting, 0, 0); + mLightY = a.getDimension(R.styleable.Lighting_lightY, 0); + mLightZ = a.getDimension(R.styleable.Lighting_lightZ, 0); + final float lightRadius = a.getDimension(R.styleable.Lighting_lightRadius, 0); + final int ambientShadowAlpha = + (int) (255 * a.getFloat(R.styleable.Lighting_ambientShadowAlpha, 0) + 0.5f); + final int spotShadowAlpha = + (int) (255 * a.getFloat(R.styleable.Lighting_spotShadowAlpha, 0) + 0.5f); + a.recycle(); + + final long rootNodePtr = nCreateRootRenderNode(); + mRootNode = RenderNode.adopt(rootNodePtr); + mRootNode.setClipToBounds(false); + mNativeProxy = nCreateProxy(true /* translucent */, rootNodePtr); + nSetName(mNativeProxy, name); + + ProcessInitializer.sInstance.init(context, mNativeProxy); + nLoadSystemProperties(mNativeProxy); + + nSetup(mNativeProxy, lightRadius, ambientShadowAlpha, spotShadowAlpha); + + mSurface = surface; + nUpdateSurface(mNativeProxy, surface); + } + + /** + * Set the light center. + */ + public void setLightCenter(final Display display, + final int windowLeft, final int windowTop) { + // Adjust light position for window offsets. + final Point displaySize = new Point(); + display.getRealSize(displaySize); + final float lightX = displaySize.x / 2f - windowLeft; + final float lightY = mLightY - windowTop; + + nSetLightCenter(mNativeProxy, lightX, lightY, mLightZ); + } + + public RenderNode getRootNode() { + return mRootNode; + } + + /** + * Draw the surface. + */ + public void draw(final FrameDrawingCallback callback) { + final long vsync = AnimationUtils.currentAnimationTimeMillis() * 1000000L; + mFrameInfo.setVsync(vsync, vsync); + mFrameInfo.addFlags(1 << 2 /* VSYNC */); + if (callback != null) { + nSetFrameCallback(mNativeProxy, callback); + } + nSyncAndDrawFrame(mNativeProxy, mFrameInfo.mFrameInfo, mFrameInfo.mFrameInfo.length); + } + + /** + * Destroy the renderer. + */ + public void destroy() { + mSurface = null; + nDestroy(mNativeProxy, mRootNode.mNativeRenderNode); + } + + @Override + protected void finalize() throws Throwable { + try { + nDeleteProxy(mNativeProxy); + mNativeProxy = 0; + } finally { + super.finalize(); + } + } + } + + /** + * Interface used to receive callbacks when a frame is being drawn. + */ + public interface FrameDrawingCallback { + /** + * Invoked during a frame drawing. + * + * @param frame The id of the frame being drawn. + */ + void onFrameDraw(long frame); + } + private static class ProcessInitializer { static ProcessInitializer sInstance = new ProcessInitializer(); @@ -970,7 +1081,10 @@ public final class ThreadedRenderer { mAppContext = context.getApplicationContext(); initSched(renderProxy); - initGraphicsStats(); + + if (mAppContext != null) { + initGraphicsStats(); + } } private void initSched(long renderProxy) { @@ -1085,6 +1199,7 @@ public final class ThreadedRenderer { private static native void nDrawRenderNode(long nativeProxy, long rootRenderNode); private static native void nSetContentDrawBounds(long nativeProxy, int left, int top, int right, int bottom); + private static native void nSetFrameCallback(long nativeProxy, FrameDrawingCallback callback); private static native long nAddFrameMetricsObserver(long nativeProxy, FrameMetricsObserver observer); private static native void nRemoveFrameMetricsObserver(long nativeProxy, long nativeObserver); @@ -1096,4 +1211,6 @@ public final class ThreadedRenderer { private static native void nSetHighContrastText(boolean enabled); // For temporary experimentation b/66945974 private static native void nHackySetRTAnimationsEnabled(boolean enabled); + private static native void nSetDebuggingEnabled(boolean enabled); + private static native void nSetIsolatedProcess(boolean enabled); } diff --git a/android/view/TouchDelegate.java b/android/view/TouchDelegate.java index dc50fa1d..d6c43e80 100644 --- a/android/view/TouchDelegate.java +++ b/android/view/TouchDelegate.java @@ -105,11 +105,13 @@ public class TouchDelegate { boolean hit = true; boolean handled = false; - switch (event.getAction()) { + switch (event.getActionMasked()) { case MotionEvent.ACTION_DOWN: mDelegateTargeted = mBounds.contains(x, y); sendToDelegate = mDelegateTargeted; break; + case MotionEvent.ACTION_POINTER_DOWN: + case MotionEvent.ACTION_POINTER_UP: case MotionEvent.ACTION_UP: case MotionEvent.ACTION_MOVE: sendToDelegate = mDelegateTargeted; diff --git a/android/view/View.java b/android/view/View.java index 3d6a6fee..97e11b15 100644 --- a/android/view/View.java +++ b/android/view/View.java @@ -75,6 +75,7 @@ import android.os.RemoteException; import android.os.SystemClock; import android.os.SystemProperties; import android.os.Trace; +import android.text.InputType; import android.text.TextUtils; import android.util.AttributeSet; import android.util.FloatProperty; @@ -726,6 +727,8 @@ import java.util.function.Predicate; * @attr ref android.R.styleable#View_nextFocusRight * @attr ref android.R.styleable#View_nextFocusUp * @attr ref android.R.styleable#View_onClick + * @attr ref android.R.styleable#View_outlineSpotShadowColor + * @attr ref android.R.styleable#View_outlineAmbientShadowColor * @attr ref android.R.styleable#View_padding * @attr ref android.R.styleable#View_paddingHorizontal * @attr ref android.R.styleable#View_paddingVertical @@ -904,6 +907,13 @@ public class View implements Drawable.Callback, KeyEvent.Callback, */ private static boolean sThrowOnInvalidFloatProperties; + /** + * Prior to P, {@code #startDragAndDrop} accepts a builder which produces an empty drag shadow. + * Currently zero size SurfaceControl cannot be created thus we create a dummy 1x1 surface + * instead. + */ + private static boolean sAcceptZeroSizeDragShadow; + /** @hide */ @IntDef({NOT_FOCUSABLE, FOCUSABLE, FOCUSABLE_AUTO}) @Retention(RetentionPolicy.SOURCE) @@ -2943,6 +2953,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * 1 PFLAG3_NO_REVEAL_ON_FOCUS * 1 PFLAG3_NOTIFY_AUTOFILL_ENTER_ON_LAYOUT * 1 PFLAG3_SCREEN_READER_FOCUSABLE + * 1 PFLAG3_AGGREGATED_VISIBLE + * 1 PFLAG3_AUTOFILLID_EXPLICITLY_SET + * 1 available * |-------|-------|-------|-------| */ @@ -3233,6 +3246,12 @@ public class View implements Drawable.Callback, KeyEvent.Callback, */ private static final int PFLAG3_AGGREGATED_VISIBLE = 0x20000000; + /** + * Used to indicate that {@link #mAutofillId} was explicitly set through + * {@link #setAutofillId(AutofillId)}. + */ + private static final int PFLAG3_AUTOFILLID_EXPLICITLY_SET = 0x40000000; + /* End of masks for mPrivateFlags3 */ /** @@ -3975,6 +3994,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, /** * Current clip bounds. to which all drawing of this view are constrained. */ + @ViewDebug.ExportedProperty(category = "drawing") Rect mClipBounds = null; private boolean mLastIsOpaque; @@ -4300,7 +4320,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, OnCapturedPointerListener mOnCapturedPointerListener; - private ArrayList<OnKeyFallbackListener> mKeyFallbackListeners; + private ArrayList<OnUnhandledKeyEventListener> mUnhandledKeyListeners; } ListenerInfo mListenerInfo; @@ -4442,6 +4462,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, private CheckForLongPress mPendingCheckForLongPress; private CheckForTap mPendingCheckForTap = null; private PerformClick mPerformClick; + private SendViewScrolledAccessibilityEvent mSendViewScrolledAccessibilityEvent; private UnsetPressedState mUnsetPressedState; @@ -4794,6 +4815,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, Canvas.sCompatibilityRestore = targetSdkVersion < Build.VERSION_CODES.M; Canvas.sCompatibilitySetBitmap = targetSdkVersion < Build.VERSION_CODES.O; + Canvas.setCompatibilityVersion(targetSdkVersion); // In M and newer, our widgets can pass a "hint" value in the size // for UNSPECIFIED MeasureSpecs. This lets child views of scrolling containers @@ -4836,6 +4858,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, sAlwaysAssignFocus = targetSdkVersion < Build.VERSION_CODES.P; + sAcceptZeroSizeDragShadow = targetSdkVersion < Build.VERSION_CODES.P; + sCompatibilityDone = true; } } @@ -5445,6 +5469,12 @@ public class View implements Drawable.Callback, KeyEvent.Callback, setAccessibilityPaneTitle(a.getString(attr)); } break; + case R.styleable.View_outlineSpotShadowColor: + setOutlineSpotShadowColor(a.getColor(attr, Color.BLACK)); + break; + case R.styleable.View_outlineAmbientShadowColor: + setOutlineAmbientShadowColor(a.getColor(attr, Color.BLACK)); + break; } } @@ -6516,7 +6546,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } finally { // Set it to already called so it's not called twice when called by // performClickInternal() - mPrivateFlags |= ~PFLAG_NOTIFY_AUTOFILL_MANAGER_ON_CLICK; + mPrivateFlags &= ~PFLAG_NOTIFY_AUTOFILL_MANAGER_ON_CLICK; } } } @@ -7201,7 +7231,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (gainFocus) { sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_FOCUSED); } else { - notifyAccessibilityStateChanged( + notifyViewAccessibilityStateChangedIfNeeded( AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } @@ -7251,7 +7281,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, // becomes true where it should issue notifyViewEntered(). afm.notifyViewEntered(this); } - } else if (!isFocused()) { + } else if (!enter && !isFocused()) { afm.notifyViewExited(this); } } @@ -7259,19 +7289,22 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** - * If this view is a visually distinct portion of a window, for example the content view of - * a fragment that is replaced, it is considered a pane for accessibility purposes. In order - * for accessibility services to understand the views role, and to announce its title as - * appropriate, such views should have pane titles. + * Visually distinct portion of a window with window-like semantics are considered panes for + * accessibility purposes. One example is the content view of a fragment that is replaced. + * In order for accessibility services to understand a pane's window-like behavior, panes + * should have descriptive titles. Views with pane titles produce {@link AccessibilityEvent}s + * when they appear, disappear, or change title. * - * @param accessibilityPaneTitle The pane's title. + * @param accessibilityPaneTitle The pane's title. Setting to {@code null} indicates that this + * View is not a pane. * * {@see AccessibilityNodeInfo#setPaneTitle(CharSequence)} */ - public void setAccessibilityPaneTitle(CharSequence accessibilityPaneTitle) { + public void setAccessibilityPaneTitle(@Nullable CharSequence accessibilityPaneTitle) { if (!TextUtils.equals(accessibilityPaneTitle, mAccessibilityPaneTitle)) { mAccessibilityPaneTitle = accessibilityPaneTitle; - notifyAccessibilityStateChanged(AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_TITLE); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_TITLE); } } @@ -7282,10 +7315,14 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * * {@see #setAccessibilityPaneTitle}. */ - public CharSequence getAccessibilityPaneTitle() { + @Nullable public CharSequence getAccessibilityPaneTitle() { return mAccessibilityPaneTitle; } + private boolean isAccessibilityPane() { + return mAccessibilityPaneTitle != null; + } + /** * Sends an accessibility event of the given type. If accessibility is * not enabled this method has no effect. The default implementation calls @@ -7849,6 +7886,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, structure.setAutofillHints(getAutofillHints()); structure.setAutofillValue(getAutofillValue()); } + structure.setImportantForAutofill(getImportantForAutofill()); } int ignoredParentLeft = 0; @@ -7930,12 +7968,26 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * optimal implementation providing this data. */ public void onProvideVirtualStructure(ViewStructure structure) { - AccessibilityNodeProvider provider = getAccessibilityNodeProvider(); + onProvideVirtualStructureCompat(structure, false); + } + + /** + * Fallback implementation to populate a ViewStructure from accessibility state. + * + * @param structure The structure to populate. + * @param forAutofill Whether the structure is needed for autofill. + */ + private void onProvideVirtualStructureCompat(ViewStructure structure, boolean forAutofill) { + final AccessibilityNodeProvider provider = getAccessibilityNodeProvider(); if (provider != null) { - AccessibilityNodeInfo info = createAccessibilityNodeInfo(); + if (android.view.autofill.Helper.sVerbose && forAutofill) { + Log.v(VIEW_LOG_TAG, "onProvideVirtualStructureCompat() for " + this); + } + + final AccessibilityNodeInfo info = createAccessibilityNodeInfo(); structure.setChildCount(1); - ViewStructure root = structure.newChild(0); - populateVirtualStructure(root, provider, info); + final ViewStructure root = structure.newChild(0); + populateVirtualStructure(root, provider, info, forAutofill); info.recycle(); } } @@ -7964,12 +8016,18 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * <li>Call {@link android.view.autofill.AutofillManager#notifyViewEntered(View, int, Rect)} * and/or {@link android.view.autofill.AutofillManager#notifyViewExited(View, int)} * when the focused virtual child changed. + * <li>Override {@link #isVisibleToUserForAutofill(int)} to allow the platform to query + * whether a given virtual view is visible to the user in order to support triggering + * save when all views of interest go away. * <li>Call * {@link android.view.autofill.AutofillManager#notifyValueChanged(View, int, AutofillValue)} * when the value of a virtual child changed. * <li>Call {@link * android.view.autofill.AutofillManager#notifyViewVisibilityChanged(View, int, boolean)} * when the visibility of a virtual child changed. + * <li>Call + * {@link android.view.autofill.AutofillManager#notifyViewClicked(View, int)} when a virtual + * child is clicked. * <li>Call {@link AutofillManager#commit()} when the autofill context of the view structure * changed and the current context should be committed (for example, when the user tapped * a {@code SUBMIT} button in an HTML page). @@ -7999,6 +8057,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * @see #AUTOFILL_FLAG_INCLUDE_NOT_IMPORTANT_VIEWS */ public void onProvideAutofillVirtualStructure(ViewStructure structure, int flags) { + if (mContext.isAutofillCompatibilityEnabled()) { + onProvideVirtualStructureCompat(structure, true); + } } /** @@ -8076,10 +8137,34 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * @attr ref android.R.styleable#Theme_autofilledHighlight */ public void autofill(@NonNull @SuppressWarnings("unused") SparseArray<AutofillValue> values) { + if (!mContext.isAutofillCompatibilityEnabled()) { + return; + } + final AccessibilityNodeProvider provider = getAccessibilityNodeProvider(); + if (provider == null) { + return; + } + final int valueCount = values.size(); + for (int i = 0; i < valueCount; i++) { + final AutofillValue value = values.valueAt(i); + if (value.isText()) { + final int virtualId = values.keyAt(i); + final CharSequence text = value.getTextValue(); + final Bundle arguments = new Bundle(); + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text); + provider.performAction(virtualId, AccessibilityNodeInfo.ACTION_SET_TEXT, arguments); + } + } } /** - * Gets the unique identifier of this view in the screen, for autofill purposes. + * Gets the unique, logical identifier of this view in the activity, for autofill purposes. + * + * <p>The autofill id is created on demand, unless it is explicitly set by + * {@link #setAutofillId(AutofillId)}. + * + * <p>See {@link #setAutofillId(AutofillId)} for more info. * * @return The View's autofill id. */ @@ -8093,6 +8178,73 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** + * Sets the unique, logical identifier of this view in the activity, for autofill purposes. + * + * <p>The autofill id is created on demand, and this method should only be called when a view is + * reused after {@link #dispatchProvideAutofillStructure(ViewStructure, int)} is called, as + * that method creates a snapshot of the view that is passed along to the autofill service. + * + * <p>This method is typically used when view subtrees are recycled to represent different + * content* —in this case, the autofill id can be saved before the view content is swapped + * out, and restored later when it's swapped back in. For example: + * + * <pre> + * EditText reusableView = ...; + * ViewGroup parentView = ...; + * AutofillManager afm = ...; + * + * // Swap out the view and change its contents + * AutofillId oldId = reusableView.getAutofillId(); + * CharSequence oldText = reusableView.getText(); + * parentView.removeView(reusableView); + * AutofillId newId = afm.getNextAutofillId(); + * reusableView.setText("New I am"); + * reusableView.setAutofillId(newId); + * parentView.addView(reusableView); + * + * // Later, swap the old content back in + * parentView.removeView(reusableView); + * reusableView.setAutofillId(oldId); + * reusableView.setText(oldText); + * parentView.addView(reusableView); + * </pre> + * + * @param id an autofill ID that is unique in the {@link android.app.Activity} hosting the view, + * or {@code null} to reset it. Usually it's an id previously allocated to another view (and + * obtained through {@link #getAutofillId()}), or a new value obtained through + * {@link AutofillManager#getNextAutofillId()}. + * + * @throws IllegalStateException if the view is already {@link #isAttachedToWindow() attached to + * a window}. + * + * @throws IllegalArgumentException if the id is an autofill id associated with a virtual view. + */ + public void setAutofillId(@Nullable AutofillId id) { + // TODO(b/37566627): add unit / CTS test for all possible combinations below + if (android.view.autofill.Helper.sVerbose) { + Log.v(VIEW_LOG_TAG, "setAutofill(): from " + mAutofillId + " to " + id); + } + if (isAttachedToWindow()) { + throw new IllegalStateException("Cannot set autofill id when view is attached"); + } + if (id != null && id.isVirtual()) { + throw new IllegalStateException("Cannot set autofill id assigned to virtual views"); + } + if (id == null && (mPrivateFlags3 & PFLAG3_AUTOFILLID_EXPLICITLY_SET) == 0) { + // Ignore reset because it was never explicitly set before. + return; + } + mAutofillId = id; + if (id != null) { + mAutofillViewId = id.getViewId(); + mPrivateFlags3 |= PFLAG3_AUTOFILLID_EXPLICITLY_SET; + } else { + mAutofillViewId = NO_ID; + mPrivateFlags3 &= ~PFLAG3_AUTOFILLID_EXPLICITLY_SET; + } + } + + /** * Describes the autofill type of this view, so an * {@link android.service.autofill.AutofillService} can create the proper {@link AutofillValue} * when autofilling the view. @@ -8309,6 +8461,11 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } } + // If the app developer explicitly set hints for it, it's important. + if (getAutofillHints() != null) { + return true; + } + // Otherwise, assume it's not important... return false; } @@ -8329,9 +8486,10 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } private void populateVirtualStructure(ViewStructure structure, - AccessibilityNodeProvider provider, AccessibilityNodeInfo info) { + AccessibilityNodeProvider provider, AccessibilityNodeInfo info, + boolean forAutofill) { structure.setId(AccessibilityNodeInfo.getVirtualDescendantId(info.getSourceNodeId()), - null, null, null); + null, null, info.getViewIdResourceName()); Rect rect = structure.getTempRect(); info.getBoundsInParent(rect); structure.setDimens(rect.left, rect.top, 0, 0, rect.width(), rect.height()); @@ -8364,21 +8522,54 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (info.isContextClickable()) { structure.setContextClickable(true); } + if (forAutofill) { + structure.setAutofillId(new AutofillId(getAutofillId(), + AccessibilityNodeInfo.getVirtualDescendantId(info.getSourceNodeId()))); + } CharSequence cname = info.getClassName(); structure.setClassName(cname != null ? cname.toString() : null); structure.setContentDescription(info.getContentDescription()); - if ((info.getText() != null || info.getError() != null)) { - structure.setText(info.getText(), info.getTextSelectionStart(), - info.getTextSelectionEnd()); + if (forAutofill) { + final int maxTextLength = info.getMaxTextLength(); + if (maxTextLength != -1) { + structure.setMaxTextLength(maxTextLength); + } + structure.setHint(info.getHintText()); + } + CharSequence text = info.getText(); + boolean hasText = text != null || info.getError() != null; + if (hasText) { + structure.setText(text, info.getTextSelectionStart(), info.getTextSelectionEnd()); + } + if (forAutofill) { + if (info.isEditable()) { + structure.setDataIsSensitive(true); + if (hasText) { + structure.setAutofillType(AUTOFILL_TYPE_TEXT); + structure.setAutofillValue(AutofillValue.forText(text)); + } + int inputType = info.getInputType(); + if (inputType == 0 && info.isPassword()) { + inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_VARIATION_PASSWORD; + } + structure.setInputType(inputType); + } else { + structure.setDataIsSensitive(false); + } } final int NCHILDREN = info.getChildCount(); if (NCHILDREN > 0) { structure.setChildCount(NCHILDREN); for (int i=0; i<NCHILDREN; i++) { + if (AccessibilityNodeInfo.getVirtualDescendantId(info.getChildNodeIds().get(i)) + == AccessibilityNodeProvider.HOST_VIEW_ID) { + Log.e(VIEW_LOG_TAG, "Virtual view pointing to its host. Ignoring"); + continue; + } AccessibilityNodeInfo cinfo = provider.createAccessibilityNodeInfo( AccessibilityNodeInfo.getVirtualDescendantId(info.getChildId(i))); ViewStructure child = structure.newChild(i); - populateVirtualStructure(child, provider, cinfo); + populateVirtualStructure(child, provider, cinfo, forAutofill); cinfo.recycle(); } } @@ -8707,6 +8898,31 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** + * Computes whether this virtual autofill view is visible to the user. + * + * <p><b>Note: </b>By default it returns {@code true}, but views providing a virtual hierarchy + * view must override it. + * + * @return Whether the view is visible on the screen. + */ + public boolean isVisibleToUserForAutofill(int virtualId) { + if (mContext.isAutofillCompatibilityEnabled()) { + final AccessibilityNodeProvider provider = getAccessibilityNodeProvider(); + if (provider != null) { + final AccessibilityNodeInfo node = provider.createAccessibilityNodeInfo(virtualId); + if (node != null) { + return node.isVisibleToUser(); + } + // if node is null, assume it's not visible anymore + } else { + Log.w(VIEW_LOG_TAG, "isVisibleToUserForAutofill(" + virtualId + "): no provider"); + } + return false; + } + return true; + } + + /** * Computes whether this view is visible to the user. Such a view is * attached, visible, all its predecessors are visible, it is not clipped * entirely by its predecessors, and has an alpha greater than zero. @@ -8715,7 +8931,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * * @hide */ - protected boolean isVisibleToUser() { + public boolean isVisibleToUser() { return isVisibleToUser(null); } @@ -8924,9 +9140,9 @@ public class View implements Drawable.Callback, KeyEvent.Callback, final boolean nonEmptyDesc = contentDescription != null && contentDescription.length() > 0; if (nonEmptyDesc && getImportantForAccessibility() == IMPORTANT_FOR_ACCESSIBILITY_AUTO) { setImportantForAccessibility(IMPORTANT_FOR_ACCESSIBILITY_YES); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } else { - notifyAccessibilityStateChanged( + notifyViewAccessibilityStateChangedIfNeeded( AccessibilityEvent.CONTENT_CHANGE_TYPE_CONTENT_DESCRIPTION); } } @@ -8959,7 +9175,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, return; } mAccessibilityTraversalBeforeId = beforeId; - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } /** @@ -9002,7 +9219,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, return; } mAccessibilityTraversalAfterId = afterId; - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } /** @@ -9044,7 +9262,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, && mID == View.NO_ID) { mID = generateViewId(); } - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } /** @@ -10544,7 +10763,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (pflags3 != mPrivateFlags3) { mPrivateFlags3 = pflags3; - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } } @@ -11374,7 +11594,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mPrivateFlags2 &= ~PFLAG2_ACCESSIBILITY_LIVE_REGION_MASK; mPrivateFlags2 |= (mode << PFLAG2_ACCESSIBILITY_LIVE_REGION_SHIFT) & PFLAG2_ACCESSIBILITY_LIVE_REGION_MASK; - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } } @@ -11431,9 +11652,10 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mPrivateFlags2 |= (mode << PFLAG2_IMPORTANT_FOR_ACCESSIBILITY_SHIFT) & PFLAG2_IMPORTANT_FOR_ACCESSIBILITY_MASK; if (!maySkipNotify || oldIncludeForAccessibility != includeForAccessibility()) { - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } else { - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } } } @@ -11519,7 +11741,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, return mode == IMPORTANT_FOR_ACCESSIBILITY_YES || isActionableForAccessibility() || hasListenersForAccessibility() || getAccessibilityNodeProvider() != null || getAccessibilityLiveRegion() != ACCESSIBILITY_LIVE_REGION_NONE - || (mAccessibilityPaneTitle != null); + || isAccessibilityPane(); } /** @@ -11609,8 +11831,51 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * * @hide */ - public void notifyAccessibilityStateChanged(int changeType) { - notifyAccessibilityStateChanged(this, changeType); + public void notifyViewAccessibilityStateChangedIfNeeded(int changeType) { + if (!AccessibilityManager.getInstance(mContext).isEnabled() || mAttachInfo == null) { + return; + } + + // Changes to views with a pane title count as window state changes, as the pane title + // marks them as significant parts of the UI. + if ((changeType != AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE) + && isAccessibilityPane()) { + // If the pane isn't visible, content changed events are sufficient unless we're + // reporting that the view just disappeared + if ((getVisibility() == VISIBLE) + || (changeType == AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED)) { + final AccessibilityEvent event = AccessibilityEvent.obtain(); + event.setEventType(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED); + event.setContentChangeTypes(changeType); + event.setSource(this); + onPopulateAccessibilityEvent(event); + if (mParent != null) { + try { + mParent.requestSendAccessibilityEvent(this, event); + } catch (AbstractMethodError e) { + Log.e(VIEW_LOG_TAG, mParent.getClass().getSimpleName() + + " does not fully implement ViewParent", e); + } + } + return; + } + } + + // If this is a live region, we should send a subtree change event + // from this view immediately. Otherwise, we can let it propagate up. + if (getAccessibilityLiveRegion() != ACCESSIBILITY_LIVE_REGION_NONE) { + final AccessibilityEvent event = AccessibilityEvent.obtain(); + event.setEventType(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); + event.setContentChangeTypes(changeType); + sendAccessibilityEventUnchecked(event); + } else if (mParent != null) { + try { + mParent.notifySubtreeAccessibilityStateChanged(this, this, changeType); + } catch (AbstractMethodError e) { + Log.e(VIEW_LOG_TAG, mParent.getClass().getSimpleName() + + " does not fully implement ViewParent", e); + } + } } /** @@ -11624,42 +11889,23 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * * @hide */ - public void notifyAccessibilitySubtreeChanged() { - if ((mPrivateFlags2 & PFLAG2_SUBTREE_ACCESSIBILITY_STATE_CHANGED) == 0) { - mPrivateFlags2 |= PFLAG2_SUBTREE_ACCESSIBILITY_STATE_CHANGED; - notifyAccessibilityStateChanged(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE); - } - } - - void notifyAccessibilityStateChanged(View source, int changeType) { + public void notifySubtreeAccessibilityStateChangedIfNeeded() { if (!AccessibilityManager.getInstance(mContext).isEnabled() || mAttachInfo == null) { return; } - // Changes to views with a pane title count as window state changes, as the pane title - // marks them as significant parts of the UI. - if (!TextUtils.isEmpty(getAccessibilityPaneTitle())) { - final AccessibilityEvent event = AccessibilityEvent.obtain(); - event.setEventType(AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED); - event.setContentChangeTypes(changeType); - onPopulateAccessibilityEvent(event); + + if ((mPrivateFlags2 & PFLAG2_SUBTREE_ACCESSIBILITY_STATE_CHANGED) == 0) { + mPrivateFlags2 |= PFLAG2_SUBTREE_ACCESSIBILITY_STATE_CHANGED; if (mParent != null) { try { - mParent.requestSendAccessibilityEvent(this, event); + mParent.notifySubtreeAccessibilityStateChanged( + this, this, AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE); } catch (AbstractMethodError e) { - Log.e(VIEW_LOG_TAG, mParent.getClass().getSimpleName() - + " does not fully implement ViewParent", e); + Log.e(VIEW_LOG_TAG, mParent.getClass().getSimpleName() + + " does not fully implement ViewParent", e); } } } - - if (mParent != null) { - try { - mParent.notifySubtreeAccessibilityStateChanged(this, source, changeType); - } catch (AbstractMethodError e) { - Log.e(VIEW_LOG_TAG, mParent.getClass().getSimpleName() - + " does not fully implement ViewParent", e); - } - } } /** @@ -11679,10 +11925,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, /** * Reset the flag indicating the accessibility state of the subtree rooted * at this view changed. - * - * @hide */ - public void resetSubtreeAccessibilityStateChanged() { + void resetSubtreeAccessibilityStateChanged() { mPrivateFlags2 &= ~PFLAG2_SUBTREE_ACCESSIBILITY_STATE_CHANGED; } @@ -11843,7 +12087,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, || getAccessibilitySelectionEnd() != end) && (start == end)) { setAccessibilitySelection(start, end); - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); return true; } } break; @@ -12643,7 +12888,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } if (!TextUtils.isEmpty(getAccessibilityPaneTitle())) { if (isVisible != oldVisible) { - notifyAccessibilityStateChanged(isVisible + notifyViewAccessibilityStateChangedIfNeeded(isVisible ? AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_APPEARED : AccessibilityEvent.CONTENT_CHANGE_TYPE_PANE_DISAPPEARED); } @@ -13672,11 +13917,15 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mAttachInfo.mUnbufferedDispatchRequested = true; } + private boolean hasSize() { + return (mBottom > mTop) && (mRight > mLeft); + } + private boolean canTakeFocus() { return ((mViewFlags & VISIBILITY_MASK) == VISIBLE) && ((mViewFlags & FOCUSABLE) == FOCUSABLE) && ((mViewFlags & ENABLED_MASK) == ENABLED) - && (sCanFocusZeroSized || !isLayoutValid() || (mBottom > mTop) && (mRight > mLeft)); + && (sCanFocusZeroSized || !isLayoutValid() || hasSize()); } /** @@ -13737,7 +13986,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, || focusableChangedByAuto == 0 || viewRootImpl == null || viewRootImpl.mThread == Thread.currentThread()) { - shouldNotifyFocusableAvailable = true; + shouldNotifyFocusableAvailable = canTakeFocus(); } } } @@ -13756,11 +14005,10 @@ public class View implements Drawable.Callback, KeyEvent.Callback, needGlobalAttributesUpdate(true); - // a view becoming visible is worth notifying the parent - // about in case nothing has focus. even if this specific view - // isn't focusable, it may contain something that is, so let - // the root view try to give this focus if nothing else does. - shouldNotifyFocusableAvailable = true; + // a view becoming visible is worth notifying the parent about in case nothing has + // focus. Even if this specific view isn't focusable, it may contain something that + // is, so let the root view try to give this focus if nothing else does. + shouldNotifyFocusableAvailable = hasSize(); } } @@ -13769,16 +14017,14 @@ public class View implements Drawable.Callback, KeyEvent.Callback, // a view becoming enabled should notify the parent as long as the view is also // visible and the parent wasn't already notified by becoming visible during this // setFlags invocation. - shouldNotifyFocusableAvailable = true; + shouldNotifyFocusableAvailable = canTakeFocus(); } else { - if (hasFocus()) clearFocus(); + if (isFocused()) clearFocus(); } } - if (shouldNotifyFocusableAvailable) { - if (mParent != null && canTakeFocus()) { - mParent.focusableViewAvailable(this); - } + if (shouldNotifyFocusableAvailable && mParent != null) { + mParent.focusableViewAvailable(this); } /* Check if the GONE bit has changed */ @@ -13860,7 +14106,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, ((!(mParent instanceof ViewGroup)) || ((ViewGroup) mParent).isShown())) { dispatchVisibilityAggregated(newVisibility == VISIBLE); } - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -13902,16 +14148,23 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } if (accessibilityEnabled) { + // If we're an accessibility pane and the visibility changed, we already have sent + // a state change, so we really don't need to report other changes. + if (isAccessibilityPane()) { + changed &= ~VISIBILITY_MASK; + } if ((changed & FOCUSABLE) != 0 || (changed & VISIBILITY_MASK) != 0 || (changed & CLICKABLE) != 0 || (changed & LONG_CLICKABLE) != 0 || (changed & CONTEXT_CLICKABLE) != 0) { if (oldIncludeForAccessibility != includeForAccessibility()) { - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } else { - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } } else if ((changed & ENABLED_MASK) != 0) { - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } } } @@ -13945,13 +14198,10 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * @param oldt Previous vertical scroll origin. */ protected void onScrollChanged(int l, int t, int oldl, int oldt) { - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); - ViewRootImpl root = getViewRootImpl(); - if (root != null) { - root.getAccessibilityState() - .getSendViewScrolledAccessibilityEvent() - .post(this, /* dx */ l - oldl, /* dy */ t - oldt); + if (AccessibilityManager.getInstance(mContext).isEnabled()) { + postSendViewScrolledAccessibilityEventCallback(l - oldl, t - oldt); } mBackgroundSizeChanged = true; @@ -14094,7 +14344,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** - * Return the width of the your view. + * Return the width of your view. * * @return The width of your view, in pixels. */ @@ -14347,7 +14597,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -14391,7 +14641,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -14435,7 +14685,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -14472,7 +14722,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -14509,7 +14759,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -14597,6 +14847,28 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** + * Returns whether or not a pivot has been set by a call to {@link #setPivotX(float)} or + * {@link #setPivotY(float)}. If no pivot has been set then the pivot will be the center + * of the view. + * + * @return True if a pivot has been set, false if the default pivot is being used + */ + public boolean isPivotSet() { + return mRenderNode.isPivotExplicitlySet(); + } + + /** + * Clears any pivot previously set by a call to {@link #setPivotX(float)} or + * {@link #setPivotY(float)}. After calling this {@link #isPivotSet()} will be false + * and the pivot used for rotation will return to default of being centered on the view. + */ + public void resetPivot() { + if (mRenderNode.resetPivot()) { + invalidateViewProperty(false, false); + } + } + + /** * The opacity of the view. This is a value from 0 to 1, where 0 means the view is * completely transparent and 1 means the view is completely opaque. * @@ -14656,10 +14928,6 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * ImageView with only the foreground image. The default implementation returns true; subclasses * should override if they have cases which can be optimized.</p> * - * <p>The current implementation of the saveLayer and saveLayerAlpha methods in {@link Canvas} - * necessitates that a View return true if it uses the methods internally without passing the - * {@link Canvas#CLIP_TO_LAYER_SAVE_FLAG}.</p> - * * <p><strong>Note:</strong> The return value of this method is ignored if {@link * #forceHasOverlappingRendering(boolean)} has been called on this view.</p> * @@ -14712,7 +14980,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (mTransformationInfo.mAlpha != alpha) { // Report visibility changes, which can affect children, to accessibility if ((alpha == 0) ^ (mTransformationInfo.mAlpha == 0)) { - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } mTransformationInfo.mAlpha = alpha; if (onSetAlpha((int) (alpha * 255))) { @@ -15214,7 +15482,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -15248,7 +15516,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, invalidateViewProperty(false, true); invalidateParentIfNeededAndWasQuickRejected(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -15418,7 +15686,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, public void invalidateOutline() { rebuildOutline(); - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); invalidateViewProperty(false, false); } @@ -15459,14 +15727,61 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** - * @hide + * Sets the color of the spot shadow that is drawn when the view has a positive Z or + * elevation value. + * <p> + * By default the shadow color is black. Generally, this color will be opaque so the intensity + * of the shadow is consistent between different views with different colors. + * <p> + * The opacity of the final spot shadow is a function of the shadow caster height, the + * alpha channel of the outlineSpotShadowColor (typically opaque), and the + * {@link android.R.attr#spotShadowAlpha} theme attribute. + * + * @attr ref android.R.styleable#View_outlineSpotShadowColor + * @param color The color this View will cast for its elevation spot shadow. + */ + public void setOutlineSpotShadowColor(@ColorInt int color) { + if (mRenderNode.setSpotShadowColor(color)) { + invalidateViewProperty(true, true); + } + } + + /** + * @return The shadow color set by {@link #setOutlineSpotShadowColor(int)}, or black if nothing + * was set + */ + public @ColorInt int getOutlineSpotShadowColor() { + return mRenderNode.getSpotShadowColor(); + } + + /** + * Sets the color of the ambient shadow that is drawn when the view has a positive Z or + * elevation value. + * <p> + * By default the shadow color is black. Generally, this color will be opaque so the intensity + * of the shadow is consistent between different views with different colors. + * <p> + * The opacity of the final ambient shadow is a function of the shadow caster height, the + * alpha channel of the outlineAmbientShadowColor (typically opaque), and the + * {@link android.R.attr#ambientShadowAlpha} theme attribute. + * + * @attr ref android.R.styleable#View_outlineAmbientShadowColor + * @param color The color this View will cast for its elevation shadow. */ - public void setShadowColor(@ColorInt int color) { - if (mRenderNode.setShadowColor(color)) { + public void setOutlineAmbientShadowColor(@ColorInt int color) { + if (mRenderNode.setAmbientShadowColor(color)) { invalidateViewProperty(true, true); } } + /** + * @return The shadow color set by {@link #setOutlineAmbientShadowColor(int)}, or black if + * nothing was set + */ + public @ColorInt int getOutlineAmbientShadowColor() { + return mRenderNode.getAmbientShadowColor(); + } + /** @hide */ public void setRevealClip(boolean shouldClip, float x, float y, float radius) { @@ -15613,7 +15928,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } invalidateParentIfNeeded(); } - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -15661,7 +15976,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } invalidateParentIfNeeded(); } - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } } @@ -16539,6 +16854,18 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** + * Post a callback to send a {@link AccessibilityEvent#TYPE_VIEW_SCROLLED} event. + * This event is sent at most once every + * {@link ViewConfiguration#getSendRecurringAccessibilityEventsInterval()}. + */ + private void postSendViewScrolledAccessibilityEventCallback(int dx, int dy) { + if (mSendViewScrolledAccessibilityEvent == null) { + mSendViewScrolledAccessibilityEvent = new SendViewScrolledAccessibilityEvent(); + } + mSendViewScrolledAccessibilityEvent.post(dx, dy); + } + + /** * Called by a parent to request that a child update its values for mScrollX * and mScrollY if necessary. This will typically be done if the child is * animating a scroll using a {@link android.widget.Scroller Scroller} @@ -17793,13 +18120,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, removeUnsetPressCallback(); removeLongPressCallback(); removePerformClickCallback(); - if (mAttachInfo != null - && mAttachInfo.mViewRootImpl.mAccessibilityState != null - && mAttachInfo.mViewRootImpl.mAccessibilityState.isScrollEventSenderInitialized()) { - mAttachInfo.mViewRootImpl.mAccessibilityState - .getSendViewScrolledAccessibilityEvent() - .cancelIfPendingFor(this); - } + cancel(mSendViewScrolledAccessibilityEvent); stopNestedScroll(); // Anything that started animating right before detach should already @@ -17847,19 +18168,20 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * currently attached to. */ public WindowId getWindowId() { - if (mAttachInfo == null) { + AttachInfo ai = mAttachInfo; + if (ai == null) { return null; } - if (mAttachInfo.mWindowId == null) { + if (ai.mWindowId == null) { try { - mAttachInfo.mIWindowId = mAttachInfo.mSession.getWindowId( - mAttachInfo.mWindowToken); - mAttachInfo.mWindowId = new WindowId( - mAttachInfo.mIWindowId); + ai.mIWindowId = ai.mSession.getWindowId(ai.mWindowToken); + if (ai.mIWindowId != null) { + ai.mWindowId = new WindowId(ai.mIWindowId); + } } catch (RemoteException e) { } } - return mAttachInfo.mWindowId; + return ai.mWindowId; } /** @@ -18255,7 +18577,17 @@ public class View implements Drawable.Callback, KeyEvent.Callback, // Hence prevent the same autofill view id from being restored multiple times. ((BaseSavedState) state).mSavedData &= ~BaseSavedState.AUTOFILL_ID; - mAutofillViewId = baseState.mAutofillViewId; + if ((mPrivateFlags3 & PFLAG3_AUTOFILLID_EXPLICITLY_SET) != 0) { + // Ignore when view already set it through setAutofillId(); + if (android.view.autofill.Helper.sDebug) { + Log.d(VIEW_LOG_TAG, "onRestoreInstanceState(): not setting autofillId to " + + baseState.mAutofillViewId + " because view explicitly set it to " + + mAutofillId); + } + } else { + mAutofillViewId = baseState.mAutofillViewId; + mAutofillId = null; // will be set on demand by getAutofillId() + } } } } @@ -19892,22 +20224,20 @@ public class View implements Drawable.Callback, KeyEvent.Callback, int solidColor = getSolidColor(); if (solidColor == 0) { - final int flags = Canvas.HAS_ALPHA_LAYER_SAVE_FLAG; - if (drawTop) { - canvas.saveLayer(left, top, right, top + length, null, flags); + canvas.saveUnclippedLayer(left, top, right, top + length); } if (drawBottom) { - canvas.saveLayer(left, bottom - length, right, bottom, null, flags); + canvas.saveUnclippedLayer(left, bottom - length, right, bottom); } if (drawLeft) { - canvas.saveLayer(left, top, left + length, bottom, null, flags); + canvas.saveUnclippedLayer(left, top, left + length, bottom); } if (drawRight) { - canvas.saveLayer(right - length, top, right, bottom, null, flags); + canvas.saveUnclippedLayer(right - length, top, right, bottom); } } else { scrollabilityCache.setFadeColor(solidColor); @@ -20427,7 +20757,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mForegroundInfo.mBoundsChanged = true; } - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } return changed; } @@ -21871,7 +22201,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, if (selected) { sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_SELECTED); } else { - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_UNDEFINED); } } } @@ -23458,8 +23789,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, * constructor variant is only useful when the {@link #onProvideShadowMetrics(Point, Point)} * and {@link #onDrawShadow(Canvas)} methods are also overridden in order * to supply the drag shadow's dimensions and appearance without - * reference to any View object. If they are not overridden, then the result is an - * invisible drag shadow. + * reference to any View object. */ public DragShadowBuilder() { mView = new WeakReference<View>(null); @@ -23605,9 +23935,19 @@ public class View implements Drawable.Callback, KeyEvent.Callback, Point shadowTouchPoint = new Point(); shadowBuilder.onProvideShadowMetrics(shadowSize, shadowTouchPoint); - if ((shadowSize.x <= 0) || (shadowSize.y <= 0) + if ((shadowSize.x < 0) || (shadowSize.y < 0) || (shadowTouchPoint.x < 0) || (shadowTouchPoint.y < 0)) { - throw new IllegalStateException("Drag shadow dimensions must be positive"); + throw new IllegalStateException("Drag shadow dimensions must not be negative"); + } + + // Create 1x1 surface when zero surface size is specified because SurfaceControl.Builder + // does not accept zero size surface. + if (shadowSize.x == 0 || shadowSize.y == 0) { + if (!sAcceptZeroSizeDragShadow) { + throw new IllegalStateException("Drag shadow dimensions must be positive"); + } + shadowSize.x = 1; + shadowSize.y = 1; } if (ViewDebug.DEBUG_DRAG) { @@ -25540,26 +25880,19 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** - * Interface definition for a callback to be invoked when a hardware key event is - * dispatched to this view during the fallback phase. This means no view in the hierarchy - * has handled this event. + * Interface definition for a callback to be invoked when a hardware key event hasn't + * been handled by the view hierarchy. */ - public interface OnKeyFallbackListener { + public interface OnUnhandledKeyEventListener { /** - * Called when a hardware key is dispatched to a view in the fallback phase. This allows - * listeners to respond to events after the view hierarchy has had a chance to respond. - * <p>Key presses in software keyboards will generally NOT trigger this method, - * although some may elect to do so in some situations. Do not assume a - * software input method has to be key-based; even if it is, it may use key presses - * in a different way than you expect, so there is no way to reliably catch soft - * input key presses. + * Called when a hardware key is dispatched to a view after being unhandled during normal + * {@link KeyEvent} dispatch. * * @param v The view the key has been dispatched to. - * @param event The KeyEvent object containing full information about - * the event. - * @return True if the listener has consumed the event, false otherwise. + * @param event The KeyEvent object containing information about the event. + * @return {@code true} if the listener has consumed the event, {@code false} otherwise. */ - boolean onKeyFallback(View v, KeyEvent event); + boolean onUnhandledKeyEvent(View v, KeyEvent event); } /** @@ -26458,6 +26791,53 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** + * Resuable callback for sending + * {@link AccessibilityEvent#TYPE_VIEW_SCROLLED} accessibility event. + */ + private class SendViewScrolledAccessibilityEvent implements Runnable { + public volatile boolean mIsPending; + public int mDeltaX; + public int mDeltaY; + + public void post(int dx, int dy) { + mDeltaX += dx; + mDeltaY += dy; + if (!mIsPending) { + mIsPending = true; + postDelayed(this, ViewConfiguration.getSendRecurringAccessibilityEventsInterval()); + } + } + + @Override + public void run() { + if (AccessibilityManager.getInstance(mContext).isEnabled()) { + AccessibilityEvent event = AccessibilityEvent.obtain( + AccessibilityEvent.TYPE_VIEW_SCROLLED); + event.setScrollDeltaX(mDeltaX); + event.setScrollDeltaY(mDeltaY); + sendAccessibilityEventUnchecked(event); + } + reset(); + } + + private void reset() { + mIsPending = false; + mDeltaX = 0; + mDeltaY = 0; + } + } + + /** + * Remove the pending callback for sending a + * {@link AccessibilityEvent#TYPE_VIEW_SCROLLED} accessibility event. + */ + private void cancel(@Nullable SendViewScrolledAccessibilityEvent callback) { + if (callback == null || !callback.mIsPending) return; + removeCallbacks(callback); + callback.reset(); + } + + /** * <p> * This class represents a delegate that can be registered in a {@link View} * to enhance accessibility support via composition rather via inheritance. @@ -26903,6 +27283,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, stream.addProperty("drawing:scaleY", getScaleY()); stream.addProperty("drawing:pivotX", getPivotX()); stream.addProperty("drawing:pivotY", getPivotY()); + stream.addProperty("drawing:clipBounds", + mClipBounds == null ? null : mClipBounds.toString()); stream.addProperty("drawing:opaque", isOpaque()); stream.addProperty("drawing:alpha", getAlpha()); stream.addProperty("drawing:transitionAlpha", getTransitionAlpha()); @@ -26914,6 +27296,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback, stream.addProperty("drawing:willNotCacheDrawing", willNotCacheDrawing()); stream.addProperty("drawing:drawingCacheEnabled", isDrawingCacheEnabled()); stream.addProperty("drawing:overlappingRendering", hasOverlappingRendering()); + stream.addProperty("drawing:outlineAmbientShadowColor", getOutlineAmbientShadowColor()); + stream.addProperty("drawing:outlineSpotShadowColor", getOutlineSpotShadowColor()); // focus stream.addProperty("focus:hasFocus", hasFocus()); @@ -27074,7 +27458,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mTooltipInfo.mTooltipPopup.show(this, x, y, fromTouch, mTooltipInfo.mTooltipText); mAttachInfo.mTooltipHost = this; // The available accessibility actions have changed - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded(CONTENT_CHANGE_TYPE_UNDEFINED); return true; } @@ -27094,7 +27478,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback, mAttachInfo.mTooltipHost = null; } // The available accessibility actions have changed - notifyAccessibilityStateChanged(CONTENT_CHANGE_TYPE_UNDEFINED); + notifyViewAccessibilityStateChangedIfNeeded(CONTENT_CHANGE_TYPE_UNDEFINED); } private boolean showLongClickTooltip(int x, int y) { @@ -27199,20 +27583,44 @@ public class View implements Drawable.Callback, KeyEvent.Callback, } /** + * @return {@code true} if the default focus highlight is enabled, {@code false} otherwies. + * @hide + */ + @TestApi + public static boolean isDefaultFocusHighlightEnabled() { + return sUseDefaultFocusHighlight; + } + + /** + * Dispatch a previously unhandled {@link KeyEvent} to this view. Unlike normal key dispatch, + * this dispatches to ALL child views until it is consumed. The dispatch order is z-order + * (visually on-top views first). + * + * @param evt the previously unhandled {@link KeyEvent}. + * @return the {@link View} which consumed the event or {@code null} if not consumed. + */ + View dispatchUnhandledKeyEvent(KeyEvent evt) { + if (onUnhandledKeyEvent(evt)) { + return this; + } + return null; + } + + /** * Allows this view to handle {@link KeyEvent}s which weren't handled by normal dispatch. This * occurs after the normal view hierarchy dispatch, but before the window callback. By default, * this will dispatch into all the listeners registered via - * {@link #addKeyFallbackListener(OnKeyFallbackListener)} in last-in-first-out order (most - * recently added will receive events first). + * {@link #addOnUnhandledKeyEventListener(OnUnhandledKeyEventListener)} in last-in-first-out + * order (most recently added will receive events first). * - * @param event A not-previously-handled event. + * @param event An unhandled event. * @return {@code true} if the event was handled, {@code false} otherwise. - * @see #addKeyFallbackListener + * @see #addOnUnhandledKeyEventListener */ - public boolean onKeyFallback(@NonNull KeyEvent event) { - if (mListenerInfo != null && mListenerInfo.mKeyFallbackListeners != null) { - for (int i = mListenerInfo.mKeyFallbackListeners.size() - 1; i >= 0; --i) { - if (mListenerInfo.mKeyFallbackListeners.get(i).onKeyFallback(this, event)) { + boolean onUnhandledKeyEvent(@NonNull KeyEvent event) { + if (mListenerInfo != null && mListenerInfo.mUnhandledKeyListeners != null) { + for (int i = mListenerInfo.mUnhandledKeyListeners.size() - 1; i >= 0; --i) { + if (mListenerInfo.mUnhandledKeyListeners.get(i).onUnhandledKeyEvent(this, event)) { return true; } } @@ -27220,31 +27628,47 @@ public class View implements Drawable.Callback, KeyEvent.Callback, return false; } + boolean hasUnhandledKeyListener() { + return (mListenerInfo != null && mListenerInfo.mUnhandledKeyListeners != null + && !mListenerInfo.mUnhandledKeyListeners.isEmpty()); + } + /** - * Adds a listener which will receive unhandled {@link KeyEvent}s. - * @param listener the receiver of fallback {@link KeyEvent}s. - * @see #onKeyFallback(KeyEvent) + * Adds a listener which will receive unhandled {@link KeyEvent}s. This must be called on the + * UI thread. + * + * @param listener a receiver of unhandled {@link KeyEvent}s. + * @see #removeOnUnhandledKeyEventListener */ - public void addKeyFallbackListener(OnKeyFallbackListener listener) { - ArrayList<OnKeyFallbackListener> fallbacks = getListenerInfo().mKeyFallbackListeners; - if (fallbacks == null) { - fallbacks = new ArrayList<>(); - getListenerInfo().mKeyFallbackListeners = fallbacks; + public void addOnUnhandledKeyEventListener(OnUnhandledKeyEventListener listener) { + ArrayList<OnUnhandledKeyEventListener> listeners = getListenerInfo().mUnhandledKeyListeners; + if (listeners == null) { + listeners = new ArrayList<>(); + getListenerInfo().mUnhandledKeyListeners = listeners; + } + listeners.add(listener); + if (listeners.size() == 1 && mParent instanceof ViewGroup) { + ((ViewGroup) mParent).incrementChildUnhandledKeyListeners(); } - fallbacks.add(listener); } /** - * Removes a listener which will receive unhandled {@link KeyEvent}s. - * @param listener the receiver of fallback {@link KeyEvent}s. - * @see #onKeyFallback(KeyEvent) + * Removes a listener which will receive unhandled {@link KeyEvent}s. This must be called on the + * UI thread. + * + * @param listener a receiver of unhandled {@link KeyEvent}s. + * @see #addOnUnhandledKeyEventListener */ - public void removeKeyFallbackListener(OnKeyFallbackListener listener) { + public void removeOnUnhandledKeyEventListener(OnUnhandledKeyEventListener listener) { if (mListenerInfo != null) { - if (mListenerInfo.mKeyFallbackListeners != null) { - mListenerInfo.mKeyFallbackListeners.remove(listener); - if (mListenerInfo.mKeyFallbackListeners.isEmpty()) { - mListenerInfo.mKeyFallbackListeners = null; + if (mListenerInfo.mUnhandledKeyListeners != null + && !mListenerInfo.mUnhandledKeyListeners.isEmpty()) { + mListenerInfo.mUnhandledKeyListeners.remove(listener); + if (mListenerInfo.mUnhandledKeyListeners.isEmpty()) { + mListenerInfo.mUnhandledKeyListeners = null; + if (mParent instanceof ViewGroup) { + ((ViewGroup) mParent).decrementChildUnhandledKeyListeners(); + } } } } diff --git a/android/view/ViewConfiguration.java b/android/view/ViewConfiguration.java index c5a94daa..7a9de45c 100644 --- a/android/view/ViewConfiguration.java +++ b/android/view/ViewConfiguration.java @@ -303,6 +303,7 @@ public class ViewConfiguration { private final long mGlobalActionsKeyTimeout; private final float mVerticalScrollFactor; private final float mHorizontalScrollFactor; + private final boolean mShowMenuShortcutsWhenKeyboardPresent; private boolean sHasPermanentMenuKey; private boolean sHasPermanentMenuKeySet; @@ -335,6 +336,7 @@ public class ViewConfiguration { mGlobalActionsKeyTimeout = GLOBAL_ACTIONS_KEY_TIMEOUT; mHorizontalScrollFactor = HORIZONTAL_SCROLL_FACTOR; mVerticalScrollFactor = VERTICAL_SCROLL_FACTOR; + mShowMenuShortcutsWhenKeyboardPresent = false; } /** @@ -428,6 +430,10 @@ public class ViewConfiguration { com.android.internal.R.dimen.config_horizontalScrollFactor); mVerticalScrollFactor = res.getDimensionPixelSize( com.android.internal.R.dimen.config_verticalScrollFactor); + + mShowMenuShortcutsWhenKeyboardPresent = res.getBoolean( + com.android.internal.R.bool.config_showMenuShortcutsWhenKeyboardPresent); + } /** @@ -910,6 +916,15 @@ public class ViewConfiguration { } /** + * Check if shortcuts should be displayed in menus. + * + * @return {@code True} if shortcuts should be displayed in menus. + */ + public boolean shouldShowMenuShortcutsWhenKeyboardPresent() { + return mShowMenuShortcutsWhenKeyboardPresent; + } + + /** * @hide * @return Whether or not marquee should use fading edges. */ diff --git a/android/view/ViewDebug.java b/android/view/ViewDebug.java index b09934e3..276f50a5 100644 --- a/android/view/ViewDebug.java +++ b/android/view/ViewDebug.java @@ -21,7 +21,7 @@ import android.content.Context; import android.content.res.Resources; import android.graphics.Bitmap; import android.graphics.Canvas; -import android.graphics.Point; +import android.graphics.Picture; import android.graphics.Rect; import android.os.Debug; import android.os.Handler; @@ -1782,27 +1782,18 @@ public class ViewDebug { * @hide */ public static class HardwareCanvasProvider implements CanvasProvider { - - private View mView; - private Point mSize; - private RenderNode mNode; - private DisplayListCanvas mCanvas; + private Picture mPicture; @Override public Canvas getCanvas(View view, int width, int height) { - mView = view; - mSize = new Point(width, height); - mNode = RenderNode.create("ViewDebug", mView); - mNode.setLeftTopRightBottom(0, 0, width, height); - mNode.setClipToBounds(false); - mCanvas = mNode.start(width, height); - return mCanvas; + mPicture = new Picture(); + return mPicture.beginRecording(width, height); } @Override public Bitmap createBitmap() { - mNode.end(mCanvas); - return ThreadedRenderer.createHardwareBitmap(mNode, mSize.x, mSize.y); + mPicture.endRecording(); + return Bitmap.createBitmap(mPicture); } } diff --git a/android/view/ViewGroup.java b/android/view/ViewGroup.java index 4631261e..6002fe51 100644 --- a/android/view/ViewGroup.java +++ b/android/view/ViewGroup.java @@ -583,6 +583,11 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager private List<Integer> mTransientIndices = null; private List<View> mTransientViews = null; + /** + * Keeps track of how many child views have UnhandledKeyEventListeners. This should only be + * updated on the UI thread so shouldn't require explicit synchronization. + */ + int mChildUnhandledKeyListeners = 0; /** * Empty ActionMode used as a sentinel in recursive entries to startActionModeForChild. @@ -3555,13 +3560,16 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager public void dispatchProvideAutofillStructure(ViewStructure structure, @AutofillFlags int flags) { super.dispatchProvideAutofillStructure(structure, flags); + if (structure.getChildCount() != 0) { return; } if (!isLaidOut()) { - Log.v(VIEW_LOG_TAG, "dispatchProvideAutofillStructure(): not laid out, ignoring " - + mChildrenCount + " children of " + getAutofillId()); + if (Helper.sVerbose) { + Log.v(VIEW_LOG_TAG, "dispatchProvideAutofillStructure(): not laid out, ignoring " + + mChildrenCount + " children of " + getAutofillId()); + } return; } @@ -3649,34 +3657,44 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager return ViewGroup.class.getName(); } + @Override + public void notifySubtreeAccessibilityStateChanged(View child, View source, int changeType) { + // If this is a live region, we should send a subtree change event + // from this view. Otherwise, we can let it propagate up. + if (getAccessibilityLiveRegion() != ACCESSIBILITY_LIVE_REGION_NONE) { + notifyViewAccessibilityStateChangedIfNeeded( + AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE); + } else if (mParent != null) { + try { + mParent.notifySubtreeAccessibilityStateChanged(this, source, changeType); + } catch (AbstractMethodError e) { + Log.e(VIEW_LOG_TAG, mParent.getClass().getSimpleName() + + " does not fully implement ViewParent", e); + } + } + } + /** @hide */ @Override - public void notifyAccessibilitySubtreeChanged() { + public void notifySubtreeAccessibilityStateChangedIfNeeded() { if (!AccessibilityManager.getInstance(mContext).isEnabled() || mAttachInfo == null) { return; } // If something important for a11y is happening in this subtree, make sure it's dispatched // from a view that is important for a11y so it doesn't get lost. - if (getImportantForAccessibility() != IMPORTANT_FOR_ACCESSIBILITY_NO_HIDE_DESCENDANTS - && !isImportantForAccessibility() - && getChildCount() > 0) { + if ((getImportantForAccessibility() != IMPORTANT_FOR_ACCESSIBILITY_NO_HIDE_DESCENDANTS) + && !isImportantForAccessibility() && (getChildCount() > 0)) { ViewParent a11yParent = getParentForAccessibility(); if (a11yParent instanceof View) { - ((View) a11yParent).notifyAccessibilitySubtreeChanged(); + ((View) a11yParent).notifySubtreeAccessibilityStateChangedIfNeeded(); return; } } - super.notifyAccessibilitySubtreeChanged(); - } - - @Override - public void notifySubtreeAccessibilityStateChanged(View child, View source, int changeType) { - notifyAccessibilityStateChanged(source, changeType); + super.notifySubtreeAccessibilityStateChangedIfNeeded(); } - /** @hide */ @Override - public void resetSubtreeAccessibilityStateChanged() { + void resetSubtreeAccessibilityStateChanged() { super.resetSubtreeAccessibilityStateChanged(); View[] children = mChildren; final int childCount = mChildrenCount; @@ -3958,15 +3976,7 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager } /** - * Layout debugging code which draws rectangles around layout params. - * - * <p>This function is called automatically when the developer setting is enabled.<p/> - * - * <p>It is strongly advised to only call this function from debug builds as there is - * a risk of leaking unwanted layout information.<p/> - * - * @param canvas the canvas on which to draw - * @param paint the paint used to draw through + * @hide */ protected void onDebugDrawMargins(Canvas canvas, Paint paint) { for (int i = 0; i < getChildCount(); i++) { @@ -3976,19 +3986,7 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager } /** - * Layout debugging code which draws rectangles around: - * <ul> - * <li>optical bounds<li/> - * <li>margins<li/> - * <li>clip bounds<li/> - * <ul/> - * - * <p>This function is called automatically when the developer setting is enabled.<p/> - * - * <p>It is strongly advised to only call this function from debug builds as there is - * a risk of leaking unwanted layout information.<p/> - * - * @param canvas the canvas on which to draw + * @hide */ protected void onDebugDraw(Canvas canvas) { Paint paint = getDebugPaint(); @@ -4292,6 +4290,13 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager recreateChildDisplayList(child); } } + final int transientCount = mTransientViews == null ? 0 : mTransientIndices.size(); + for (int i = 0; i < transientCount; ++i) { + View child = mTransientViews.get(i); + if (((child.mViewFlags & VISIBILITY_MASK) == VISIBLE || child.getAnimation() != null)) { + recreateChildDisplayList(child); + } + } if (mOverlay != null) { View overlayView = mOverlay.getOverlayView(); recreateChildDisplayList(overlayView); @@ -5055,6 +5060,9 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager child.assignParent(this); } else { child.mParent = this; + if (child.hasUnhandledKeyListener()) { + incrementChildUnhandledKeyListeners(); + } } final boolean childHasFocus = child.hasFocus(); @@ -5088,7 +5096,7 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager } if (child.getVisibility() != View.GONE) { - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } if (mTransientIndices != null) { @@ -5110,6 +5118,20 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager // manually assembling the hierarchy, update the ancestor default-focus chain. setDefaultFocus(child); } + + touchAccessibilityNodeProviderIfNeeded(child); + } + + /** + * We may need to touch the provider to bring up the a11y layer. In a11y mode + * clients inspect the screen or the user touches it which triggers bringing up + * of the a11y infrastructure while in autofill mode we want the infra up and + * running from the beginning since we watch for a11y events to drive autofill. + */ + private void touchAccessibilityNodeProviderIfNeeded(View child) { + if (mContext.isAutofillCompatibilityEnabled()) { + child.getAccessibilityNodeProvider(); + } } private void addInArray(View child, int index) { @@ -5345,6 +5367,10 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager removeFromArray(index); + if (view.hasUnhandledKeyListener()) { + decrementChildUnhandledKeyListeners(); + } + if (view == mDefaultFocus) { clearDefaultFocus(view); } @@ -5358,7 +5384,7 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager dispatchViewRemoved(view); if (view.getVisibility() != View.GONE) { - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } int transientCount = mTransientIndices == null ? 0 : mTransientIndices.size(); @@ -6077,7 +6103,7 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager if (invalidate) { invalidateViewProperty(false, false); } - notifyAccessibilitySubtreeChanged(); + notifySubtreeAccessibilityStateChangedIfNeeded(); } @Override @@ -7523,6 +7549,62 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager } } + @Override + boolean hasUnhandledKeyListener() { + return (mChildUnhandledKeyListeners > 0) || super.hasUnhandledKeyListener(); + } + + void incrementChildUnhandledKeyListeners() { + mChildUnhandledKeyListeners += 1; + if (mChildUnhandledKeyListeners == 1) { + if (mParent instanceof ViewGroup) { + ((ViewGroup) mParent).incrementChildUnhandledKeyListeners(); + } + } + } + + void decrementChildUnhandledKeyListeners() { + mChildUnhandledKeyListeners -= 1; + if (mChildUnhandledKeyListeners == 0) { + if (mParent instanceof ViewGroup) { + ((ViewGroup) mParent).decrementChildUnhandledKeyListeners(); + } + } + } + + @Override + View dispatchUnhandledKeyEvent(KeyEvent evt) { + if (!hasUnhandledKeyListener()) { + return null; + } + ArrayList<View> orderedViews = buildOrderedChildList(); + if (orderedViews != null) { + try { + for (int i = orderedViews.size() - 1; i >= 0; --i) { + View v = orderedViews.get(i); + View consumer = v.dispatchUnhandledKeyEvent(evt); + if (consumer != null) { + return consumer; + } + } + } finally { + orderedViews.clear(); + } + } else { + for (int i = getChildCount() - 1; i >= 0; --i) { + View v = getChildAt(i); + View consumer = v.dispatchUnhandledKeyEvent(evt); + if (consumer != null) { + return consumer; + } + } + } + if (onUnhandledKeyEvent(evt)) { + return this; + } + return null; + } + /** * LayoutParams are used by views to tell their parents how they want to be * laid out. See @@ -7705,14 +7787,10 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager /** * Use {@code canvas} to draw suitable debugging annotations for these LayoutParameters. * - * <p>This function is called automatically when the developer setting is enabled.<p/> - * - * <p>It is strongly advised to only call this function from debug builds as there is - * a risk of leaking unwanted layout information.<p/> - * * @param view the view that contains these layout parameters * @param canvas the canvas on which to draw - * @param paint the paint used to draw through + * + * @hide */ public void onDebugDraw(View view, Canvas canvas, Paint paint) { } @@ -8216,6 +8294,9 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager return ((mMarginFlags & LAYOUT_DIRECTION_MASK) == View.LAYOUT_DIRECTION_RTL); } + /** + * @hide + */ @Override public void onDebugDraw(View view, Canvas canvas, Paint paint) { Insets oi = isLayoutModeOptical(view.mParent) ? view.getOpticalInsets() : Insets.NONE; diff --git a/android/view/ViewGroup_Delegate.java b/android/view/ViewGroup_Delegate.java index 6daae200..10a4f317 100644 --- a/android/view/ViewGroup_Delegate.java +++ b/android/view/ViewGroup_Delegate.java @@ -79,12 +79,13 @@ public class ViewGroup_Delegate { } Bitmap bitmap = Bitmap_Delegate.createBitmap(shadow, false, Density.getEnum(canvas.getDensity())); + canvas.save(); Rect clipBounds = canvas.getClipBounds(); Rect newBounds = new Rect(clipBounds); newBounds.inset((int)-elevation, (int)-elevation); canvas.clipRect(newBounds, Op.REPLACE); canvas.drawBitmap(bitmap, 0, 0, null); - canvas.clipRect(clipBounds, Op.REPLACE); + canvas.restore(); } private static float getElevation(View child, ViewGroup parent) { @@ -145,11 +146,11 @@ public class ViewGroup_Delegate { canvas.concat(transformToApply.getMatrix()); canvas.translate(transX, transY); } - if (!childHasIdentityMatrix) { - canvas.translate(-transX, -transY); - canvas.concat(child.getMatrix()); - canvas.translate(transX, transY); - } + } + if (!childHasIdentityMatrix) { + canvas.translate(-transX, -transY); + canvas.concat(child.getMatrix()); + canvas.translate(transX, transY); } } diff --git a/android/view/ViewRootImpl.java b/android/view/ViewRootImpl.java index 30f584c5..433c90b3 100644 --- a/android/view/ViewRootImpl.java +++ b/android/view/ViewRootImpl.java @@ -29,6 +29,7 @@ import static android.view.WindowManager.LayoutParams.TYPE_VOLUME_OVERLAY; import android.Manifest; import android.animation.LayoutTransition; import android.annotation.NonNull; +import android.annotation.Nullable; import android.app.ActivityManager; import android.app.ActivityThread; import android.app.ResourcesManager; @@ -71,6 +72,7 @@ import android.os.Trace; import android.util.AndroidRuntimeException; import android.util.DisplayMetrics; import android.util.Log; +import android.util.LongArray; import android.util.MergedConfiguration; import android.util.Slog; import android.util.SparseArray; @@ -89,13 +91,12 @@ import android.view.accessibility.AccessibilityManager.HighTextContrastChangeLis import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; import android.view.accessibility.AccessibilityNodeProvider; -import android.view.accessibility.AccessibilityViewHierarchyState; import android.view.accessibility.AccessibilityWindowInfo; import android.view.accessibility.IAccessibilityInteractionConnection; import android.view.accessibility.IAccessibilityInteractionConnectionCallback; -import android.view.accessibility.ThrottlingAccessibilityEventSender; import android.view.animation.AccelerateDecelerateInterpolator; import android.view.animation.Interpolator; +import android.view.autofill.AutofillManager; import android.view.inputmethod.InputMethodManager; import android.widget.Scroller; @@ -115,6 +116,9 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.lang.ref.WeakReference; import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Queue; import java.util.concurrent.CountDownLatch; /** @@ -376,7 +380,7 @@ public final class ViewRootImpl implements ViewParent, InputStage mFirstPostImeInputStage; InputStage mSyntheticInputStage; - private final KeyFallbackManager mKeyFallbackManager = new KeyFallbackManager(); + private final UnhandledKeyManager mUnhandledKeyManager = new UnhandledKeyManager(); boolean mWindowAttributesChanged = false; int mWindowAttributesChangesFlag = 0; @@ -461,6 +465,10 @@ public final class ViewRootImpl implements ViewParent, new AccessibilityInteractionConnectionManager(); final HighContrastTextManager mHighContrastTextManager; + SendWindowContentChangedAccessibilityEvent mSendWindowContentChangedAccessibilityEvent; + + HashSet<View> mTempHashSet; + private final int mDensity; private final int mNoncompatDensity; @@ -475,8 +483,6 @@ public final class ViewRootImpl implements ViewParent, private boolean mNeedsRendererSetup; - protected AccessibilityViewHierarchyState mAccessibilityState; - /** * Consistency verifier for debugging purposes. */ @@ -748,7 +754,7 @@ public final class ViewRootImpl implements ViewParent, mAttachInfo.mRecomputeGlobalAttributes = true; collectViewAttributes(); res = mWindowSession.addToDisplay(mWindow, mSeq, mWindowAttributes, - getHostVisibility(), mDisplay.getDisplayId(), + getHostVisibility(), mDisplay.getDisplayId(), mWinFrame, mAttachInfo.mContentInsets, mAttachInfo.mStableInsets, mAttachInfo.mOutsets, mAttachInfo.mDisplayCutout, mInputChannel); } catch (RemoteException e) { @@ -895,6 +901,26 @@ public final class ViewRootImpl implements ViewParent, return mWindowAttributes.getTitle(); } + /** + * @return the width of the root view. Note that this will return {@code -1} until the first + * layout traversal, when the width is set. + * + * @hide + */ + public int getWidth() { + return mWidth; + } + + /** + * @return the height of the root view. Note that this will return {@code -1} until the first + * layout traversal, when the height is set. + * + * @hide + */ + public int getHeight() { + return mHeight; + } + void destroyHardwareResources() { if (mAttachInfo.mThreadedRenderer != null) { mAttachInfo.mThreadedRenderer.destroyHardwareResources(mView); @@ -1686,8 +1712,8 @@ public final class ViewRootImpl implements ViewParent, desiredWindowWidth = size.x; desiredWindowHeight = size.y; } else { - desiredWindowWidth = dipToPx(config.screenWidthDp); - desiredWindowHeight = dipToPx(config.screenHeightDp); + desiredWindowWidth = mWinFrame.width(); + desiredWindowHeight = mWinFrame.height(); } // We used to use the following condition to choose 32 bits drawing caches: @@ -1974,6 +2000,7 @@ public final class ViewRootImpl implements ViewParent, final boolean outsetsChanged = !mPendingOutsets.equals(mAttachInfo.mOutsets); final boolean surfaceSizeChanged = (relayoutResult & WindowManagerGlobal.RELAYOUT_RES_SURFACE_RESIZED) != 0; + surfaceChanged |= surfaceSizeChanged; final boolean alwaysConsumeNavBarChanged = mPendingAlwaysConsumeNavBar != mAttachInfo.mAlwaysConsumeNavBar; if (contentInsetsChanged) { @@ -2547,6 +2574,10 @@ public final class ViewRootImpl implements ViewParent, ~WindowManager.LayoutParams .SOFT_INPUT_IS_FORWARD_NAVIGATION; mHasHadWindowFocus = true; + + // Refocusing a window that has a focused view should fire a + // focus event for the view since the global focused view changed. + fireAccessibilityFocusEventIfHasFocusedNode(); } else { if (mPointerCapture) { handlePointerCaptureChanged(false); @@ -2556,6 +2587,86 @@ public final class ViewRootImpl implements ViewParent, mFirstInputStage.onWindowFocusChanged(hasWindowFocus); } + private void fireAccessibilityFocusEventIfHasFocusedNode() { + if (!AccessibilityManager.getInstance(mContext).isEnabled()) { + return; + } + final View focusedView = mView.findFocus(); + if (focusedView == null) { + return; + } + final AccessibilityNodeProvider provider = focusedView.getAccessibilityNodeProvider(); + if (provider == null) { + focusedView.sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_FOCUSED); + } else { + final AccessibilityNodeInfo focusedNode = findFocusedVirtualNode(provider); + if (focusedNode != null) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId( + focusedNode.getSourceNodeId()); + // This is a best effort since clearing and setting the focus via the + // provider APIs could have side effects. We don't have a provider API + // similar to that on View to ask a given event to be fired. + final AccessibilityEvent event = AccessibilityEvent.obtain( + AccessibilityEvent.TYPE_VIEW_FOCUSED); + event.setSource(focusedView, virtualId); + event.setPackageName(focusedNode.getPackageName()); + event.setChecked(focusedNode.isChecked()); + event.setContentDescription(focusedNode.getContentDescription()); + event.setPassword(focusedNode.isPassword()); + event.getText().add(focusedNode.getText()); + event.setEnabled(focusedNode.isEnabled()); + focusedView.getParent().requestSendAccessibilityEvent(focusedView, event); + focusedNode.recycle(); + } + } + } + + private AccessibilityNodeInfo findFocusedVirtualNode(AccessibilityNodeProvider provider) { + AccessibilityNodeInfo focusedNode = provider.findFocus( + AccessibilityNodeInfo.FOCUS_INPUT); + if (focusedNode != null) { + return focusedNode; + } + + if (!mContext.isAutofillCompatibilityEnabled()) { + return null; + } + + // Unfortunately some provider implementations don't properly + // implement AccessibilityNodeProvider#findFocus + AccessibilityNodeInfo current = provider.createAccessibilityNodeInfo( + AccessibilityNodeProvider.HOST_VIEW_ID); + if (current.isFocused()) { + return current; + } + + final Queue<AccessibilityNodeInfo> fringe = new LinkedList<>(); + fringe.offer(current); + + while (!fringe.isEmpty()) { + current = fringe.poll(); + final LongArray childNodeIds = current.getChildNodeIds(); + if (childNodeIds== null || childNodeIds.size() <= 0) { + continue; + } + final int childCount = childNodeIds.size(); + for (int i = 0; i < childCount; i++) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId( + childNodeIds.get(i)); + final AccessibilityNodeInfo child = provider.createAccessibilityNodeInfo(virtualId); + if (child != null) { + if (child.isFocused()) { + return child; + } + fringe.offer(child); + } + } + current.recycle(); + } + + return null; + } + private void handleOutOfResourcesException(Surface.OutOfResourcesException e) { Log.e(mTag, "OutOfResourcesException initializing HW surface", e); try { @@ -3816,6 +3927,7 @@ public final class ViewRootImpl implements ViewParent, private final static int MSG_DISPATCH_APP_VISIBILITY = 8; private final static int MSG_DISPATCH_GET_NEW_SURFACE = 9; private final static int MSG_DISPATCH_KEY_FROM_IME = 11; + private final static int MSG_DISPATCH_KEY_FROM_AUTOFILL = 12; private final static int MSG_CHECK_FOCUS = 13; private final static int MSG_CLOSE_SYSTEM_DIALOGS = 14; private final static int MSG_DISPATCH_DRAG_EVENT = 15; @@ -3857,6 +3969,8 @@ public final class ViewRootImpl implements ViewParent, return "MSG_DISPATCH_GET_NEW_SURFACE"; case MSG_DISPATCH_KEY_FROM_IME: return "MSG_DISPATCH_KEY_FROM_IME"; + case MSG_DISPATCH_KEY_FROM_AUTOFILL: + return "MSG_DISPATCH_KEY_FROM_AUTOFILL"; case MSG_CHECK_FOCUS: return "MSG_CHECK_FOCUS"; case MSG_CLOSE_SYSTEM_DIALOGS: @@ -4034,6 +4148,13 @@ public final class ViewRootImpl implements ViewParent, } enqueueInputEvent(event, null, QueuedInputEvent.FLAG_DELIVER_POST_IME, true); } break; + case MSG_DISPATCH_KEY_FROM_AUTOFILL: { + if (LOCAL_LOGV) { + Log.v(TAG, "Dispatching key " + msg.obj + " from Autofill to " + mView); + } + KeyEvent event = (KeyEvent) msg.obj; + enqueueInputEvent(event, null, 0, true); + } break; case MSG_CHECK_FOCUS: { InputMethodManager imm = InputMethodManager.peekInstance(); if (imm != null) { @@ -4324,7 +4445,8 @@ public final class ViewRootImpl implements ViewParent, Slog.w(mTag, "Dropping event due to root view being removed: " + q.mEvent); return true; } else if ((!mAttachInfo.mHasWindowFocus - && !q.mEvent.isFromSource(InputDevice.SOURCE_CLASS_POINTER)) || mStopped + && !q.mEvent.isFromSource(InputDevice.SOURCE_CLASS_POINTER) + && !isAutofillUiShowing()) || mStopped || (mIsAmbientMode && !q.mEvent.isFromSource(InputDevice.SOURCE_CLASS_BUTTON)) || (mPausedForTransition && !isBack(q.mEvent))) { // This is a focus event and the window doesn't currently have input focus or @@ -4659,6 +4781,14 @@ public final class ViewRootImpl implements ViewParent, ensureTouchMode(event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)); } + if (action == MotionEvent.ACTION_DOWN) { + // Upon motion event within app window, close autofill ui. + AutofillManager afm = getAutofillManager(); + if (afm != null) { + afm.requestHideFillUi(); + } + } + if (action == MotionEvent.ACTION_DOWN && mAttachInfo.mTooltipHost != null) { mAttachInfo.mTooltipHost.hideTooltip(); } @@ -4845,10 +4975,10 @@ public final class ViewRootImpl implements ViewParent, private int processKeyEvent(QueuedInputEvent q) { final KeyEvent event = (KeyEvent)q.mEvent; - mKeyFallbackManager.mDispatched = false; + mUnhandledKeyManager.mDispatched = false; - if (mKeyFallbackManager.hasFocus() - && mKeyFallbackManager.dispatchUnique(mView, event)) { + if (mUnhandledKeyManager.hasFocus() + && mUnhandledKeyManager.dispatchUnique(mView, event)) { return FINISH_HANDLED; } @@ -4861,7 +4991,7 @@ public final class ViewRootImpl implements ViewParent, return FINISH_NOT_HANDLED; } - if (mKeyFallbackManager.dispatchUnique(mView, event)) { + if (mUnhandledKeyManager.dispatchUnique(mView, event)) { return FINISH_HANDLED; } @@ -6297,6 +6427,28 @@ public final class ViewRootImpl implements ViewParent, return mAudioManager; } + private @Nullable AutofillManager getAutofillManager() { + if (mView instanceof ViewGroup) { + ViewGroup decorView = (ViewGroup) mView; + if (decorView.getChildCount() > 0) { + // We cannot use decorView's Context for querying AutofillManager: DecorView's + // context is based on Application Context, it would allocate a different + // AutofillManager instance. + return decorView.getChildAt(0).getContext() + .getSystemService(AutofillManager.class); + } + } + return null; + } + + private boolean isAutofillUiShowing() { + AutofillManager afm = getAutofillManager(); + if (afm == null) { + return false; + } + return afm.isAutofillUiShowing(); + } + public AccessibilityInteractionController getAccessibilityInteractionController() { if (mView == null) { throw new IllegalStateException("getAccessibilityInteractionController" @@ -6318,18 +6470,24 @@ public final class ViewRootImpl implements ViewParent, params.backup(); mTranslator.translateWindowLayout(params); } + if (params != null) { if (DBG) Log.d(mTag, "WindowLayout in layoutWindow:" + params); - } - if (params != null && mOrigWindowType != params.type) { - // For compatibility with old apps, don't crash here. - if (mTargetSdkVersion < Build.VERSION_CODES.ICE_CREAM_SANDWICH) { - Slog.w(mTag, "Window type can not be changed after " - + "the window is added; ignoring change of " + mView); - params.type = mOrigWindowType; + if (mOrigWindowType != params.type) { + // For compatibility with old apps, don't crash here. + if (mTargetSdkVersion < Build.VERSION_CODES.ICE_CREAM_SANDWICH) { + Slog.w(mTag, "Window type can not be changed after " + + "the window is added; ignoring change of " + mView); + params.type = mOrigWindowType; + } + } + + if (mSurface.isValid()) { + params.frameNumber = mSurface.getNextFrameNumber(); } } + int relayoutResult = mWindowSession.relayout( mWindow, mSeq, params, (int) (mView.getMeasuredWidth() * appScale + 0.5f), @@ -7153,6 +7311,12 @@ public final class ViewRootImpl implements ViewParent, mHandler.sendMessage(msg); } + public void dispatchKeyFromAutofill(KeyEvent event) { + Message msg = mHandler.obtainMessage(MSG_DISPATCH_KEY_FROM_AUTOFILL, event); + msg.setAsynchronous(true); + mHandler.sendMessage(msg); + } + /** * Reinject unhandled {@link InputEvent}s in order to synthesize fallbacks events. * @@ -7258,9 +7422,11 @@ public final class ViewRootImpl implements ViewParent, * {@link ViewConfiguration#getSendRecurringAccessibilityEventsInterval()}. */ private void postSendWindowContentChangedCallback(View source, int changeType) { - getAccessibilityState() - .getSendWindowContentChangedAccessibilityEvent() - .runOrPost(source, changeType); + if (mSendWindowContentChangedAccessibilityEvent == null) { + mSendWindowContentChangedAccessibilityEvent = + new SendWindowContentChangedAccessibilityEvent(); + } + mSendWindowContentChangedAccessibilityEvent.runOrPost(source, changeType); } /** @@ -7268,18 +7434,9 @@ public final class ViewRootImpl implements ViewParent, * {@link AccessibilityEvent#TYPE_WINDOW_CONTENT_CHANGED} event. */ private void removeSendWindowContentChangedCallback() { - if (mAccessibilityState != null - && mAccessibilityState.isWindowContentChangedEventSenderInitialized()) { - ThrottlingAccessibilityEventSender.cancelIfPending( - mAccessibilityState.getSendWindowContentChangedAccessibilityEvent()); - } - } - - AccessibilityViewHierarchyState getAccessibilityState() { - if (mAccessibilityState == null) { - mAccessibilityState = new AccessibilityViewHierarchyState(); + if (mSendWindowContentChangedAccessibilityEvent != null) { + mHandler.removeCallbacks(mSendWindowContentChangedAccessibilityEvent); } - return mAccessibilityState; } @Override @@ -7317,8 +7474,12 @@ public final class ViewRootImpl implements ViewParent, return false; } - // Send any pending event to prevent reordering - flushPendingAccessibilityEvents(); + // Immediately flush pending content changed event (if any) to preserve event order + if (event.getEventType() != AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED + && mSendWindowContentChangedAccessibilityEvent != null + && mSendWindowContentChangedAccessibilityEvent.mSource != null) { + mSendWindowContentChangedAccessibilityEvent.removeCallbacksAndRun(); + } // Intercept accessibility focus events fired by virtual nodes to keep // track of accessibility focus position in such nodes. @@ -7362,19 +7523,6 @@ public final class ViewRootImpl implements ViewParent, return true; } - /** @hide */ - public void flushPendingAccessibilityEvents() { - if (mAccessibilityState != null) { - if (mAccessibilityState.isScrollEventSenderInitialized()) { - mAccessibilityState.getSendViewScrolledAccessibilityEvent().sendNowIfPending(); - } - if (mAccessibilityState.isWindowContentChangedEventSenderInitialized()) { - mAccessibilityState.getSendWindowContentChangedAccessibilityEvent() - .sendNowIfPending(); - } - } - } - /** * Updates the focused virtual view, when necessary, in response to a * content changed event. @@ -7509,6 +7657,39 @@ public final class ViewRootImpl implements ViewParent, return View.TEXT_ALIGNMENT_RESOLVED_DEFAULT; } + private View getCommonPredecessor(View first, View second) { + if (mTempHashSet == null) { + mTempHashSet = new HashSet<View>(); + } + HashSet<View> seen = mTempHashSet; + seen.clear(); + View firstCurrent = first; + while (firstCurrent != null) { + seen.add(firstCurrent); + ViewParent firstCurrentParent = firstCurrent.mParent; + if (firstCurrentParent instanceof View) { + firstCurrent = (View) firstCurrentParent; + } else { + firstCurrent = null; + } + } + View secondCurrent = second; + while (secondCurrent != null) { + if (seen.contains(secondCurrent)) { + seen.clear(); + return secondCurrent; + } + ViewParent secondCurrentParent = secondCurrent.mParent; + if (secondCurrentParent instanceof View) { + secondCurrent = (View) secondCurrentParent; + } else { + secondCurrent = null; + } + } + seen.clear(); + return null; + } + void checkThread() { if (mThread != Thread.currentThread()) { throw new CalledFromWrongThreadException( @@ -7617,7 +7798,7 @@ public final class ViewRootImpl implements ViewParent, * @return {@code true} if the event was handled, {@code false} otherwise. */ public boolean dispatchKeyFallbackEvent(KeyEvent event) { - return mKeyFallbackManager.dispatch(mView, event); + return mUnhandledKeyManager.dispatch(mView, event); } class TakenSurfaceHolder extends BaseSurfaceHolder { @@ -8119,18 +8300,91 @@ public final class ViewRootImpl implements ViewParent, } } - private static class KeyFallbackManager { + private class SendWindowContentChangedAccessibilityEvent implements Runnable { + private int mChangeTypes = 0; + + public View mSource; + public long mLastEventTimeMillis; + + @Override + public void run() { + // Protect against re-entrant code and attempt to do the right thing in the case that + // we're multithreaded. + View source = mSource; + mSource = null; + if (source == null) { + Log.e(TAG, "Accessibility content change has no source"); + return; + } + // The accessibility may be turned off while we were waiting so check again. + if (AccessibilityManager.getInstance(mContext).isEnabled()) { + mLastEventTimeMillis = SystemClock.uptimeMillis(); + AccessibilityEvent event = AccessibilityEvent.obtain(); + event.setEventType(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); + event.setContentChangeTypes(mChangeTypes); + source.sendAccessibilityEventUnchecked(event); + } else { + mLastEventTimeMillis = 0; + } + // In any case reset to initial state. + source.resetSubtreeAccessibilityStateChanged(); + mChangeTypes = 0; + } + + public void runOrPost(View source, int changeType) { + if (mHandler.getLooper() != Looper.myLooper()) { + CalledFromWrongThreadException e = new CalledFromWrongThreadException("Only the " + + "original thread that created a view hierarchy can touch its views."); + // TODO: Throw the exception + Log.e(TAG, "Accessibility content change on non-UI thread. Future Android " + + "versions will throw an exception.", e); + // Attempt to recover. This code does not eliminate the thread safety issue, but + // it should force any issues to happen near the above log. + mHandler.removeCallbacks(this); + if (mSource != null) { + // Dispatch whatever was pending. It's still possible that the runnable started + // just before we removed the callbacks, and bad things will happen, but at + // least they should happen very close to the logged error. + run(); + } + } + if (mSource != null) { + // If there is no common predecessor, then mSource points to + // a removed view, hence in this case always prefer the source. + View predecessor = getCommonPredecessor(mSource, source); + mSource = (predecessor != null) ? predecessor : source; + mChangeTypes |= changeType; + return; + } + mSource = source; + mChangeTypes = changeType; + final long timeSinceLastMillis = SystemClock.uptimeMillis() - mLastEventTimeMillis; + final long minEventIntevalMillis = + ViewConfiguration.getSendRecurringAccessibilityEventsInterval(); + if (timeSinceLastMillis >= minEventIntevalMillis) { + removeCallbacksAndRun(); + } else { + mHandler.postDelayed(this, minEventIntevalMillis - timeSinceLastMillis); + } + } + + public void removeCallbacksAndRun() { + mHandler.removeCallbacks(this); + run(); + } + } + + private static class UnhandledKeyManager { - // This is used to ensure that key-fallback events are only dispatched once. We attempt + // This is used to ensure that unhandled events are only dispatched once. We attempt // to dispatch more than once in order to achieve a certain order. Specifically, if we - // are in an Activity or Dialog (and have a Window.Callback), the keyfallback events should + // are in an Activity or Dialog (and have a Window.Callback), the unhandled events should // be dispatched after the view hierarchy, but before the Activity. However, if we aren't - // in an activity, we still want key fallbacks to be dispatched. + // in an activity, we still want unhandled keys to be dispatched. boolean mDispatched = false; SparseBooleanArray mCapturedKeys = new SparseBooleanArray(); - WeakReference<View> mFallbackReceiver = null; - int mVisitCount = 0; + WeakReference<View> mCurrentReceiver = null; private void updateCaptureState(KeyEvent event) { if (event.getAction() == KeyEvent.ACTION_DOWN) { @@ -8147,56 +8401,28 @@ public final class ViewRootImpl implements ViewParent, updateCaptureState(event); - if (mFallbackReceiver != null) { - View target = mFallbackReceiver.get(); + if (mCurrentReceiver != null) { + View target = mCurrentReceiver.get(); if (mCapturedKeys.size() == 0) { - mFallbackReceiver = null; + mCurrentReceiver = null; } if (target != null && target.isAttachedToWindow()) { - return target.onKeyFallback(event); + target.onUnhandledKeyEvent(event); } // consume anyways so that we don't feed uncaptured key events to other views return true; } - boolean result = dispatchInZOrder(root, event); - Trace.traceEnd(Trace.TRACE_TAG_VIEW); - return result; - } - - private boolean dispatchInZOrder(View view, KeyEvent evt) { - if (view instanceof ViewGroup) { - ViewGroup vg = (ViewGroup) view; - ArrayList<View> orderedViews = vg.buildOrderedChildList(); - if (orderedViews != null) { - try { - for (int i = orderedViews.size() - 1; i >= 0; --i) { - View v = orderedViews.get(i); - if (dispatchInZOrder(v, evt)) { - return true; - } - } - } finally { - orderedViews.clear(); - } - } else { - for (int i = vg.getChildCount() - 1; i >= 0; --i) { - View v = vg.getChildAt(i); - if (dispatchInZOrder(v, evt)) { - return true; - } - } - } + View consumer = root.dispatchUnhandledKeyEvent(event); + if (consumer != null) { + mCurrentReceiver = new WeakReference<>(consumer); } - if (view.onKeyFallback(evt)) { - mFallbackReceiver = new WeakReference<>(view); - return true; - } - return false; + Trace.traceEnd(Trace.TRACE_TAG_VIEW); + return consumer != null; } boolean hasFocus() { - return mFallbackReceiver != null; + return mCurrentReceiver != null; } boolean dispatchUnique(View root, KeyEvent event) { diff --git a/android/view/ViewStructure.java b/android/view/ViewStructure.java index 1d94abeb..3f7ab2ae 100644 --- a/android/view/ViewStructure.java +++ b/android/view/ViewStructure.java @@ -23,6 +23,8 @@ import android.graphics.Rect; import android.os.Bundle; import android.os.LocaleList; import android.util.Pair; +import android.view.View.AutofillImportance; +import android.view.ViewStructure.HtmlInfo; import android.view.autofill.AutofillId; import android.view.autofill.AutofillValue; @@ -347,6 +349,12 @@ public abstract class ViewStructure { public abstract void setAutofillOptions(CharSequence[] options); /** + * Sets the {@link View#setImportantForAutofill(int) importantForAutofill mode} of the + * view associated with this node. + */ + public void setImportantForAutofill(@AutofillImportance int mode) {} + + /** * Sets the {@link android.text.InputType} bits of this node. * * @param inputType inputType bits as defined by {@link android.text.InputType}. diff --git a/android/view/Window.java b/android/view/Window.java index 5bd0782d..93b3fc25 100644 --- a/android/view/Window.java +++ b/android/view/Window.java @@ -25,7 +25,6 @@ import android.annotation.LayoutRes; import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.StyleRes; -import android.annotation.SystemApi; import android.app.WindowConfiguration; import android.content.Context; import android.content.pm.ActivityInfo; @@ -1255,14 +1254,6 @@ public abstract class Window { } /** @hide */ - @SystemApi - public void setDisableWallpaperTouchEvents(boolean disable) { - setPrivateFlags(disable - ? WindowManager.LayoutParams.PRIVATE_FLAG_DISABLE_WALLPAPER_TOUCH_EVENTS : 0, - WindowManager.LayoutParams.PRIVATE_FLAG_DISABLE_WALLPAPER_TOUCH_EVENTS); - } - - /** @hide */ public abstract void alwaysReadCloseOnTouchAttr(); /** @hide */ diff --git a/android/view/WindowId.java b/android/view/WindowId.java index c4cda2c7..12e58f14 100644 --- a/android/view/WindowId.java +++ b/android/view/WindowId.java @@ -16,6 +16,8 @@ package android.view; +import android.annotation.NonNull; +import android.annotation.Nullable; import android.os.Handler; import android.os.IBinder; import android.os.Message; @@ -35,6 +37,7 @@ import java.util.HashMap; * that doesn't allow the other process to negatively harm your window. */ public class WindowId implements Parcelable { + @NonNull private final IWindowId mToken; /** @@ -74,8 +77,7 @@ public class WindowId implements Parcelable { } }; - final HashMap<IBinder, WindowId> mRegistrations - = new HashMap<IBinder, WindowId>(); + final HashMap<IBinder, WindowId> mRegistrations = new HashMap<>(); class H extends Handler { @Override @@ -163,10 +165,9 @@ public class WindowId implements Parcelable { * same package. */ @Override - public boolean equals(Object otherObj) { + public boolean equals(@Nullable Object otherObj) { if (otherObj instanceof WindowId) { - return mToken.asBinder().equals(((WindowId) otherObj) - .mToken.asBinder()); + return mToken.asBinder().equals(((WindowId) otherObj).mToken.asBinder()); } return false; } @@ -182,7 +183,7 @@ public class WindowId implements Parcelable { sb.append("IntentSender{"); sb.append(Integer.toHexString(System.identityHashCode(this))); sb.append(": "); - sb.append(mToken != null ? mToken.asBinder() : null); + sb.append(mToken.asBinder()); sb.append('}'); return sb.toString(); } @@ -195,30 +196,32 @@ public class WindowId implements Parcelable { out.writeStrongBinder(mToken.asBinder()); } - public static final Parcelable.Creator<WindowId> CREATOR - = new Parcelable.Creator<WindowId>() { + public static final Parcelable.Creator<WindowId> CREATOR = new Parcelable.Creator<WindowId>() { + @Override public WindowId createFromParcel(Parcel in) { IBinder target = in.readStrongBinder(); return target != null ? new WindowId(target) : null; } + @Override public WindowId[] newArray(int size) { return new WindowId[size]; } }; /** @hide */ + @NonNull public IWindowId getTarget() { return mToken; } /** @hide */ - public WindowId(IWindowId target) { + public WindowId(@NonNull IWindowId target) { mToken = target; } /** @hide */ - public WindowId(IBinder target) { + public WindowId(@NonNull IBinder target) { mToken = IWindowId.Stub.asInterface(target); } } diff --git a/android/view/WindowInfo.java b/android/view/WindowInfo.java index bb9e391d..7bae28a4 100644 --- a/android/view/WindowInfo.java +++ b/android/view/WindowInfo.java @@ -21,6 +21,7 @@ import android.os.IBinder; import android.os.Parcel; import android.os.Parcelable; import android.util.Pools; +import android.view.accessibility.AccessibilityNodeInfo; import java.util.ArrayList; import java.util.List; @@ -46,7 +47,7 @@ public class WindowInfo implements Parcelable { public final Rect boundsInScreen = new Rect(); public List<IBinder> childTokens; public CharSequence title; - public int accessibilityIdOfAnchor = View.NO_ID; + public long accessibilityIdOfAnchor = AccessibilityNodeInfo.UNDEFINED_NODE_ID; public boolean inPictureInPicture; private WindowInfo() { @@ -105,7 +106,7 @@ public class WindowInfo implements Parcelable { parcel.writeInt(focused ? 1 : 0); boundsInScreen.writeToParcel(parcel, flags); parcel.writeCharSequence(title); - parcel.writeInt(accessibilityIdOfAnchor); + parcel.writeLong(accessibilityIdOfAnchor); parcel.writeInt(inPictureInPicture ? 1 : 0); if (childTokens != null && !childTokens.isEmpty()) { @@ -142,7 +143,7 @@ public class WindowInfo implements Parcelable { focused = (parcel.readInt() == 1); boundsInScreen.readFromParcel(parcel); title = parcel.readCharSequence(); - accessibilityIdOfAnchor = parcel.readInt(); + accessibilityIdOfAnchor = parcel.readLong(); inPictureInPicture = (parcel.readInt() == 1); final boolean hasChildren = (parcel.readInt() == 1); diff --git a/android/view/WindowManager.java b/android/view/WindowManager.java index 1c5e8719..f6181d70 100644 --- a/android/view/WindowManager.java +++ b/android/view/WindowManager.java @@ -63,6 +63,7 @@ import android.os.Parcelable; import android.text.TextUtils; import android.util.Log; import android.util.proto.ProtoOutputStream; +import android.view.accessibility.AccessibilityNodeInfo; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -235,6 +236,18 @@ public interface WindowManager extends ViewManager { int TRANSIT_KEYGUARD_UNOCCLUDE = 23; /** + * A translucent activity is being opened. + * @hide + */ + int TRANSIT_TRANSLUCENT_ACTIVITY_OPEN = 24; + + /** + * A translucent activity is being closed. + * @hide + */ + int TRANSIT_TRANSLUCENT_ACTIVITY_CLOSE = 25; + + /** * @hide */ @IntDef(prefix = { "TRANSIT_" }, value = { @@ -257,7 +270,9 @@ public interface WindowManager extends ViewManager { TRANSIT_KEYGUARD_GOING_AWAY, TRANSIT_KEYGUARD_GOING_AWAY_ON_WALLPAPER, TRANSIT_KEYGUARD_OCCLUDE, - TRANSIT_KEYGUARD_UNOCCLUDE + TRANSIT_KEYGUARD_UNOCCLUDE, + TRANSIT_TRANSLUCENT_ACTIVITY_OPEN, + TRANSIT_TRANSLUCENT_ACTIVITY_CLOSE }) @Retention(RetentionPolicy.SOURCE) @interface TransitionType {} @@ -1832,7 +1847,9 @@ public interface WindowManager extends ViewManager { public static final int SOFT_INPUT_MASK_STATE = 0x0f; /** - * Visibility state for {@link #softInputMode}: no state has been specified. + * Visibility state for {@link #softInputMode}: no state has been specified. The system may + * show or hide the software keyboard for better user experience when the window gains + * focus. */ public static final int SOFT_INPUT_STATE_UNSPECIFIED = 0; @@ -2219,7 +2236,7 @@ public interface WindowManager extends ViewManager { @IntDef( flag = true, value = {LAYOUT_IN_DISPLAY_CUTOUT_MODE_DEFAULT, - LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS, + LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES, LAYOUT_IN_DISPLAY_CUTOUT_MODE_NEVER}) @interface LayoutInDisplayCutoutMode {} @@ -2230,56 +2247,107 @@ public interface WindowManager extends ViewManager { * Defaults to {@link #LAYOUT_IN_DISPLAY_CUTOUT_MODE_DEFAULT}. * * @see #LAYOUT_IN_DISPLAY_CUTOUT_MODE_DEFAULT - * @see #LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS + * @see #LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES * @see #LAYOUT_IN_DISPLAY_CUTOUT_MODE_NEVER * @see DisplayCutout + * @see android.R.attr#windowLayoutInDisplayCutoutMode + * android:windowLayoutInDisplayCutoutMode */ @LayoutInDisplayCutoutMode public int layoutInDisplayCutoutMode = LAYOUT_IN_DISPLAY_CUTOUT_MODE_DEFAULT; /** * The window is allowed to extend into the {@link DisplayCutout} area, only if the - * {@link DisplayCutout} is fully contained within the status bar. Otherwise, the window is + * {@link DisplayCutout} is fully contained within a system bar. Otherwise, the window is * laid out such that it does not overlap with the {@link DisplayCutout} area. * * <p> - * In practice, this means that if the window did not set FLAG_FULLSCREEN or - * SYSTEM_UI_FLAG_FULLSCREEN, it can extend into the cutout area in portrait. - * Otherwise (i.e. fullscreen or landscape) it is laid out such that it does overlap the + * In practice, this means that if the window did not set {@link #FLAG_FULLSCREEN} or + * {@link View#SYSTEM_UI_FLAG_FULLSCREEN}, it can extend into the cutout area in portrait + * if the cutout is at the top edge. Similarly for + * {@link View#SYSTEM_UI_FLAG_HIDE_NAVIGATION} and a cutout at the bottom of the screen. + * Otherwise (i.e. fullscreen or landscape) it is laid out such that it does not overlap the * cutout area. * * <p> - * The usual precautions for not overlapping with the status bar are sufficient for ensuring - * that no important content overlaps with the DisplayCutout. + * The usual precautions for not overlapping with the status and navigation bar are + * sufficient for ensuring that no important content overlaps with the DisplayCutout. * * @see DisplayCutout * @see WindowInsets + * @see #layoutInDisplayCutoutMode + * @see android.R.attr#windowLayoutInDisplayCutoutMode + * android:windowLayoutInDisplayCutoutMode */ public static final int LAYOUT_IN_DISPLAY_CUTOUT_MODE_DEFAULT = 0; /** - * The window is always allowed to extend into the {@link DisplayCutout} area, - * even if fullscreen or in landscape. + * @deprecated use {@link #LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES} + * @hide + */ + @Deprecated + public static final int LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS = 1; + + /** + * The window is always allowed to extend into the {@link DisplayCutout} areas on the short + * edges of the screen. + * + * The window will never extend into a {@link DisplayCutout} area on the long edges of the + * screen. * * <p> * The window must make sure that no important content overlaps with the * {@link DisplayCutout}. * + * <p> + * In this mode, the window extends under cutouts on the short edge of the display in both + * portrait and landscape, regardless of whether the window is hiding the system bars:<br/> + * <img src="{@docRoot}reference/android/images/display_cutout/short_edge/fullscreen_top_no_letterbox.png" + * height="720" + * alt="Screenshot of a fullscreen activity on a display with a cutout at the top edge in + * portrait, no letterbox is applied."/> + * + * <img src="{@docRoot}reference/android/images/display_cutout/short_edge/landscape_top_no_letterbox.png" + * width="720" + * alt="Screenshot of an activity on a display with a cutout at the top edge in landscape, + * no letterbox is applied."/> + * + * <p> + * A cutout in the corner is considered to be on the short edge: <br/> + * <img src="{@docRoot}reference/android/images/display_cutout/short_edge/fullscreen_corner_no_letterbox.png" + * height="720" + * alt="Screenshot of a fullscreen activity on a display with a cutout in the corner in + * portrait, no letterbox is applied."/> + * + * <p> + * On the other hand, should the cutout be on the long edge of the display, a letterbox will + * be applied such that the window does not extend into the cutout on either long edge: + * <br/> + * <img src="{@docRoot}reference/android/images/display_cutout/short_edge/portrait_side_letterbox.png" + * height="720" + * alt="Screenshot of an activity on a display with a cutout on the long edge in portrait, + * letterbox is applied."/> + * * @see DisplayCutout * @see WindowInsets#getDisplayCutout() + * @see #layoutInDisplayCutoutMode + * @see android.R.attr#windowLayoutInDisplayCutoutMode + * android:windowLayoutInDisplayCutoutMode */ - public static final int LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS = 1; + public static final int LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES = 1; /** * The window is never allowed to overlap with the DisplayCutout area. * * <p> - * This should be used with windows that transiently set SYSTEM_UI_FLAG_FULLSCREEN to - * avoid a relayout of the window when the flag is set or cleared. + * This should be used with windows that transiently set + * {@link View#SYSTEM_UI_FLAG_FULLSCREEN} or {@link View#SYSTEM_UI_FLAG_HIDE_NAVIGATION} + * to avoid a relayout of the window when the respective flag is set or cleared. * * @see DisplayCutout - * @see View#SYSTEM_UI_FLAG_FULLSCREEN SYSTEM_UI_FLAG_FULLSCREEN - * @see View#SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN + * @see #layoutInDisplayCutoutMode + * @see android.R.attr#windowLayoutInDisplayCutoutMode + * android:windowLayoutInDisplayCutoutMode */ public static final int LAYOUT_IN_DISPLAY_CUTOUT_MODE_NEVER = 2; @@ -2344,7 +2412,7 @@ public interface WindowManager extends ViewManager { * * @hide */ - public int accessibilityIdOfAnchor = -1; + public long accessibilityIdOfAnchor = AccessibilityNodeInfo.UNDEFINED_NODE_ID; /** * The window title isn't kept in sync with what is displayed in the title bar, so we @@ -2370,6 +2438,13 @@ public interface WindowManager extends ViewManager { public long hideTimeoutMilliseconds = -1; /** + * A frame number in which changes requested in this layout will be rendered. + * + * @hide + */ + public long frameNumber = -1; + + /** * The color mode requested by this window. The target display may * not be able to honor the request. When the color mode is not set * to {@link ActivityInfo#COLOR_MODE_DEFAULT}, it might override the @@ -2538,10 +2613,11 @@ public interface WindowManager extends ViewManager { out.writeInt(hasManualSurfaceInsets ? 1 : 0); out.writeInt(preservePreviousSurfaceInsets ? 1 : 0); out.writeInt(needsMenuKey); - out.writeInt(accessibilityIdOfAnchor); + out.writeLong(accessibilityIdOfAnchor); TextUtils.writeToParcel(accessibilityTitle, out, parcelableFlags); out.writeInt(mColorMode); out.writeLong(hideTimeoutMilliseconds); + out.writeLong(frameNumber); } public static final Parcelable.Creator<LayoutParams> CREATOR @@ -2594,10 +2670,11 @@ public interface WindowManager extends ViewManager { hasManualSurfaceInsets = in.readInt() != 0; preservePreviousSurfaceInsets = in.readInt() != 0; needsMenuKey = in.readInt(); - accessibilityIdOfAnchor = in.readInt(); + accessibilityIdOfAnchor = in.readLong(); accessibilityTitle = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(in); mColorMode = in.readInt(); hideTimeoutMilliseconds = in.readLong(); + frameNumber = in.readLong(); } @SuppressWarnings({"PointlessBitwiseExpression"}) @@ -2798,6 +2875,10 @@ public interface WindowManager extends ViewManager { changes |= SURFACE_INSETS_CHANGED; } + // The frame number changing is only relevant in the context of other + // changes, and so we don't need to track it with a flag. + frameNumber = o.frameNumber; + if (hasManualSurfaceInsets != o.hasManualSurfaceInsets) { hasManualSurfaceInsets = o.hasManualSurfaceInsets; changes |= SURFACE_INSETS_CHANGED; @@ -2855,9 +2936,8 @@ public interface WindowManager extends ViewManager { /** * @hide */ - public String toString(String prefix) { - StringBuilder sb = new StringBuilder(256); - sb.append("{("); + public void dumpDimensions(StringBuilder sb) { + sb.append('('); sb.append(x); sb.append(','); sb.append(y); @@ -2868,6 +2948,15 @@ public interface WindowManager extends ViewManager { sb.append((height == MATCH_PARENT ? "fill" : (height == WRAP_CONTENT ? "wrap" : String.valueOf(height)))); sb.append(")"); + } + + /** + * @hide + */ + public String toString(String prefix) { + StringBuilder sb = new StringBuilder(256); + sb.append('{'); + dumpDimensions(sb); if (horizontalMargin != 0) { sb.append(" hm="); sb.append(horizontalMargin); diff --git a/android/view/WindowManagerPolicyConstants.java b/android/view/WindowManagerPolicyConstants.java index a6f36bbf..23dc9da6 100644 --- a/android/view/WindowManagerPolicyConstants.java +++ b/android/view/WindowManagerPolicyConstants.java @@ -51,6 +51,12 @@ public interface WindowManagerPolicyConstants { int NAV_BAR_BOTTOM = 1 << 2; /** + * Broadcast sent when a user activity is detected. + */ + String ACTION_USER_ACTIVITY_NOTIFICATION = + "android.intent.action.USER_ACTIVITY_NOTIFICATION"; + + /** * Sticky broadcast of the current HDMI plugged state. */ String ACTION_HDMI_PLUGGED = "android.intent.action.HDMI_PLUGGED"; diff --git a/android/view/accessibility/AccessibilityManager.java b/android/view/accessibility/AccessibilityManager.java index dd8ba556..84b40641 100644 --- a/android/view/accessibility/AccessibilityManager.java +++ b/android/view/accessibility/AccessibilityManager.java @@ -17,6 +17,7 @@ package android.view.accessibility; import android.accessibilityservice.AccessibilityServiceInfo; +import android.accessibilityservice.AccessibilityServiceInfo.FeedbackType; import android.annotation.NonNull; import android.annotation.Nullable; import android.content.Context; @@ -24,6 +25,7 @@ import android.content.pm.ServiceInfo; import android.os.Handler; import android.view.IWindow; import android.view.View; +import android.view.accessibility.AccessibilityEvent.EventType; import java.util.Collections; import java.util.List; @@ -90,6 +92,60 @@ public final class AccessibilityManager { public void onHighTextContrastStateChanged(boolean enabled); } + /** + * Policy to inject behavior into the accessibility manager. + * + * @hide + */ + public interface AccessibilityPolicy { + /** + * Checks whether accessibility is enabled. + * + * @param accessibilityEnabled Whether the accessibility layer is enabled. + * @return whether accessibility is enabled. + */ + boolean isEnabled(boolean accessibilityEnabled); + + /** + * Notifies the policy for an accessibility event. + * + * @param event The event. + * @param accessibilityEnabled Whether the accessibility layer is enabled. + * @param relevantEventTypes The events relevant events. + * @return The event to dispatch or null. + */ + @Nullable AccessibilityEvent onAccessibilityEvent(@NonNull AccessibilityEvent event, + boolean accessibilityEnabled, @EventType int relevantEventTypes); + + /** + * Gets the list of relevant events. + * + * @param relevantEventTypes The relevant events. + * @return The relevant events to report. + */ + @EventType int getRelevantEventTypes(@EventType int relevantEventTypes); + + /** + * Gets the list of installed services to report. + * + * @param installedService The installed services. + * @return The services to report. + */ + @NonNull List<AccessibilityServiceInfo> getInstalledAccessibilityServiceList( + @Nullable List<AccessibilityServiceInfo> installedService); + + /** + * Gets the list of enabled accessibility services. + * + * @param feedbackTypeFlags The feedback type to query for. + * @param enabledService The enabled services. + * @return The services to report. + */ + @Nullable List<AccessibilityServiceInfo> getEnabledAccessibilityServiceList( + @FeedbackType int feedbackTypeFlags, + @Nullable List<AccessibilityServiceInfo> enabledService); + } + private final IAccessibilityManagerClient.Stub mClient = new IAccessibilityManagerClient.Stub() { public void setState(int state) { diff --git a/android/view/accessibility/AccessibilityNodeInfo.java b/android/view/accessibility/AccessibilityNodeInfo.java index 23e7d619..4c437dd4 100644 --- a/android/view/accessibility/AccessibilityNodeInfo.java +++ b/android/view/accessibility/AccessibilityNodeInfo.java @@ -3173,6 +3173,15 @@ public class AccessibilityNodeInfo implements Parcelable { */ @Override public void writeToParcel(Parcel parcel, int flags) { + writeToParcelNoRecycle(parcel, flags); + // Since instances of this class are fetched via synchronous i.e. blocking + // calls in IPCs we always recycle as soon as the instance is marshaled. + recycle(); + } + + /** @hide */ + @TestApi + public void writeToParcelNoRecycle(Parcel parcel, int flags) { // Write bit set of indices of fields with values differing from default long nonDefaultFields = 0; int fieldIndex = 0; // index of the current field @@ -3194,7 +3203,7 @@ public class AccessibilityNodeInfo implements Parcelable { fieldIndex++; if (mConnectionId != DEFAULT.mConnectionId) nonDefaultFields |= bitAt(fieldIndex); fieldIndex++; - if (!Objects.equals(mChildNodeIds, DEFAULT.mChildNodeIds)) { + if (!LongArray.elementsEqual(mChildNodeIds, DEFAULT.mChildNodeIds)) { nonDefaultFields |= bitAt(fieldIndex); } fieldIndex++; @@ -3324,7 +3333,7 @@ public class AccessibilityNodeInfo implements Parcelable { final int actionCount = mActions.size(); int nonStandardActionCount = 0; - int defaultStandardActions = 0; + long defaultStandardActions = 0; for (int i = 0; i < actionCount; i++) { AccessibilityAction action = mActions.get(i); if (isDefaultStandardAction(action)) { @@ -3333,7 +3342,7 @@ public class AccessibilityNodeInfo implements Parcelable { nonStandardActionCount++; } } - parcel.writeInt(defaultStandardActions); + parcel.writeLong(defaultStandardActions); parcel.writeInt(nonStandardActionCount); for (int i = 0; i < actionCount; i++) { @@ -3344,7 +3353,7 @@ public class AccessibilityNodeInfo implements Parcelable { } } } else { - parcel.writeInt(0); + parcel.writeLong(0); parcel.writeInt(0); } } @@ -3406,10 +3415,6 @@ public class AccessibilityNodeInfo implements Parcelable { + " vs " + fieldIndex); } } - - // Since instances of this class are fetched via synchronous i.e. blocking - // calls in IPCs we always recycle as soon as the instance is marshaled. - recycle(); } /** @@ -3535,7 +3540,7 @@ public class AccessibilityNodeInfo implements Parcelable { } if (isBitSet(nonDefaultFields, fieldIndex++)) { - final int standardActions = parcel.readInt(); + final long standardActions = parcel.readLong(); addStandardActions(standardActions); final int nonStandardActionCount = parcel.readInt(); for (int i = 0; i < nonStandardActionCount; i++) { @@ -3557,7 +3562,7 @@ public class AccessibilityNodeInfo implements Parcelable { if (isBitSet(nonDefaultFields, fieldIndex++)) { mContentDescription = parcel.readCharSequence(); } - if (isBitSet(nonDefaultFields, fieldIndex++)) mPaneTitle = parcel.readString(); + if (isBitSet(nonDefaultFields, fieldIndex++)) mPaneTitle = parcel.readCharSequence(); if (isBitSet(nonDefaultFields, fieldIndex++)) mTooltipText = parcel.readCharSequence(); if (isBitSet(nonDefaultFields, fieldIndex++)) mViewIdResourceName = parcel.readString(); @@ -3616,7 +3621,7 @@ public class AccessibilityNodeInfo implements Parcelable { } private static boolean isDefaultStandardAction(AccessibilityAction action) { - return action.mSerializationFlag != -1 && TextUtils.isEmpty(action.getLabel()); + return (action.mSerializationFlag != -1L) && TextUtils.isEmpty(action.getLabel()); } private static AccessibilityAction getActionSingleton(int actionId) { @@ -3631,7 +3636,7 @@ public class AccessibilityNodeInfo implements Parcelable { return null; } - private static AccessibilityAction getActionSingletonBySerializationFlag(int flag) { + private static AccessibilityAction getActionSingletonBySerializationFlag(long flag) { final int actions = AccessibilityAction.sStandardActions.size(); for (int i = 0; i < actions; i++) { AccessibilityAction currentAction = AccessibilityAction.sStandardActions.valueAt(i); @@ -3643,10 +3648,10 @@ public class AccessibilityNodeInfo implements Parcelable { return null; } - private void addStandardActions(int serializationIdMask) { - int remainingIds = serializationIdMask; + private void addStandardActions(long serializationIdMask) { + long remainingIds = serializationIdMask; while (remainingIds > 0) { - final int id = 1 << Integer.numberOfTrailingZeros(remainingIds); + final long id = 1L << Long.numberOfTrailingZeros(remainingIds); remainingIds &= ~id; AccessibilityAction action = getActionSingletonBySerializationFlag(id); addAction(action); @@ -3853,6 +3858,7 @@ public class AccessibilityNodeInfo implements Parcelable { builder.append("; password: ").append(isPassword()); builder.append("; scrollable: ").append(isScrollable()); builder.append("; importantForAccessibility: ").append(isImportantForAccessibility()); + builder.append("; visible: ").append(isVisibleToUser()); builder.append("; actions: ").append(mActions); return builder.toString(); @@ -4271,7 +4277,7 @@ public class AccessibilityNodeInfo implements Parcelable { private final CharSequence mLabel; /** @hide */ - public int mSerializationFlag = -1; + public long mSerializationFlag = -1L; /** * Creates a new AccessibilityAction. For adding a standard action without a specific label, @@ -4305,7 +4311,7 @@ public class AccessibilityNodeInfo implements Parcelable { private AccessibilityAction(int standardActionId) { this(standardActionId, null); - mSerializationFlag = (int) bitAt(sStandardActions.size()); + mSerializationFlag = bitAt(sStandardActions.size()); sStandardActions.add(this); } diff --git a/android/view/accessibility/AccessibilityViewHierarchyState.java b/android/view/accessibility/AccessibilityViewHierarchyState.java deleted file mode 100644 index 447fafaa..00000000 --- a/android/view/accessibility/AccessibilityViewHierarchyState.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.accessibility; - -import android.annotation.NonNull; -import android.annotation.Nullable; - -/** - * Accessibility-related state of a {@link android.view.ViewRootImpl} - * - * @hide - */ -public class AccessibilityViewHierarchyState { - private @Nullable SendViewScrolledAccessibilityEvent mSendViewScrolledAccessibilityEvent; - private @Nullable SendWindowContentChangedAccessibilityEvent - mSendWindowContentChangedAccessibilityEvent; - - /** - * @return a {@link SendViewScrolledAccessibilityEvent}, creating one if needed - */ - public @NonNull SendViewScrolledAccessibilityEvent getSendViewScrolledAccessibilityEvent() { - if (mSendViewScrolledAccessibilityEvent == null) { - mSendViewScrolledAccessibilityEvent = new SendViewScrolledAccessibilityEvent(); - } - return mSendViewScrolledAccessibilityEvent; - } - - public boolean isScrollEventSenderInitialized() { - return mSendViewScrolledAccessibilityEvent != null; - } - - /** - * @return a {@link SendWindowContentChangedAccessibilityEvent}, creating one if needed - */ - public @NonNull SendWindowContentChangedAccessibilityEvent - getSendWindowContentChangedAccessibilityEvent() { - if (mSendWindowContentChangedAccessibilityEvent == null) { - mSendWindowContentChangedAccessibilityEvent = - new SendWindowContentChangedAccessibilityEvent(); - } - return mSendWindowContentChangedAccessibilityEvent; - } - - public boolean isWindowContentChangedEventSenderInitialized() { - return mSendWindowContentChangedAccessibilityEvent != null; - } -} diff --git a/android/view/accessibility/SendViewScrolledAccessibilityEvent.java b/android/view/accessibility/SendViewScrolledAccessibilityEvent.java deleted file mode 100644 index 40a1b6a2..00000000 --- a/android/view/accessibility/SendViewScrolledAccessibilityEvent.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.accessibility; - - -import android.annotation.NonNull; -import android.view.View; - -/** - * Sender for {@link AccessibilityEvent#TYPE_VIEW_SCROLLED} accessibility event. - * - * @hide - */ -public class SendViewScrolledAccessibilityEvent extends ThrottlingAccessibilityEventSender { - - public int mDeltaX; - public int mDeltaY; - - /** - * Post a scroll event to be sent for the given view - */ - public void post(View source, int dx, int dy) { - if (!isPendingFor(source)) sendNowIfPending(); - - mDeltaX += dx; - mDeltaY += dy; - - if (!isPendingFor(source)) scheduleFor(source); - } - - @Override - protected void performSendEvent(@NonNull View source) { - AccessibilityEvent event = AccessibilityEvent.obtain(AccessibilityEvent.TYPE_VIEW_SCROLLED); - event.setScrollDeltaX(mDeltaX); - event.setScrollDeltaY(mDeltaY); - source.sendAccessibilityEventUnchecked(event); - } - - @Override - protected void resetState(@NonNull View source) { - mDeltaX = 0; - mDeltaY = 0; - } -} diff --git a/android/view/accessibility/SendWindowContentChangedAccessibilityEvent.java b/android/view/accessibility/SendWindowContentChangedAccessibilityEvent.java deleted file mode 100644 index df38fba5..00000000 --- a/android/view/accessibility/SendWindowContentChangedAccessibilityEvent.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.accessibility; - - -import static com.android.internal.util.ObjectUtils.firstNotNull; - -import android.annotation.NonNull; -import android.annotation.Nullable; -import android.view.View; -import android.view.ViewParent; - -import java.util.HashSet; - -/** - * @hide - */ -public class SendWindowContentChangedAccessibilityEvent - extends ThrottlingAccessibilityEventSender { - - private int mChangeTypes = 0; - - private HashSet<View> mTempHashSet; - - @Override - protected void performSendEvent(@NonNull View source) { - AccessibilityEvent event = AccessibilityEvent.obtain(); - event.setEventType(AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); - event.setContentChangeTypes(mChangeTypes); - source.sendAccessibilityEventUnchecked(event); - } - - @Override - protected void resetState(@Nullable View source) { - if (source != null) { - source.resetSubtreeAccessibilityStateChanged(); - } - mChangeTypes = 0; - } - - /** - * Post the {@link AccessibilityEvent#TYPE_WINDOW_CONTENT_CHANGED} event with the given - * {@link AccessibilityEvent#getContentChangeTypes change type} for the given view - */ - public void runOrPost(View source, int changeType) { - if (source.getAccessibilityLiveRegion() != View.ACCESSIBILITY_LIVE_REGION_NONE) { - sendNowIfPending(); - mChangeTypes = changeType; - sendNow(source); - } else { - mChangeTypes |= changeType; - scheduleFor(source); - } - } - - @Override - protected @Nullable View tryMerge(@NonNull View oldSource, @NonNull View newSource) { - // If there is no common predecessor, then oldSource points to - // a removed view, hence in this case always prefer the newSource. - return firstNotNull( - getCommonPredecessor(oldSource, newSource), - newSource); - } - - private View getCommonPredecessor(View first, View second) { - if (mTempHashSet == null) { - mTempHashSet = new HashSet<>(); - } - HashSet<View> seen = mTempHashSet; - seen.clear(); - View firstCurrent = first; - while (firstCurrent != null) { - seen.add(firstCurrent); - ViewParent firstCurrentParent = firstCurrent.getParent(); - if (firstCurrentParent instanceof View) { - firstCurrent = (View) firstCurrentParent; - } else { - firstCurrent = null; - } - } - View secondCurrent = second; - while (secondCurrent != null) { - if (seen.contains(secondCurrent)) { - seen.clear(); - return secondCurrent; - } - ViewParent secondCurrentParent = secondCurrent.getParent(); - if (secondCurrentParent instanceof View) { - secondCurrent = (View) secondCurrentParent; - } else { - secondCurrent = null; - } - } - seen.clear(); - return null; - } -} diff --git a/android/view/accessibility/ThrottlingAccessibilityEventSender.java b/android/view/accessibility/ThrottlingAccessibilityEventSender.java deleted file mode 100644 index 66fa3010..00000000 --- a/android/view/accessibility/ThrottlingAccessibilityEventSender.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.accessibility; - - -import android.annotation.NonNull; -import android.annotation.Nullable; -import android.os.Handler; -import android.os.Looper; -import android.os.SystemClock; -import android.util.Log; -import android.view.View; -import android.view.ViewConfiguration; -import android.view.ViewRootImpl; -import android.view.ViewRootImpl.CalledFromWrongThreadException; - -/** - * A throttling {@link AccessibilityEvent} sender that relies on its currently associated - * 'source' view's {@link View#postDelayed delayed execution} to delay and possibly - * {@link #tryMerge merge} together any events that come in less than - * {@link ViewConfiguration#getSendRecurringAccessibilityEventsInterval - * the configured amount of milliseconds} apart. - * - * The suggested usage is to create a singleton extending this class, holding any state specific to - * the particular event type that the subclass represents, and have an 'entrypoint' method that - * delegates to {@link #scheduleFor(View)}. - * For example: - * - * {@code - * public void post(View view, String text, int resId) { - * mText = text; - * mId = resId; - * scheduleFor(view); - * } - * } - * - * @see #scheduleFor(View) - * @see #tryMerge(View, View) - * @see #performSendEvent(View) - * @hide - */ -public abstract class ThrottlingAccessibilityEventSender { - - private static final boolean DEBUG = false; - private static final String LOG_TAG = "ThrottlingA11ySender"; - - View mSource; - private long mLastSendTimeMillis = Long.MIN_VALUE; - private boolean mIsPending = false; - - private final Runnable mWorker = () -> { - View source = mSource; - if (DEBUG) Log.d(LOG_TAG, thisClass() + ".run(mSource = " + source + ")"); - - if (!checkAndResetIsPending() || source == null) { - resetStateInternal(); - return; - } - - // Accessibility may be turned off while we were waiting - if (isAccessibilityEnabled(source)) { - mLastSendTimeMillis = SystemClock.uptimeMillis(); - performSendEvent(source); - } - resetStateInternal(); - }; - - /** - * Populate and send an {@link AccessibilityEvent} using the given {@code source} view, as well - * as any extra data from this instance's state. - * - * Send the event via {@link View#sendAccessibilityEventUnchecked(AccessibilityEvent)} or - * {@link View#sendAccessibilityEvent(int)} on the provided {@code source} view to allow for - * overrides of those methods on {@link View} subclasses to take effect, and/or make sure that - * an {@link View#getAccessibilityDelegate() accessibility delegate} is not ignored if any. - */ - protected abstract void performSendEvent(@NonNull View source); - - /** - * Perform optional cleanup after {@link #performSendEvent} - * - * @param source the view this event was associated with - */ - protected abstract void resetState(@Nullable View source); - - /** - * Attempt to merge the pending events for source views {@code oldSource} and {@code newSource} - * into one, with source set to the resulting {@link View} - * - * A result of {@code null} means merger is not possible, resulting in the currently pending - * event being flushed before proceeding. - */ - protected @Nullable View tryMerge(@NonNull View oldSource, @NonNull View newSource) { - return null; - } - - /** - * Schedules a {@link #performSendEvent} with the source {@link View} set to given - * {@code source} - * - * If an event is already scheduled a {@link #tryMerge merge} will be attempted. - * If merging is not possible (as indicated by the null result from {@link #tryMerge}), - * the currently scheduled event will be {@link #sendNow sent immediately} and the new one - * will be scheduled afterwards. - */ - protected final void scheduleFor(@NonNull View source) { - if (DEBUG) Log.d(LOG_TAG, thisClass() + ".scheduleFor(source = " + source + ")"); - - Handler uiHandler = source.getHandler(); - if (uiHandler == null || uiHandler.getLooper() != Looper.myLooper()) { - CalledFromWrongThreadException e = new CalledFromWrongThreadException( - "Expected to be called from main thread but was called from " - + Thread.currentThread()); - // TODO: Throw the exception - Log.e(LOG_TAG, "Accessibility content change on non-UI thread. Future Android " - + "versions will throw an exception.", e); - } - - if (!isAccessibilityEnabled(source)) return; - - if (mIsPending) { - View merged = tryMerge(mSource, source); - if (merged != null) { - setSource(merged); - return; - } else { - sendNow(); - } - } - - setSource(source); - - final long timeSinceLastMillis = SystemClock.uptimeMillis() - mLastSendTimeMillis; - final long minEventIntervalMillis = - ViewConfiguration.getSendRecurringAccessibilityEventsInterval(); - if (timeSinceLastMillis >= minEventIntervalMillis) { - sendNow(); - } else { - mSource.postDelayed(mWorker, minEventIntervalMillis - timeSinceLastMillis); - } - } - - static boolean isAccessibilityEnabled(@NonNull View contextProvider) { - return AccessibilityManager.getInstance(contextProvider.getContext()).isEnabled(); - } - - protected final void sendNow(View source) { - setSource(source); - sendNow(); - } - - private void sendNow() { - mSource.removeCallbacks(mWorker); - mWorker.run(); - } - - /** - * Flush the event if one is pending - */ - public void sendNowIfPending() { - if (mIsPending) sendNow(); - } - - /** - * Cancel the event if one is pending and is for the given view - */ - public final void cancelIfPendingFor(@NonNull View source) { - if (isPendingFor(source)) cancelIfPending(this); - } - - /** - * @return whether an event is currently pending for the given source view - */ - protected final boolean isPendingFor(@Nullable View source) { - return mIsPending && mSource == source; - } - - /** - * Cancel the event if one is not null and pending - */ - public static void cancelIfPending(@Nullable ThrottlingAccessibilityEventSender sender) { - if (sender == null || !sender.checkAndResetIsPending()) return; - sender.mSource.removeCallbacks(sender.mWorker); - sender.resetStateInternal(); - } - - void resetStateInternal() { - if (DEBUG) Log.d(LOG_TAG, thisClass() + ".resetStateInternal()"); - - resetState(mSource); - setSource(null); - } - - boolean checkAndResetIsPending() { - if (mIsPending) { - mIsPending = false; - return true; - } else { - return false; - } - } - - private void setSource(@Nullable View source) { - if (DEBUG) Log.d(LOG_TAG, thisClass() + ".setSource(" + source + ")"); - - if (source == null && mIsPending) { - Log.e(LOG_TAG, "mSource nullified while callback still pending: " + this); - return; - } - - if (source != null && !mIsPending) { - // At most one can be pending at any given time - View oldSource = mSource; - if (oldSource != null) { - ViewRootImpl viewRootImpl = oldSource.getViewRootImpl(); - if (viewRootImpl != null) { - viewRootImpl.flushPendingAccessibilityEvents(); - } - } - mIsPending = true; - } - mSource = source; - } - - String thisClass() { - return getClass().getSimpleName(); - } - - @Override - public String toString() { - return thisClass() + "(" + mSource + ")"; - } - -} diff --git a/android/view/animation/Animation.java b/android/view/animation/Animation.java index 474db128..64686dd9 100644 --- a/android/view/animation/Animation.java +++ b/android/view/animation/Animation.java @@ -206,6 +206,8 @@ public abstract class Animation implements Cloneable { */ private boolean mDetachWallpaper = false; + private boolean mShowWallpaper; + private boolean mMore = true; private boolean mOneMoreTime = true; @@ -253,7 +255,10 @@ public abstract class Animation implements Cloneable { setBackgroundColor(a.getInt(com.android.internal.R.styleable.Animation_background, 0)); - setDetachWallpaper(a.getBoolean(com.android.internal.R.styleable.Animation_detachWallpaper, false)); + setDetachWallpaper( + a.getBoolean(com.android.internal.R.styleable.Animation_detachWallpaper, false)); + setShowWallpaper( + a.getBoolean(com.android.internal.R.styleable.Animation_showWallpaper, false)); final int resID = a.getResourceId(com.android.internal.R.styleable.Animation_interpolator, 0); @@ -661,6 +666,18 @@ public abstract class Animation implements Cloneable { } /** + * If this animation is run as a window animation, this will make the wallpaper visible behind + * the animation. + * + * @param showWallpaper Whether the wallpaper should be shown during the animation. + * @attr ref android.R.styleable#Animation_detachWallpaper + * @hide + */ + public void setShowWallpaper(boolean showWallpaper) { + mShowWallpaper = showWallpaper; + } + + /** * Gets the acceleration curve type for this animation. * * @return the {@link Interpolator} associated to this animation @@ -775,6 +792,16 @@ public abstract class Animation implements Cloneable { } /** + * @return If run as a window animation, returns whether the wallpaper will be shown behind + * during the animation. + * @attr ref android.R.styleable#Animation_showWallpaper + * @hide + */ + public boolean getShowWallpaper() { + return mShowWallpaper; + } + + /** * <p>Indicates whether or not this animation will affect the transformation * matrix. For instance, a fade animation will not affect the matrix whereas * a scale animation will.</p> diff --git a/android/view/animation/AnimationUtils.java b/android/view/animation/AnimationUtils.java index 990fbdb0..29f8442b 100644 --- a/android/view/animation/AnimationUtils.java +++ b/android/view/animation/AnimationUtils.java @@ -18,6 +18,7 @@ package android.view.animation; import android.annotation.AnimRes; import android.annotation.InterpolatorRes; +import android.annotation.TestApi; import android.content.Context; import android.content.res.Resources; import android.content.res.Resources.NotFoundException; @@ -58,14 +59,43 @@ public class AnimationUtils { } }; - /** @hide */ + /** + * Locks AnimationUtils{@link #currentAnimationTimeMillis()} to a fixed value for the current + * thread. This is used by {@link android.view.Choreographer} to ensure that all accesses + * during a vsync update are synchronized to the timestamp of the vsync. + * + * It is also exposed to tests to allow for rapid, flake-free headless testing. + * + * Must be followed by a call to {@link #unlockAnimationClock()} to allow time to + * progress. Failing to do this will result in stuck animations, scrolls, and flings. + * + * Note that time is not allowed to "rewind" and must perpetually flow forward. So the + * lock may fail if the time is in the past from a previously returned value, however + * time will be frozen for the duration of the lock. The clock is a thread-local, so + * ensure that {@link #lockAnimationClock(long)}, {@link #unlockAnimationClock()}, and + * {@link #currentAnimationTimeMillis()} are all called on the same thread. + * + * This is also not reference counted in any way. Any call to {@link #unlockAnimationClock()} + * will unlock the clock for everyone on the same thread. It is therefore recommended + * for tests to use their own thread to ensure that there is no collision with any existing + * {@link android.view.Choreographer} instance. + * + * @hide + * */ + @TestApi public static void lockAnimationClock(long vsyncMillis) { AnimationState state = sAnimationState.get(); state.animationClockLocked = true; state.currentVsyncTimeMillis = vsyncMillis; } - /** @hide */ + /** + * Frees the time lock set in place by {@link #lockAnimationClock(long)}. Must be called + * to allow the animation clock to self-update. + * + * @hide + */ + @TestApi public static void unlockAnimationClock() { sAnimationState.get().animationClockLocked = false; } diff --git a/android/view/autofill/AutofillId.java b/android/view/autofill/AutofillId.java index 5ce2421a..cb1d89c5 100644 --- a/android/view/autofill/AutofillId.java +++ b/android/view/autofill/AutofillId.java @@ -38,6 +38,7 @@ public final class AutofillId implements Parcelable { } /** @hide */ + @TestApi public AutofillId(AutofillId parent, int virtualChildId) { mVirtual = true; mViewId = parent.mViewId; diff --git a/android/view/autofill/AutofillManager.java b/android/view/autofill/AutofillManager.java index 4b24a71c..88300dbd 100644 --- a/android/view/autofill/AutofillManager.java +++ b/android/view/autofill/AutofillManager.java @@ -20,14 +20,18 @@ import static android.service.autofill.FillRequest.FLAG_MANUAL_REQUEST; import static android.view.autofill.Helper.sDebug; import static android.view.autofill.Helper.sVerbose; +import android.accessibilityservice.AccessibilityServiceInfo; import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; +import android.annotation.RequiresFeature; import android.annotation.SystemService; import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.IntentSender; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; import android.graphics.Rect; import android.metrics.LogMaker; import android.os.Bundle; @@ -41,13 +45,24 @@ import android.util.ArrayMap; import android.util.ArraySet; import android.util.Log; import android.util.SparseArray; +import android.view.Choreographer; +import android.view.KeyEvent; import android.view.View; +import android.view.accessibility.AccessibilityEvent; +import android.view.accessibility.AccessibilityManager; +import android.view.accessibility.AccessibilityNodeInfo; +import android.view.accessibility.AccessibilityNodeProvider; +import android.view.accessibility.AccessibilityWindowInfo; import com.android.internal.annotations.GuardedBy; import com.android.internal.logging.MetricsLogger; import com.android.internal.logging.nano.MetricsProto.MetricsEvent; +import com.android.internal.util.ArrayUtils; import com.android.internal.util.Preconditions; +import org.xmlpull.v1.XmlPullParserException; + +import java.io.IOException; import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -58,7 +73,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -// TODO: use java.lang.ref.Cleaner once Android supports Java 9 +//TODO: use java.lang.ref.Cleaner once Android supports Java 9 import sun.misc.Cleaner; /** @@ -122,6 +137,7 @@ import sun.misc.Cleaner; * <p>It is safe to call into its methods from any thread. */ @SystemService(Context.AUTOFILL_MANAGER_SERVICE) +@RequiresFeature(PackageManager.FEATURE_AUTOFILL) public final class AutofillManager { private static final String TAG = "AutofillManager"; @@ -178,7 +194,6 @@ public final class AutofillManager { private static final String STATE_TAG = "android:state"; private static final String LAST_AUTOFILLED_DATA_TAG = "android:lastAutoFilledData"; - /** @hide */ public static final int ACTION_START_SESSION = 1; /** @hide */ public static final int ACTION_VIEW_ENTERED = 2; /** @hide */ public static final int ACTION_VIEW_EXITED = 3; @@ -259,6 +274,16 @@ public final class AutofillManager { public static final int STATE_DISABLED_BY_SERVICE = 4; /** + * Same as {@link #STATE_UNKNOWN}, but used on + * {@link AutofillManagerClient#setSessionFinished(int)} when the session was finished because + * the URL bar changed on client mode + * + * @hide + */ + public static final int STATE_UNKNOWN_COMPAT_MODE = 5; + + + /** * Timeout in ms for calls to the field classification service. * @hide */ @@ -343,6 +368,16 @@ public final class AutofillManager { @GuardedBy("mLock") @Nullable private ArraySet<AutofillId> mFillableIds; + /** id of last requested autofill ui */ + @Nullable private AutofillId mIdShownFillUi; + + /** + * Views that were already "entered" - if they're entered again when the session is not active, + * they're ignored + * */ + @GuardedBy("mLock") + @Nullable private ArraySet<AutofillId> mEnteredIds; + /** If set, session is commited when the field is clicked. */ @GuardedBy("mLock") @Nullable private AutofillId mSaveTriggerId; @@ -355,6 +390,10 @@ public final class AutofillManager { @GuardedBy("mLock") private boolean mSaveOnFinish; + /** If compatibility mode is enabled - this is a bridge to interact with a11y */ + @GuardedBy("mLock") + private CompatibilityBridge mCompatibilityBridge; + /** @hide */ public interface AutofillClient { /** @@ -364,13 +403,13 @@ public final class AutofillManager { * @param intent The authentication intent. * @param fillInIntent The authentication fill-in intent. */ - void autofillCallbackAuthenticate(int authenticationId, IntentSender intent, + void autofillClientAuthenticate(int authenticationId, IntentSender intent, Intent fillInIntent); /** * Tells the client this manager has state to be reset. */ - void autofillCallbackResetableStateAvailable(); + void autofillClientResetableStateAvailable(); /** * Request showing the autofill UI. @@ -382,29 +421,43 @@ public final class AutofillManager { * @param presenter The presenter that controls the fill UI window. * @return Whether the UI was shown. */ - boolean autofillCallbackRequestShowFillUi(@NonNull View anchor, int width, int height, + boolean autofillClientRequestShowFillUi(@NonNull View anchor, int width, int height, @Nullable Rect virtualBounds, IAutofillWindowPresenter presenter); /** + * Dispatch unhandled keyevent from Autofill window + * @param anchor The real view the UI needs to anchor to. + * @param keyEvent Unhandled KeyEvent from autofill window. + */ + void autofillClientDispatchUnhandledKey(@NonNull View anchor, @NonNull KeyEvent keyEvent); + + /** * Request hiding the autofill UI. * * @return Whether the UI was hidden. */ - boolean autofillCallbackRequestHideFillUi(); + boolean autofillClientRequestHideFillUi(); + + /** + * Gets whether the fill UI is currenlty being shown. + * + * @return Whether the fill UI is currently being shown + */ + boolean autofillClientIsFillUiShowing(); /** * Checks if views are currently attached and visible. * * @return And array with {@code true} iff the view is attached or visible */ - @NonNull boolean[] getViewVisibility(@NonNull int[] viewId); + @NonNull boolean[] autofillClientGetViewVisibility(@NonNull AutofillId[] autofillIds); /** * Checks is the client is currently visible as understood by autofill. * * @return {@code true} if the client is currently visible */ - boolean isVisibleForAutofill(); + boolean autofillClientIsVisibleForAutofill(); /** * Client might disable enter/exit event e.g. when activity is paused. @@ -414,30 +467,61 @@ public final class AutofillManager { /** * Finds views by traversing the hierarchies of the client. * - * @param viewIds The autofill ids of the views to find + * @param autofillIds The autofill ids of the views to find * * @return And array containing the views (empty if no views found). */ - @NonNull View[] findViewsByAutofillIdTraversal(@NonNull int[] viewIds); + @NonNull View[] autofillClientFindViewsByAutofillIdTraversal( + @NonNull AutofillId[] autofillIds); /** * Finds a view by traversing the hierarchies of the client. * - * @param viewId The autofill id of the views to find + * @param autofillId The autofill id of the views to find + * + * @return The view, or {@code null} if not found + */ + @Nullable View autofillClientFindViewByAutofillIdTraversal(@NonNull AutofillId autofillId); + + /** + * Finds a view by a11y id in a given client window. + * + * @param viewId The accessibility id of the views to find + * @param windowId The accessibility window id where to search * * @return The view, or {@code null} if not found */ - @Nullable View findViewByAutofillIdTraversal(int viewId); + @Nullable View autofillClientFindViewByAccessibilityIdTraversal(int viewId, int windowId); /** * Runs the specified action on the UI thread. */ - void runOnUiThread(Runnable action); + void autofillClientRunOnUiThread(Runnable action); /** * Gets the complete component name of this client. */ - ComponentName getComponentName(); + ComponentName autofillClientGetComponentName(); + + /** + * Gets the activity token + */ + @Nullable IBinder autofillClientGetActivityToken(); + + /** + * @return Whether compatibility mode is enabled. + */ + boolean autofillClientIsCompatibilityModeEnabled(); + + /** + * Gets the next unique autofill ID. + * + * <p>Typically used to manage views whose content is recycled - see + * {@link View#setAutofillId(AutofillId)} for more info. + * + * @return An ID that is unique in the activity. + */ + @Nullable AutofillId autofillClientGetNextAutofillId(); } /** @@ -449,6 +533,19 @@ public final class AutofillManager { } /** + * @hide + */ + public void enableCompatibilityMode() { + synchronized (mLock) { + // The accessibility manager is a singleton so we may need to plug + // different bridge based on which activity is currently focused + // in the current process. Since compat would be rarely used, just + // create and register a new instance every time. + mCompatibilityBridge = new CompatibilityBridge(); + } + } + + /** * Restore state after activity lifecycle * * @param savedInstanceState The state to be restored @@ -477,7 +574,8 @@ public final class AutofillManager { if (client != null) { try { final boolean sessionWasRestored = mService.restoreSession(mSessionId, - mContext.getActivityToken(), mServiceClient.asBinder()); + client.autofillClientGetActivityToken(), + mServiceClient.asBinder()); if (!sessionWasRestored) { Log.w(TAG, "Session " + mSessionId + " could not be restored"); @@ -488,7 +586,7 @@ public final class AutofillManager { Log.d(TAG, "session " + mSessionId + " was restored"); } - client.autofillCallbackResetableStateAvailable(); + client.autofillClientResetableStateAvailable(); } } catch (RemoteException e) { Log.e(TAG, "Could not figure out if there was an autofill session", e); @@ -501,22 +599,29 @@ public final class AutofillManager { /** * Called once the client becomes visible. * - * @see AutofillClient#isVisibleForAutofill() + * @see AutofillClient#autofillClientIsVisibleForAutofill() * * {@hide} */ public void onVisibleForAutofill() { - synchronized (mLock) { - if (mEnabled && isActiveLocked() && mTrackedViews != null) { - mTrackedViews.onVisibleForAutofillLocked(); + // This gets called when the client just got visible at which point the visibility + // of the tracked views may not have been computed (due to a pending layout, etc). + // While generally we have no way to know when the UI has settled. We will evaluate + // the tracked views state at the end of next frame to guarantee that everything + // that may need to be laid out is laid out. + Choreographer.getInstance().postCallback(Choreographer.CALLBACK_COMMIT, () -> { + synchronized (mLock) { + if (mEnabled && isActiveLocked() && mTrackedViews != null) { + mTrackedViews.onVisibleForAutofillChangedLocked(); + } } - } + }, null); } /** * Called once the client becomes invisible. * - * @see AutofillClient#isVisibleForAutofill() + * @see AutofillClient#autofillClientIsVisibleForAutofill() * * {@hide} */ @@ -551,6 +656,14 @@ public final class AutofillManager { } /** + * @hide + */ + @GuardedBy("mLock") + public boolean isCompatibilityModeEnabledLocked() { + return mCompatibilityBridge != null; + } + + /** * Checks whether autofill is enabled for the current user. * * <p>Typically used to determine whether the option to explicitly request autofill should @@ -636,24 +749,37 @@ public final class AutofillManager { notifyViewEntered(view, 0); } - private boolean shouldIgnoreViewEnteredLocked(@NonNull View view, int flags) { + @GuardedBy("mLock") + private boolean shouldIgnoreViewEnteredLocked(@NonNull AutofillId id, int flags) { if (isDisabledByServiceLocked()) { if (sVerbose) { - Log.v(TAG, "ignoring notifyViewEntered(flags=" + flags + ", view=" + view - + ") on state " + getStateAsStringLocked()); + Log.v(TAG, "ignoring notifyViewEntered(flags=" + flags + ", view=" + id + + ") on state " + getStateAsStringLocked() + " because disabled by svc"); } return true; } - if (sVerbose && isFinishedLocked()) { - Log.v(TAG, "not ignoring notifyViewEntered(flags=" + flags + ", view=" + view - + ") on state " + getStateAsStringLocked()); + if (isFinishedLocked()) { + // Session already finished: ignore if automatic request and view already entered + if ((flags & FLAG_MANUAL_REQUEST) == 0 && mEnteredIds != null + && mEnteredIds.contains(id)) { + if (sVerbose) { + Log.v(TAG, "ignoring notifyViewEntered(flags=" + flags + ", view=" + id + + ") on state " + getStateAsStringLocked() + + " because view was already entered: " + mEnteredIds); + } + return true; + } + } + if (sVerbose) { + Log.v(TAG, "not ignoring notifyViewEntered(flags=" + flags + ", view=" + id + + ", state " + getStateAsStringLocked() + ", enteredIds=" + mEnteredIds); } return false; } private boolean isClientVisibleForAutofillLocked() { final AutofillClient client = getClient(); - return client != null && client.isVisibleForAutofill(); + return client != null && client.autofillClientIsVisibleForAutofill(); } private boolean isClientDisablingEnterExitEvent() { @@ -676,8 +802,10 @@ public final class AutofillManager { } /** Returns AutofillCallback if need fire EVENT_INPUT_UNAVAILABLE */ + @GuardedBy("mLock") private AutofillCallback notifyViewEnteredLocked(@NonNull View view, int flags) { - if (shouldIgnoreViewEnteredLocked(view, flags)) return null; + final AutofillId id = view.getAutofillId(); + if (shouldIgnoreViewEnteredLocked(id, flags)) return null; AutofillCallback callback = null; @@ -690,7 +818,6 @@ public final class AutofillManager { } else { // don't notify entered when Activity is already in background if (!isClientDisablingEnterExitEvent()) { - final AutofillId id = getAutofillId(view); final AutofillValue value = view.getAutofillValue(); if (!isActiveLocked()) { @@ -700,6 +827,7 @@ public final class AutofillManager { // Update focus on existing session. updateSessionLocked(id, null, value, ACTION_VIEW_ENTERED, flags); } + addEnteredIdLocked(id); } } return callback; @@ -719,13 +847,14 @@ public final class AutofillManager { } } + @GuardedBy("mLock") void notifyViewExitedLocked(@NonNull View view) { ensureServiceClientAddedIfNeededLocked(); if (mEnabled && isActiveLocked()) { // dont notify exited when Activity is already in background if (!isClientDisablingEnterExitEvent()) { - final AutofillId id = getAutofillId(view); + final AutofillId id = view.getAutofillId(); // Update focus on existing session. updateSessionLocked(id, null, null, ACTION_VIEW_EXITED, 0); @@ -768,6 +897,7 @@ public final class AutofillManager { if (mEnabled && isActiveLocked()) { final AutofillId id = virtual ? getAutofillId(view, virtualId) : view.getAutofillId(); + if (sVerbose) Log.v(TAG, "visibility changed for " + id + ": " + isVisible); if (!isVisible && mFillableIds != null) { if (mFillableIds.contains(id)) { if (sDebug) Log.d(TAG, "Hidding UI when view " + id + " became invisible"); @@ -776,6 +906,8 @@ public final class AutofillManager { } if (mTrackedViews != null) { mTrackedViews.notifyViewVisibilityChangedLocked(id, isVisible); + } else if (sVerbose) { + Log.v(TAG, "Ignoring visibility change on " + id + ": no tracked views"); } } } @@ -820,10 +952,12 @@ public final class AutofillManager { } /** Returns AutofillCallback if need fire EVENT_INPUT_UNAVAILABLE */ + @GuardedBy("mLock") private AutofillCallback notifyViewEnteredLocked(View view, int virtualId, Rect bounds, int flags) { + final AutofillId id = getAutofillId(view, virtualId); AutofillCallback callback = null; - if (shouldIgnoreViewEnteredLocked(view, flags)) return callback; + if (shouldIgnoreViewEnteredLocked(id, flags)) return callback; ensureServiceClientAddedIfNeededLocked(); @@ -834,8 +968,6 @@ public final class AutofillManager { } else { // don't notify entered when Activity is already in background if (!isClientDisablingEnterExitEvent()) { - final AutofillId id = getAutofillId(view, virtualId); - if (!isActiveLocked()) { // Starts new session. startSessionLocked(id, bounds, null, flags); @@ -843,11 +975,20 @@ public final class AutofillManager { // Update focus on existing session. updateSessionLocked(id, bounds, null, ACTION_VIEW_ENTERED, flags); } + addEnteredIdLocked(id); } } return callback; } + @GuardedBy("mLock") + private void addEnteredIdLocked(@NonNull AutofillId id) { + if (mEnteredIds == null) { + mEnteredIds = new ArraySet<>(1); + } + mEnteredIds.add(id); + } + /** * Called when a virtual view that supports autofill is exited. * @@ -855,6 +996,7 @@ public final class AutofillManager { * @param virtualId id identifying the virtual child inside the parent view. */ public void notifyViewExited(@NonNull View view, int virtualId) { + if (sVerbose) Log.v(TAG, "notifyViewExited(" + view.getAutofillId() + ", " + virtualId); if (!hasAutofillFeature()) { return; } @@ -863,6 +1005,7 @@ public final class AutofillManager { } } + @GuardedBy("mLock") private void notifyViewExitedLocked(@NonNull View view, int virtualId) { ensureServiceClientAddedIfNeededLocked(); @@ -896,7 +1039,7 @@ public final class AutofillManager { if (mLastAutofilledData == null) { view.setAutofilled(false); } else { - id = getAutofillId(view); + id = view.getAutofillId(); if (mLastAutofilledData.containsKey(id)) { value = view.getAutofillValue(); valueWasRead = true; @@ -913,15 +1056,15 @@ public final class AutofillManager { } if (!mEnabled || !isActiveLocked()) { - if (sVerbose && mEnabled) { - Log.v(TAG, "notifyValueChanged(" + view + "): ignoring on state " - + getStateAsStringLocked()); + if (sVerbose) { + Log.v(TAG, "notifyValueChanged(" + view.getAutofillId() + + "): ignoring on state " + getStateAsStringLocked()); } return; } if (id == null) { - id = getAutofillId(view); + id = view.getAutofillId(); } if (!valueWasRead) { @@ -945,6 +1088,10 @@ public final class AutofillManager { } synchronized (mLock) { if (!mEnabled || !isActiveLocked()) { + if (sVerbose) { + Log.v(TAG, "notifyValueChanged(" + view.getAutofillId() + ":" + virtualId + + "): ignoring on state " + getStateAsStringLocked()); + } return; } @@ -953,18 +1100,35 @@ public final class AutofillManager { } } + /** + * Called to indicate a {@link View} is clicked. + * + * @param view view that has been clicked. + */ + public void notifyViewClicked(@NonNull View view) { + notifyViewClicked(view.getAutofillId()); + } /** - * Called when a {@link View} is clicked. Currently only used by views that should trigger save. + * Called to indicate a virtual view has been clicked. * - * @hide + * @param view the virtual view parent. + * @param virtualId id identifying the virtual child inside the parent view. */ - public void notifyViewClicked(View view) { - final AutofillId id = view.getAutofillId(); + public void notifyViewClicked(@NonNull View view, int virtualId) { + notifyViewClicked(getAutofillId(view, virtualId)); + } + private void notifyViewClicked(AutofillId id) { + if (!hasAutofillFeature()) { + return; + } if (sVerbose) Log.v(TAG, "notifyViewClicked(): id=" + id + ", trigger=" + mSaveTriggerId); synchronized (mLock) { + if (!mEnabled || !isActiveLocked()) { + return; + } if (mSaveTriggerId != null && mSaveTriggerId.equals(id)) { if (sDebug) Log.d(TAG, "triggering commit by click of " + id); commitLocked(); @@ -979,16 +1143,16 @@ public final class AutofillManager { * * @hide */ - public void onActivityFinished() { + public void onActivityFinishing() { if (!hasAutofillFeature()) { return; } synchronized (mLock) { if (mSaveOnFinish) { - if (sDebug) Log.d(TAG, "Committing session on finish() as requested by service"); + if (sDebug) Log.d(TAG, "onActivityFinishing(): calling commitLocked()"); commitLocked(); } else { - if (sDebug) Log.d(TAG, "Cancelling session on finish() as requested by service"); + if (sDebug) Log.d(TAG, "onActivityFinishing(): calling cancelLocked()"); cancelLocked(); } } @@ -1009,11 +1173,13 @@ public final class AutofillManager { if (!hasAutofillFeature()) { return; } + if (sVerbose) Log.v(TAG, "commit() called by app"); synchronized (mLock) { commitLocked(); } } + @GuardedBy("mLock") private void commitLocked() { if (!mEnabled && !isActiveLocked()) { return; @@ -1042,6 +1208,7 @@ public final class AutofillManager { } } + @GuardedBy("mLock") private void cancelLocked() { if (!mEnabled && !isActiveLocked()) { return; @@ -1099,6 +1266,30 @@ public final class AutofillManager { } /** + * Gets the id of the {@link UserData} used for + * <a href="AutofillService.html#FieldClassification">field classification</a>. + * + * <p>This method is useful when the service must check the status of the {@link UserData} in + * the device without fetching the whole object. + * + * <p><b>Note:</b> This method should only be called by an app providing an autofill service, + * and it's ignored if the caller currently doesn't have an enabled autofill service for + * the user. + * + * @return id of the {@link UserData} previously set by {@link #setUserData(UserData)} + * or {@code null} if it was reset or if the caller currently does not have an enabled autofill + * service for the user. + */ + @Nullable public String getUserDataId() { + try { + return mService.getUserDataId(); + } catch (RemoteException e) { + e.rethrowFromSystemServer(); + return null; + } + } + + /** * Gets the user data used for * <a href="AutofillService.html#FieldClassification">field classification</a>. * @@ -1119,7 +1310,7 @@ public final class AutofillManager { } /** - * Sets the user data used for + * Sets the {@link UserData} used for * <a href="AutofillService.html#FieldClassification">field classification</a> * * <p><b>Note:</b> This method should only be called by an app providing an autofill service, @@ -1226,6 +1417,15 @@ public final class AutofillManager { return client; } + /** + * Check if autofill ui is showing, must be called on UI thread. + * @hide + */ + public boolean isAutofillUiShowing() { + final AutofillClient client = mContext.getAutofillClient(); + return client != null && client.autofillClientIsFillUiShowing(); + } + /** @hide */ public void onAuthenticationResult(int authenticationId, Intent data, View focusView) { if (!hasAutofillFeature()) { @@ -1273,19 +1473,41 @@ public final class AutofillManager { } } - private static AutofillId getAutofillId(View view) { - return new AutofillId(view.getAutofillViewId()); + /** + * Gets the next unique autofill ID for the activity context. + * + * <p>Typically used to manage views whose content is recycled - see + * {@link View#setAutofillId(AutofillId)} for more info. + * + * @return An ID that is unique in the activity, or {@code null} if autofill is not supported in + * the {@link Context} associated with this {@link AutofillManager}. + */ + @Nullable + public AutofillId getNextAutofillId() { + final AutofillClient client = getClient(); + if (client == null) return null; + + final AutofillId id = client.autofillClientGetNextAutofillId(); + + if (id == null && sDebug) { + Log.d(TAG, "getNextAutofillId(): client " + client + " returned null"); + } + + return id; } private static AutofillId getAutofillId(View parent, int virtualId) { return new AutofillId(parent.getAutofillViewId(), virtualId); } + @GuardedBy("mLock") private void startSessionLocked(@NonNull AutofillId id, @NonNull Rect bounds, @NonNull AutofillValue value, int flags) { if (sVerbose) { Log.v(TAG, "startSessionLocked(): id=" + id + ", bounds=" + bounds + ", value=" + value - + ", flags=" + flags + ", state=" + getStateAsStringLocked()); + + ", flags=" + flags + ", state=" + getStateAsStringLocked() + + ", compatMode=" + isCompatibilityModeEnabledLocked() + + ", enteredIds=" + mEnteredIds); } if (mState != STATE_UNKNOWN && !isFinishedLocked() && (flags & FLAG_MANUAL_REQUEST) == 0) { if (sVerbose) { @@ -1296,20 +1518,22 @@ public final class AutofillManager { } try { final AutofillClient client = getClient(); - if (client == null) return; // NOTE: getClient() already logd it.. + if (client == null) return; // NOTE: getClient() already logged it.. - mSessionId = mService.startSession(mContext.getActivityToken(), + mSessionId = mService.startSession(client.autofillClientGetActivityToken(), mServiceClient.asBinder(), id, bounds, value, mContext.getUserId(), - mCallback != null, flags, client.getComponentName()); + mCallback != null, flags, client.autofillClientGetComponentName(), + isCompatibilityModeEnabledLocked()); if (mSessionId != NO_SESSION) { mState = STATE_ACTIVE; } - client.autofillCallbackResetableStateAvailable(); + client.autofillClientResetableStateAvailable(); } catch (RemoteException e) { throw e.rethrowFromSystemServer(); } } + @GuardedBy("mLock") private void finishSessionLocked() { if (sVerbose) Log.v(TAG, "finishSessionLocked(): " + getStateAsStringLocked()); @@ -1321,9 +1545,10 @@ public final class AutofillManager { throw e.rethrowFromSystemServer(); } - resetSessionLocked(); + resetSessionLocked(/* resetEnteredIds= */ true); } + @GuardedBy("mLock") private void cancelSessionLocked() { if (sVerbose) Log.v(TAG, "cancelSessionLocked(): " + getStateAsStringLocked()); @@ -1335,20 +1560,26 @@ public final class AutofillManager { throw e.rethrowFromSystemServer(); } - resetSessionLocked(); + resetSessionLocked(/* resetEnteredIds= */ true); } - private void resetSessionLocked() { + @GuardedBy("mLock") + private void resetSessionLocked(boolean resetEnteredIds) { mSessionId = NO_SESSION; mState = STATE_UNKNOWN; mTrackedViews = null; mFillableIds = null; mSaveTriggerId = null; + mIdShownFillUi = null; + if (resetEnteredIds) { + mEnteredIds = null; + } } + @GuardedBy("mLock") private void updateSessionLocked(AutofillId id, Rect bounds, AutofillValue value, int action, int flags) { - if (sVerbose && action != ACTION_VIEW_EXITED) { + if (sVerbose) { Log.v(TAG, "updateSessionLocked(): id=" + id + ", bounds=" + bounds + ", value=" + value + ", action=" + action + ", flags=" + flags); } @@ -1359,14 +1590,16 @@ public final class AutofillManager { final AutofillClient client = getClient(); if (client == null) return; // NOTE: getClient() already logd it.. - final int newId = mService.updateOrRestartSession(mContext.getActivityToken(), + final int newId = mService.updateOrRestartSession( + client.autofillClientGetActivityToken(), mServiceClient.asBinder(), id, bounds, value, mContext.getUserId(), - mCallback != null, flags, client.getComponentName(), mSessionId, action); + mCallback != null, flags, client.autofillClientGetComponentName(), + mSessionId, action, isCompatibilityModeEnabledLocked()); if (newId != mSessionId) { if (sDebug) Log.d(TAG, "Session restarted: " + mSessionId + "=>" + newId); mSessionId = newId; mState = (mSessionId == NO_SESSION) ? STATE_UNKNOWN : STATE_ACTIVE; - client.autofillCallbackResetableStateAvailable(); + client.autofillClientResetableStateAvailable(); } } else { mService.updateSession(mSessionId, id, bounds, value, action, flags, @@ -1378,6 +1611,7 @@ public final class AutofillManager { } } + @GuardedBy("mLock") private void ensureServiceClientAddedIfNeededLocked() { if (getClient() == null) { return; @@ -1465,9 +1699,10 @@ public final class AutofillManager { AutofillClient client = getClient(); if (client != null) { - if (client.autofillCallbackRequestShowFillUi(anchor, width, height, - anchorBounds, presenter) && mCallback != null) { + if (client.autofillClientRequestShowFillUi(anchor, width, height, + anchorBounds, presenter)) { callback = mCallback; + mIdShownFillUi = id; } } } @@ -1492,7 +1727,25 @@ public final class AutofillManager { // clear mOnInvisibleCalled and we will see if receive onInvisibleForAutofill() // before onAuthenticationResult() mOnInvisibleCalled = false; - client.autofillCallbackAuthenticate(authenticationId, intent, fillInIntent); + client.autofillClientAuthenticate(authenticationId, intent, fillInIntent); + } + } + } + } + + private void dispatchUnhandledKey(int sessionId, AutofillId id, KeyEvent keyEvent) { + final View anchor = findView(id); + if (anchor == null) { + return; + } + + AutofillCallback callback = null; + synchronized (mLock) { + if (mSessionId == sessionId) { + AutofillClient client = getClient(); + + if (client != null) { + client.autofillClientDispatchUnhandledKey(anchor, keyEvent); } } } @@ -1515,7 +1768,7 @@ public final class AutofillManager { mEnabled = (flags & SET_STATE_FLAG_ENABLED) != 0; if (!mEnabled || (flags & SET_STATE_FLAG_RESET_SESSION) != 0) { // Reset the session state - resetSessionLocked(); + resetSessionLocked(/* resetEnteredIds= */ true); } if ((flags & SET_STATE_FLAG_RESET_CLIENT) != 0) { // Reset connection to system @@ -1543,7 +1796,7 @@ public final class AutofillManager { if (mLastAutofilledData == null) { mLastAutofilledData = new ParcelableMap(1); } - mLastAutofilledData.put(getAutofillId(view), targetValue); + mLastAutofilledData.put(view.getAutofillId(), targetValue); } view.setAutofilled(true); } @@ -1563,7 +1816,10 @@ public final class AutofillManager { final int itemCount = ids.size(); int numApplied = 0; ArrayMap<View, SparseArray<AutofillValue>> virtualValues = null; - final View[] views = client.findViewsByAutofillIdTraversal(getViewIds(ids)); + final View[] views = client.autofillClientFindViewsByAutofillIdTraversal( + Helper.toArray(ids)); + + ArrayList<AutofillId> failedIds = null; for (int i = 0; i < itemCount; i++) { final AutofillId id = ids.get(i); @@ -1571,7 +1827,14 @@ public final class AutofillManager { final int viewId = id.getViewId(); final View view = views[i]; if (view == null) { - Log.w(TAG, "autofill(): no View with id " + viewId); + // Most likely view has been removed after the initial request was sent to the + // the service; this is fine, but we need to update the view status in the + // server side so it can be triggered again. + Log.d(TAG, "autofill(): no View with id " + id); + if (failedIds == null) { + failedIds = new ArrayList<>(); + } + failedIds.add(id); continue; } if (id.isVirtual()) { @@ -1605,12 +1868,28 @@ public final class AutofillManager { } } + if (failedIds != null) { + if (sVerbose) { + Log.v(TAG, "autofill(): total failed views: " + failedIds); + } + try { + mService.setAutofillFailure(mSessionId, failedIds, mContext.getUserId()); + } catch (RemoteException e) { + // In theory, we could ignore this error since it's not a big deal, but + // in reality, we rather crash the app anyways, as the failure could be + // a consequence of something going wrong on the server side... + e.rethrowFromSystemServer(); + } + } + if (virtualValues != null) { for (int i = 0; i < virtualValues.size(); i++) { final View parent = virtualValues.keyAt(i); final SparseArray<AutofillValue> childrenValues = virtualValues.valueAt(i); parent.autofill(childrenValues); numApplied += childrenValues.size(); + // TODO: we should provide a callback so the parent can call failures; something + // like notifyAutofillFailed(View view, int[] childrenIds); } } @@ -1703,22 +1982,44 @@ public final class AutofillManager { * Marks the state of the session as finished. * * @param newState {@link #STATE_FINISHED} (because the autofill service returned a {@code null} - * FillResponse), {@link #STATE_UNKNOWN} (because the session was removed), or - * {@link #STATE_DISABLED_BY_SERVICE} (because the autofill service disabled further autofill - * requests for the activity). + * FillResponse), {@link #STATE_UNKNOWN} (because the session was removed), + * {@link #STATE_UNKNOWN_COMPAT_MODE} (beucase the session was finished when the URL bar + * changed on compat mode), or {@link #STATE_DISABLED_BY_SERVICE} (because the autofill service + * disabled further autofill requests for the activity). */ private void setSessionFinished(int newState) { synchronized (mLock) { - if (sVerbose) Log.v(TAG, "setSessionFinished(): from " + mState + " to " + newState); - resetSessionLocked(); - mState = newState; + if (sVerbose) { + Log.v(TAG, "setSessionFinished(): from " + getStateAsStringLocked() + " to " + + getStateAsString(newState)); + } + if (newState == STATE_UNKNOWN_COMPAT_MODE) { + resetSessionLocked(/* resetEnteredIds= */ true); + mState = STATE_UNKNOWN; + } else { + resetSessionLocked(/* resetEnteredIds= */ false); + mState = newState; + } } } - private void requestHideFillUi(AutofillId id) { - final View anchor = findView(id); + /** @hide */ + public void requestHideFillUi() { + requestHideFillUi(mIdShownFillUi, true); + } + + private void requestHideFillUi(AutofillId id, boolean force) { + final View anchor = id == null ? null : findView(id); if (sVerbose) Log.v(TAG, "requestHideFillUi(" + id + "): anchor = " + anchor); if (anchor == null) { + if (force) { + // When user taps outside autofill window, force to close fill ui even id does + // not match. + AutofillClient client = getClient(); + if (client != null) { + client.autofillClientRequestHideFillUi(); + } + } return; } requestHideFillUi(id, anchor); @@ -1734,7 +2035,8 @@ public final class AutofillManager { // service being uninstalled and the UI being dismissed. AutofillClient client = getClient(); if (client != null) { - if (client.autofillCallbackRequestHideFillUi() && mCallback != null) { + if (client.autofillClientRequestHideFillUi()) { + mIdShownFillUi = null; callback = mCallback; } } @@ -1783,35 +2085,6 @@ public final class AutofillManager { } /** - * Get an array of viewIds from a List of {@link AutofillId}. - * - * @param autofillIds The autofill ids to convert - * - * @return The array of viewIds. - */ - // TODO: move to Helper as static method - @NonNull private int[] getViewIds(@NonNull AutofillId[] autofillIds) { - final int numIds = autofillIds.length; - final int[] viewIds = new int[numIds]; - for (int i = 0; i < numIds; i++) { - viewIds[i] = autofillIds[i].getViewId(); - } - - return viewIds; - } - - // TODO: move to Helper as static method - @NonNull private int[] getViewIds(@NonNull List<AutofillId> autofillIds) { - final int numIds = autofillIds.size(); - final int[] viewIds = new int[numIds]; - for (int i = 0; i < numIds; i++) { - viewIds[i] = autofillIds.get(i).getViewId(); - } - - return viewIds; - } - - /** * Find a single view by its id. * * @param autofillId The autofill id of the view @@ -1820,12 +2093,10 @@ public final class AutofillManager { */ private View findView(@NonNull AutofillId autofillId) { final AutofillClient client = getClient(); - - if (client == null) { - return null; + if (client != null) { + return client.autofillClientFindViewByAutofillIdTraversal(autofillId); } - - return client.findViewByAutofillIdTraversal(autofillId.getViewId()); + return null; } /** @hide */ @@ -1869,37 +2140,51 @@ public final class AutofillManager { pw.print(pfx2); pw.print("invisible:"); pw.println(mTrackedViews.mInvisibleTrackedIds); } pw.print(pfx); pw.print("fillable ids: "); pw.println(mFillableIds); + pw.print(pfx); pw.print("entered ids: "); pw.println(mEnteredIds); pw.print(pfx); pw.print("save trigger id: "); pw.println(mSaveTriggerId); pw.print(pfx); pw.print("save on finish(): "); pw.println(mSaveOnFinish); + pw.print(pfx); pw.print("compat mode enabled: "); pw.println( + isCompatibilityModeEnabledLocked()); pw.print(pfx); pw.print("debug: "); pw.print(sDebug); pw.print(" verbose: "); pw.println(sVerbose); } + @GuardedBy("mLock") private String getStateAsStringLocked() { - switch (mState) { + return getStateAsString(mState); + } + + @NonNull + private static String getStateAsString(int state) { + switch (state) { case STATE_UNKNOWN: - return "STATE_UNKNOWN"; + return "UNKNOWN"; case STATE_ACTIVE: - return "STATE_ACTIVE"; + return "ACTIVE"; case STATE_FINISHED: - return "STATE_FINISHED"; + return "FINISHED"; case STATE_SHOWING_SAVE_UI: - return "STATE_SHOWING_SAVE_UI"; + return "SHOWING_SAVE_UI"; case STATE_DISABLED_BY_SERVICE: - return "STATE_DISABLED_BY_SERVICE"; + return "DISABLED_BY_SERVICE"; + case STATE_UNKNOWN_COMPAT_MODE: + return "UNKNOWN_COMPAT_MODE"; default: - return "INVALID:" + mState; + return "INVALID:" + state; } } + @GuardedBy("mLock") private boolean isActiveLocked() { return mState == STATE_ACTIVE; } + @GuardedBy("mLock") private boolean isDisabledByServiceLocked() { return mState == STATE_DISABLED_BY_SERVICE; } + @GuardedBy("mLock") private boolean isFinishedLocked() { return mState == STATE_FINISHED; } @@ -1910,7 +2195,242 @@ public final class AutofillManager { if (sVerbose) Log.v(TAG, "ignoring post() because client is null"); return; } - client.runOnUiThread(runnable); + client.autofillClientRunOnUiThread(runnable); + } + + /** + * Implementation of the accessibility based compatibility. + */ + private final class CompatibilityBridge implements AccessibilityManager.AccessibilityPolicy { + @GuardedBy("mLock") + private final Rect mFocusedBounds = new Rect(); + @GuardedBy("mLock") + private final Rect mTempBounds = new Rect(); + + @GuardedBy("mLock") + private int mFocusedWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + @GuardedBy("mLock") + private long mFocusedNodeId = AccessibilityNodeInfo.UNDEFINED_NODE_ID; + + // Need to report a fake service in case a11y clients check the service list + @NonNull + @GuardedBy("mLock") + AccessibilityServiceInfo mCompatServiceInfo; + + CompatibilityBridge() { + final AccessibilityManager am = AccessibilityManager.getInstance(mContext); + am.setAccessibilityPolicy(this); + } + + private AccessibilityServiceInfo getCompatServiceInfo() { + synchronized (mLock) { + if (mCompatServiceInfo != null) { + return mCompatServiceInfo; + } + final Intent intent = new Intent(); + intent.setComponent(new ComponentName("android", + "com.android.server.autofill.AutofillCompatAccessibilityService")); + final ResolveInfo resolveInfo = mContext.getPackageManager().resolveService( + intent, PackageManager.MATCH_SYSTEM_ONLY | PackageManager.GET_META_DATA); + try { + mCompatServiceInfo = new AccessibilityServiceInfo(resolveInfo, mContext); + } catch (XmlPullParserException | IOException e) { + Log.e(TAG, "Cannot find compat autofill service:" + intent); + throw new IllegalStateException("Cannot find compat autofill service"); + } + return mCompatServiceInfo; + } + } + + @Override + public boolean isEnabled(boolean accessibilityEnabled) { + return true; + } + + @Override + public int getRelevantEventTypes(int relevantEventTypes) { + return relevantEventTypes | AccessibilityEvent.TYPE_VIEW_FOCUSED + | AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED + | AccessibilityEvent.TYPE_VIEW_CLICKED + | AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED; + } + + @Override + public List<AccessibilityServiceInfo> getInstalledAccessibilityServiceList( + List<AccessibilityServiceInfo> installedServices) { + if (installedServices == null) { + installedServices = new ArrayList<>(); + } + installedServices.add(getCompatServiceInfo()); + return installedServices; + } + + @Override + public List<AccessibilityServiceInfo> getEnabledAccessibilityServiceList( + int feedbackTypeFlags, List<AccessibilityServiceInfo> enabledService) { + if (enabledService == null) { + enabledService = new ArrayList<>(); + } + enabledService.add(getCompatServiceInfo()); + return enabledService; + } + + @Override + public AccessibilityEvent onAccessibilityEvent(AccessibilityEvent event, + boolean accessibilityEnabled, int relevantEventTypes) { + switch (event.getEventType()) { + case AccessibilityEvent.TYPE_VIEW_FOCUSED: { + synchronized (mLock) { + if (mFocusedWindowId == event.getWindowId() + && mFocusedNodeId == event.getSourceNodeId()) { + return event; + } + if (mFocusedWindowId != AccessibilityWindowInfo.UNDEFINED_WINDOW_ID + && mFocusedNodeId != AccessibilityNodeInfo.UNDEFINED_NODE_ID) { + notifyViewExited(mFocusedWindowId, mFocusedNodeId); + mFocusedWindowId = AccessibilityWindowInfo.UNDEFINED_WINDOW_ID; + mFocusedNodeId = AccessibilityNodeInfo.UNDEFINED_NODE_ID; + mFocusedBounds.set(0, 0, 0, 0); + } + final int windowId = event.getWindowId(); + final long nodeId = event.getSourceNodeId(); + if (notifyViewEntered(windowId, nodeId, mFocusedBounds)) { + mFocusedWindowId = windowId; + mFocusedNodeId = nodeId; + } + } + } break; + + case AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED: { + synchronized (mLock) { + if (mFocusedWindowId == event.getWindowId() + && mFocusedNodeId == event.getSourceNodeId()) { + notifyValueChanged(event.getWindowId(), event.getSourceNodeId()); + } + } + } break; + + case AccessibilityEvent.TYPE_VIEW_CLICKED: { + synchronized (mLock) { + notifyViewClicked(event.getWindowId(), event.getSourceNodeId()); + } + } break; + + case AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED: { + final AutofillClient client = getClient(); + if (client != null) { + synchronized (mLock) { + if (client.autofillClientIsFillUiShowing()) { + notifyViewEntered(mFocusedWindowId, mFocusedNodeId, mFocusedBounds); + } + updateTrackedViewsLocked(); + } + } + } break; + } + + return accessibilityEnabled ? event : null; + } + + private boolean notifyViewEntered(int windowId, long nodeId, Rect focusedBounds) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId(nodeId); + if (!isVirtualNode(virtualId)) { + return false; + } + final View view = findViewByAccessibilityId(windowId, nodeId); + if (view == null) { + return false; + } + final AccessibilityNodeInfo node = findVirtualNodeByAccessibilityId(view, virtualId); + if (node == null) { + return false; + } + if (!node.isEditable()) { + return false; + } + final Rect newBounds = mTempBounds; + node.getBoundsInScreen(newBounds); + if (newBounds.equals(focusedBounds)) { + return false; + } + focusedBounds.set(newBounds); + AutofillManager.this.notifyViewEntered(view, virtualId, newBounds); + return true; + } + + private void notifyViewExited(int windowId, long nodeId) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId(nodeId); + if (!isVirtualNode(virtualId)) { + return; + } + final View view = findViewByAccessibilityId(windowId, nodeId); + if (view == null) { + return; + } + AutofillManager.this.notifyViewExited(view, virtualId); + } + + private void notifyValueChanged(int windowId, long nodeId) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId(nodeId); + if (!isVirtualNode(virtualId)) { + return; + } + final View view = findViewByAccessibilityId(windowId, nodeId); + if (view == null) { + return; + } + final AccessibilityNodeInfo node = findVirtualNodeByAccessibilityId(view, virtualId); + if (node == null) { + return; + } + AutofillManager.this.notifyValueChanged(view, virtualId, + AutofillValue.forText(node.getText())); + } + + private void notifyViewClicked(int windowId, long nodeId) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId(nodeId); + if (!isVirtualNode(virtualId)) { + return; + } + final View view = findViewByAccessibilityId(windowId, nodeId); + if (view == null) { + return; + } + final AccessibilityNodeInfo node = findVirtualNodeByAccessibilityId(view, virtualId); + if (node == null) { + return; + } + AutofillManager.this.notifyViewClicked(view, virtualId); + } + + @GuardedBy("mLock") + private void updateTrackedViewsLocked() { + if (mTrackedViews != null) { + mTrackedViews.onVisibleForAutofillChangedLocked(); + } + } + + private View findViewByAccessibilityId(int windowId, long nodeId) { + final AutofillClient client = getClient(); + if (client == null) { + return null; + } + final int viewId = AccessibilityNodeInfo.getAccessibilityViewId(nodeId); + return client.autofillClientFindViewByAccessibilityIdTraversal(viewId, windowId); + } + + private AccessibilityNodeInfo findVirtualNodeByAccessibilityId(View view, int virtualId) { + final AccessibilityNodeProvider provider = view.getAccessibilityNodeProvider(); + if (provider == null) { + return null; + } + return provider.createAccessibilityNodeInfo(virtualId); + } + + private boolean isVirtualNode(int nodeId) { + return nodeId != AccessibilityNodeProvider.HOST_VIEW_ID + && nodeId != AccessibilityNodeInfo.UNDEFINED_ITEM_ID; + } } /** @@ -1989,11 +2509,12 @@ public final class AutofillManager { */ TrackedViews(@Nullable AutofillId[] trackedIds) { final AutofillClient client = getClient(); - if (trackedIds != null && client != null) { + if (!ArrayUtils.isEmpty(trackedIds) && client != null) { final boolean[] isVisible; - if (client.isVisibleForAutofill()) { - isVisible = client.getViewVisibility(getViewIds(trackedIds)); + if (client.autofillClientIsVisibleForAutofill()) { + if (sVerbose) Log.v(TAG, "client is visible, check tracked ids"); + isVisible = client.autofillClientGetViewVisibility(trackedIds); } else { // All false isVisible = new boolean[trackedIds.length]; @@ -2012,7 +2533,7 @@ public final class AutofillManager { } if (sVerbose) { - Log.v(TAG, "TrackedViews(trackedIds=" + trackedIds + "): " + Log.v(TAG, "TrackedViews(trackedIds=" + Arrays.toString(trackedIds) + "): " + " mVisibleTrackedIds=" + mVisibleTrackedIds + " mInvisibleTrackedIds=" + mInvisibleTrackedIds); } @@ -2028,9 +2549,10 @@ public final class AutofillManager { * @param id the id of the view/virtual view whose visibility changed. * @param isVisible visible if the view is visible in the view hierarchy. */ + @GuardedBy("mLock") void notifyViewVisibilityChangedLocked(@NonNull AutofillId id, boolean isVisible) { if (sDebug) { - Log.d(TAG, "notifyViewVisibilityChanged(): id=" + id + " isVisible=" + Log.d(TAG, "notifyViewVisibilityChangedLocked(): id=" + id + " isVisible=" + isVisible); } @@ -2059,20 +2581,25 @@ public final class AutofillManager { /** * Called once the client becomes visible. * - * @see AutofillClient#isVisibleForAutofill() + * @see AutofillClient#autofillClientIsVisibleForAutofill() */ - void onVisibleForAutofillLocked() { + @GuardedBy("mLock") + void onVisibleForAutofillChangedLocked() { // The visibility of the views might have changed while the client was not be visible, // hence update the visibility state for all views. AutofillClient client = getClient(); ArraySet<AutofillId> updatedVisibleTrackedIds = null; ArraySet<AutofillId> updatedInvisibleTrackedIds = null; if (client != null) { + if (sVerbose) { + Log.v(TAG, "onVisibleForAutofillChangedLocked(): inv= " + mInvisibleTrackedIds + + " vis=" + mVisibleTrackedIds); + } if (mInvisibleTrackedIds != null) { final ArrayList<AutofillId> orderedInvisibleIds = new ArrayList<>(mInvisibleTrackedIds); - final boolean[] isVisible = client.getViewVisibility( - getViewIds(orderedInvisibleIds)); + final boolean[] isVisible = client.autofillClientGetViewVisibility( + Helper.toArray(orderedInvisibleIds)); final int numInvisibleTrackedIds = orderedInvisibleIds.size(); for (int i = 0; i < numInvisibleTrackedIds; i++) { @@ -2092,8 +2619,8 @@ public final class AutofillManager { if (mVisibleTrackedIds != null) { final ArrayList<AutofillId> orderedVisibleIds = new ArrayList<>(mVisibleTrackedIds); - final boolean[] isVisible = client.getViewVisibility( - getViewIds(orderedVisibleIds)); + final boolean[] isVisible = client.autofillClientGetViewVisibility( + Helper.toArray(orderedVisibleIds)); final int numVisibleTrackedIds = orderedVisibleIds.size(); for (int i = 0; i < numVisibleTrackedIds; i++) { @@ -2116,6 +2643,9 @@ public final class AutofillManager { } if (mVisibleTrackedIds == null) { + if (sVerbose) { + Log.v(TAG, "onVisibleForAutofillChangedLocked(): no more visible ids"); + } finishSessionLocked(); } } @@ -2232,7 +2762,7 @@ public final class AutofillManager { public void requestHideFillUi(int sessionId, AutofillId id) { final AutofillManager afm = mAfm.get(); if (afm != null) { - afm.post(() -> afm.requestHideFillUi(id)); + afm.post(() -> afm.requestHideFillUi(id, false)); } } @@ -2245,6 +2775,14 @@ public final class AutofillManager { } @Override + public void dispatchUnhandledKey(int sessionId, AutofillId id, KeyEvent fullScreen) { + final AutofillManager afm = mAfm.get(); + if (afm != null) { + afm.post(() -> afm.dispatchUnhandledKey(sessionId, id, fullScreen)); + } + } + + @Override public void startIntentSender(IntentSender intentSender, Intent intent) { final AutofillManager afm = mAfm.get(); if (afm != null) { diff --git a/android/view/autofill/AutofillManagerInternal.java b/android/view/autofill/AutofillManagerInternal.java index fc5d306d..155fe721 100644 --- a/android/view/autofill/AutofillManagerInternal.java +++ b/android/view/autofill/AutofillManagerInternal.java @@ -15,6 +15,9 @@ */ package android.view.autofill; +import android.annotation.NonNull; +import android.annotation.UserIdInt; + /** * Autofill Manager local system service interface. * @@ -26,4 +29,14 @@ public abstract class AutofillManagerInternal { * Notifies the manager that the back key was pressed. */ public abstract void onBackKeyPressed(); + + /** + * Gets whether compatibility mode is enabled for a package + * + * @param packageName The package for which to query. + * @param versionCode The package version code. + * @param userId The user id for which to query. + */ + public abstract boolean isCompatibilityModeRequested(@NonNull String packageName, + long versionCode, @UserIdInt int userId); } diff --git a/android/view/autofill/AutofillPopupWindow.java b/android/view/autofill/AutofillPopupWindow.java index e80fdd93..1da998d0 100644 --- a/android/view/autofill/AutofillPopupWindow.java +++ b/android/view/autofill/AutofillPopupWindow.java @@ -46,6 +46,7 @@ public class AutofillPopupWindow extends PopupWindow { private final WindowPresenter mWindowPresenter; private WindowManager.LayoutParams mWindowLayoutParams; + private boolean mFullScreen; private final View.OnAttachStateChangeListener mOnAttachStateChangeListener = new View.OnAttachStateChangeListener() { @@ -104,12 +105,17 @@ public class AutofillPopupWindow extends PopupWindow { */ public void update(View anchor, int offsetX, int offsetY, int width, int height, Rect virtualBounds) { + mFullScreen = width == LayoutParams.MATCH_PARENT && height == LayoutParams.MATCH_PARENT; // If we are showing the popup for a virtual view we use a fake view which // delegates to the anchor but present itself with the same bounds as the // virtual view. This ensures that the location logic in popup works // symmetrically when the dropdown is below and above the anchor. final View actualAnchor; - if (virtualBounds != null) { + if (mFullScreen) { + offsetX = 0; + offsetY = 0; + actualAnchor = anchor; + } else if (virtualBounds != null) { final int[] mLocationOnScreen = new int[] {virtualBounds.left, virtualBounds.top}; actualAnchor = new View(anchor.getContext()) { @Override @@ -209,6 +215,17 @@ public class AutofillPopupWindow extends PopupWindow { } @Override + protected boolean findDropDownPosition(View anchor, LayoutParams outParams, + int xOffset, int yOffset, int width, int height, int gravity, boolean allowScroll) { + if (mFullScreen) { + // Do not patch LayoutParams if force full screen + return false; + } + return super.findDropDownPosition(anchor, outParams, xOffset, yOffset, + width, height, gravity, allowScroll); + } + + @Override public void showAsDropDown(View anchor, int xoff, int yoff, int gravity) { if (sVerbose) { Log.v(TAG, "showAsDropDown(): anchor=" + anchor + ", xoff=" + xoff + ", yoff=" + yoff diff --git a/android/view/autofill/Helper.java b/android/view/autofill/Helper.java index 4b2c53c7..48d0dbb3 100644 --- a/android/view/autofill/Helper.java +++ b/android/view/autofill/Helper.java @@ -19,6 +19,8 @@ package android.view.autofill; import android.annotation.NonNull; import android.annotation.Nullable; +import java.util.Collection; + /** @hide */ public final class Helper { @@ -59,6 +61,20 @@ public final class Helper { builder.append(" ]"); } + /** + * Convers a collaction of {@link AutofillId AutofillIds} to an array. + * @param collection The collection. + * @return The array. + */ + public static @NonNull AutofillId[] toArray(Collection<AutofillId> collection) { + if (collection == null) { + return new AutofillId[0]; + } + final AutofillId[] array = new AutofillId[collection.size()]; + collection.toArray(array); + return array; + } + private Helper() { throw new UnsupportedOperationException("contains static members only"); } diff --git a/android/view/inputmethod/InputConnection.java b/android/view/inputmethod/InputConnection.java index eba91763..e5545405 100644 --- a/android/view/inputmethod/InputConnection.java +++ b/android/view/inputmethod/InputConnection.java @@ -21,7 +21,6 @@ import android.annotation.Nullable; import android.inputmethodservice.InputMethodService; import android.os.Bundle; import android.os.Handler; -import android.os.LocaleList; import android.view.KeyCharacterMap; import android.view.KeyEvent; @@ -899,37 +898,4 @@ public interface InputConnection { */ boolean commitContent(@NonNull InputContentInfo inputContentInfo, int flags, @Nullable Bundle opts); - - /** - * Called by the input method to tell a hint about the locales of text to be committed. - * - * <p>This is just a hint for editor authors (and the system) to choose better options when - * they have to disambiguate languages, like editor authors can do for input methods with - * {@link EditorInfo#hintLocales}.</p> - * - * <p>The language hint provided by this callback should have higher priority than - * {@link InputMethodSubtype#getLanguageTag()}, which cannot be updated dynamically.</p> - * - * <p>Note that in general it is discouraged for input method to specify - * {@link android.text.style.LocaleSpan} when inputting text, mainly because of application - * compatibility concerns.</p> - * <ul> - * <li>When an existing text that already has {@link android.text.style.LocaleSpan} is being - * modified by both the input method and application, there is no reliable and easy way to - * keep track of who modified {@link android.text.style.LocaleSpan}. For instance, if the - * text was updated by JavaScript, it it highly likely that span information is completely - * removed, while some input method attempts to preserve spans if possible.</li> - * <li>There is no clear semantics regarding whether {@link android.text.style.LocaleSpan} - * means a weak (ignorable) hint or a strong hint. This becomes more problematic when - * multiple {@link android.text.style.LocaleSpan} instances are specified to the same - * text region, especially when those spans are conflicting.</li> - * </ul> - * @param languageHint list of languages sorted by the priority and/or probability - */ - default void reportLanguageHint(@NonNull LocaleList languageHint) { - // Intentionally empty. - // - // We need to have *some* default implementation for the source compatibility. - // See Bug 72127682 for details. - } } diff --git a/android/view/inputmethod/InputConnectionWrapper.java b/android/view/inputmethod/InputConnectionWrapper.java index cbe6856b..f671e22b 100644 --- a/android/view/inputmethod/InputConnectionWrapper.java +++ b/android/view/inputmethod/InputConnectionWrapper.java @@ -16,10 +16,8 @@ package android.view.inputmethod; -import android.annotation.NonNull; import android.os.Bundle; import android.os.Handler; -import android.os.LocaleList; import android.view.KeyEvent; /** @@ -305,13 +303,4 @@ public class InputConnectionWrapper implements InputConnection { public boolean commitContent(InputContentInfo inputContentInfo, int flags, Bundle opts) { return mTarget.commitContent(inputContentInfo, flags, opts); } - - /** - * {@inheritDoc} - * @throws NullPointerException if the target is {@code null}. - */ - @Override - public void reportLanguageHint(@NonNull LocaleList languageHint) { - mTarget.reportLanguageHint(languageHint); - } } diff --git a/android/view/inputmethod/InputMethodInfo.java b/android/view/inputmethod/InputMethodInfo.java index c69543f6..f0f30a0d 100644 --- a/android/view/inputmethod/InputMethodInfo.java +++ b/android/view/inputmethod/InputMethodInfo.java @@ -261,8 +261,7 @@ public final class InputMethodInfo implements Parcelable { mIsDefaultResId = isDefaultResId; mIsAuxIme = isAuxIme; mSupportsSwitchingToNextInputMethod = supportsSwitchingToNextInputMethod; - // TODO(b/68948291): remove this meta-data before release. - mIsVrOnly = isVrOnly || service.serviceInfo.metaData.getBoolean("isVrOnly", false); + mIsVrOnly = isVrOnly; } InputMethodInfo(Parcel source) { diff --git a/android/view/inputmethod/InputMethodManager.java b/android/view/inputmethod/InputMethodManager.java index 7db5c320..41047281 100644 --- a/android/view/inputmethod/InputMethodManager.java +++ b/android/view/inputmethod/InputMethodManager.java @@ -20,9 +20,12 @@ import static android.Manifest.permission.WRITE_SECURE_SETTINGS; import android.annotation.NonNull; import android.annotation.Nullable; +import android.annotation.RequiresFeature; import android.annotation.RequiresPermission; import android.annotation.SystemService; +import android.annotation.TestApi; import android.content.Context; +import android.content.pm.PackageManager; import android.graphics.Rect; import android.inputmethodservice.InputMethodService; import android.net.Uri; @@ -212,6 +215,7 @@ import java.util.concurrent.TimeUnit; * </ul> */ @SystemService(Context.INPUT_METHOD_SERVICE) +@RequiresFeature(PackageManager.FEATURE_INPUT_METHODS) public final class InputMethodManager { static final boolean DEBUG = false; static final String TAG = "InputMethodManager"; @@ -1178,8 +1182,9 @@ public final class InputMethodManager { } } - /* + /** * This method toggles the input method window display. + * * If the input window is already displayed, it gets hidden. * If not the input window will be displayed. * @param showFlags Provides additional operating flags. May be @@ -1188,7 +1193,6 @@ public final class InputMethodManager { * @param hideFlags Provides additional operating flags. May be * 0 or have the {@link #HIDE_IMPLICIT_ONLY}, * {@link #HIDE_NOT_ALWAYS} bit set. - * @hide */ public void toggleSoftInput(int showFlags, int hideFlags) { if (mCurMethod != null) { @@ -1808,9 +1812,9 @@ public final class InputMethodManager { * when it was started, which allows it to perform this operation on * itself. * @param id The unique identifier for the new input method to be switched to. - * @deprecated Use {@link InputMethodService#setInputMethod(String)} instead. This method - * was intended for IME developers who should be accessing APIs through the service. APIs in - * this class are intended for app developers interacting with the IME. + * @deprecated Use {@link InputMethodService#switchInputMethod(String)} + * instead. This method was intended for IME developers who should be accessing APIs through + * the service. APIs in this class are intended for app developers interacting with the IME. */ @Deprecated public void setInputMethod(IBinder token, String id) { @@ -1837,7 +1841,7 @@ public final class InputMethodManager { * @param id The unique identifier for the new input method to be switched to. * @param subtype The new subtype of the new input method to be switched to. * @deprecated Use - * {@link InputMethodService#setInputMethodAndSubtype(String, InputMethodSubtype)} + * {@link InputMethodService#switchInputMethod(String, InputMethodSubtype)} * instead. This method was intended for IME developers who should be accessing APIs through * the service. APIs in this class are intended for app developers interacting with the IME. */ @@ -2137,6 +2141,26 @@ public final class InputMethodManager { } /** + * A test API for CTS to make sure that {@link #showInputMethodPicker()} works as expected. + * + * <p>When customizing the implementation of {@link #showInputMethodPicker()} API, make sure + * that this test API returns when and only while and only while + * {@link #showInputMethodPicker()} is showing UI. Otherwise your OS implementation may not + * pass CTS.</p> + * + * @return {@code true} while and only while {@link #showInputMethodPicker()} is showing UI. + * @hide + */ + @TestApi + public boolean isInputMethodPickerShown() { + try { + return mService.isInputMethodPickerShownForTest(); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + + /** * Show the settings for enabling subtypes of the specified input method. * @param imiId An input method, whose subtypes settings will be shown. If imiId is null, * subtypes of all input methods will be shown. @@ -2293,22 +2317,22 @@ public final class InputMethodManager { * which allows it to perform this operation on itself. * @return true if the current input method and subtype was successfully switched to the last * used input method and subtype. - * @deprecated Use {@link InputMethodService#switchToLastInputMethod()} instead. This method + * @deprecated Use {@link InputMethodService#switchToPreviousInputMethod()} instead. This method * was intended for IME developers who should be accessing APIs through the service. APIs in * this class are intended for app developers interacting with the IME. */ @Deprecated public boolean switchToLastInputMethod(IBinder imeToken) { - return switchToLastInputMethodInternal(imeToken); + return switchToPreviousInputMethodInternal(imeToken); } /** * @hide */ - public boolean switchToLastInputMethodInternal(IBinder imeToken) { + public boolean switchToPreviousInputMethodInternal(IBinder imeToken) { synchronized (mH) { try { - return mService.switchToLastInputMethod(imeToken); + return mService.switchToPreviousInputMethod(imeToken); } catch (RemoteException e) { throw e.rethrowFromSystemServer(); } diff --git a/android/view/inputmethod/InputMethodManager_Delegate.java b/android/view/inputmethod/InputMethodManager_Delegate.java index 7c98847a..b2a183b5 100644 --- a/android/view/inputmethod/InputMethodManager_Delegate.java +++ b/android/view/inputmethod/InputMethodManager_Delegate.java @@ -16,10 +16,10 @@ package android.view.inputmethod; -import com.android.layoutlib.bridge.android.BridgeIInputMethodManager; +import com.android.internal.view.IInputMethodManager; +import com.android.layoutlib.bridge.util.ReflectionUtils; import com.android.tools.layoutlib.annotations.LayoutlibDelegate; -import android.content.Context; import android.os.Looper; @@ -39,8 +39,8 @@ public class InputMethodManager_Delegate { synchronized (InputMethodManager.class) { InputMethodManager imm = InputMethodManager.peekInstance(); if (imm == null) { - imm = new InputMethodManager( - new BridgeIInputMethodManager(), Looper.getMainLooper()); + imm = new InputMethodManager(ReflectionUtils.createProxy(IInputMethodManager.class), + Looper.getMainLooper()); InputMethodManager.sInstance = imm; } return imm; diff --git a/android/view/textclassifier/DefaultLogger.java b/android/view/textclassifier/DefaultLogger.java new file mode 100644 index 00000000..203ca560 --- /dev/null +++ b/android/view/textclassifier/DefaultLogger.java @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; +import android.content.Context; +import android.metrics.LogMaker; +import android.util.Log; + +import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.logging.MetricsLogger; +import com.android.internal.logging.nano.MetricsProto.MetricsEvent; +import com.android.internal.util.Preconditions; + +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.StringJoiner; + +/** + * Default Logger. + * Used internally by TextClassifierImpl. + * @hide + */ +public final class DefaultLogger extends Logger { + + private static final String LOG_TAG = "DefaultLogger"; + static final String CLASSIFIER_ID = "androidtc"; + + private static final int START_EVENT_DELTA = MetricsEvent.FIELD_SELECTION_SINCE_START; + private static final int PREV_EVENT_DELTA = MetricsEvent.FIELD_SELECTION_SINCE_PREVIOUS; + private static final int INDEX = MetricsEvent.FIELD_SELECTION_SESSION_INDEX; + private static final int WIDGET_TYPE = MetricsEvent.FIELD_SELECTION_WIDGET_TYPE; + private static final int WIDGET_VERSION = MetricsEvent.FIELD_SELECTION_WIDGET_VERSION; + private static final int MODEL_NAME = MetricsEvent.FIELD_TEXTCLASSIFIER_MODEL; + private static final int ENTITY_TYPE = MetricsEvent.FIELD_SELECTION_ENTITY_TYPE; + private static final int SMART_START = MetricsEvent.FIELD_SELECTION_SMART_RANGE_START; + private static final int SMART_END = MetricsEvent.FIELD_SELECTION_SMART_RANGE_END; + private static final int EVENT_START = MetricsEvent.FIELD_SELECTION_RANGE_START; + private static final int EVENT_END = MetricsEvent.FIELD_SELECTION_RANGE_END; + private static final int SESSION_ID = MetricsEvent.FIELD_SELECTION_SESSION_ID; + + private static final String ZERO = "0"; + private static final String UNKNOWN = "unknown"; + + private final MetricsLogger mMetricsLogger; + + public DefaultLogger(@NonNull Config config) { + super(config); + mMetricsLogger = new MetricsLogger(); + } + + @VisibleForTesting + public DefaultLogger(@NonNull Config config, @NonNull MetricsLogger metricsLogger) { + super(config); + mMetricsLogger = Preconditions.checkNotNull(metricsLogger); + } + + @Override + public boolean isSmartSelection(@NonNull String signature) { + return CLASSIFIER_ID.equals(SignatureParser.getClassifierId(signature)); + } + + @Override + public void writeEvent(@NonNull SelectionEvent event) { + Preconditions.checkNotNull(event); + final LogMaker log = new LogMaker(MetricsEvent.TEXT_SELECTION_SESSION) + .setType(getLogType(event)) + .setSubtype(getLogSubType(event)) + .setPackageName(event.getPackageName()) + .addTaggedData(START_EVENT_DELTA, event.getDurationSinceSessionStart()) + .addTaggedData(PREV_EVENT_DELTA, event.getDurationSincePreviousEvent()) + .addTaggedData(INDEX, event.getEventIndex()) + .addTaggedData(WIDGET_TYPE, event.getWidgetType()) + .addTaggedData(WIDGET_VERSION, event.getWidgetVersion()) + .addTaggedData(MODEL_NAME, SignatureParser.getModelName(event.getResultId())) + .addTaggedData(ENTITY_TYPE, event.getEntityType()) + .addTaggedData(SMART_START, event.getSmartStart()) + .addTaggedData(SMART_END, event.getSmartEnd()) + .addTaggedData(EVENT_START, event.getStart()) + .addTaggedData(EVENT_END, event.getEnd()) + .addTaggedData(SESSION_ID, event.getSessionId()); + mMetricsLogger.write(log); + debugLog(log); + } + + private static int getLogType(SelectionEvent event) { + switch (event.getEventType()) { + case SelectionEvent.ACTION_OVERTYPE: + return MetricsEvent.ACTION_TEXT_SELECTION_OVERTYPE; + case SelectionEvent.ACTION_COPY: + return MetricsEvent.ACTION_TEXT_SELECTION_COPY; + case SelectionEvent.ACTION_PASTE: + return MetricsEvent.ACTION_TEXT_SELECTION_PASTE; + case SelectionEvent.ACTION_CUT: + return MetricsEvent.ACTION_TEXT_SELECTION_CUT; + case SelectionEvent.ACTION_SHARE: + return MetricsEvent.ACTION_TEXT_SELECTION_SHARE; + case SelectionEvent.ACTION_SMART_SHARE: + return MetricsEvent.ACTION_TEXT_SELECTION_SMART_SHARE; + case SelectionEvent.ACTION_DRAG: + return MetricsEvent.ACTION_TEXT_SELECTION_DRAG; + case SelectionEvent.ACTION_ABANDON: + return MetricsEvent.ACTION_TEXT_SELECTION_ABANDON; + case SelectionEvent.ACTION_OTHER: + return MetricsEvent.ACTION_TEXT_SELECTION_OTHER; + case SelectionEvent.ACTION_SELECT_ALL: + return MetricsEvent.ACTION_TEXT_SELECTION_SELECT_ALL; + case SelectionEvent.ACTION_RESET: + return MetricsEvent.ACTION_TEXT_SELECTION_RESET; + case SelectionEvent.EVENT_SELECTION_STARTED: + return MetricsEvent.ACTION_TEXT_SELECTION_START; + case SelectionEvent.EVENT_SELECTION_MODIFIED: + return MetricsEvent.ACTION_TEXT_SELECTION_MODIFY; + case SelectionEvent.EVENT_SMART_SELECTION_SINGLE: + return MetricsEvent.ACTION_TEXT_SELECTION_SMART_SINGLE; + case SelectionEvent.EVENT_SMART_SELECTION_MULTI: + return MetricsEvent.ACTION_TEXT_SELECTION_SMART_MULTI; + case SelectionEvent.EVENT_AUTO_SELECTION: + return MetricsEvent.ACTION_TEXT_SELECTION_AUTO; + default: + return MetricsEvent.VIEW_UNKNOWN; + } + } + + private static int getLogSubType(SelectionEvent event) { + switch (event.getInvocationMethod()) { + case SelectionEvent.INVOCATION_MANUAL: + return MetricsEvent.TEXT_SELECTION_INVOCATION_MANUAL; + case SelectionEvent.INVOCATION_LINK: + return MetricsEvent.TEXT_SELECTION_INVOCATION_LINK; + default: + return MetricsEvent.TEXT_SELECTION_INVOCATION_UNKNOWN; + } + } + + private static String getLogTypeString(int logType) { + switch (logType) { + case MetricsEvent.ACTION_TEXT_SELECTION_OVERTYPE: + return "OVERTYPE"; + case MetricsEvent.ACTION_TEXT_SELECTION_COPY: + return "COPY"; + case MetricsEvent.ACTION_TEXT_SELECTION_PASTE: + return "PASTE"; + case MetricsEvent.ACTION_TEXT_SELECTION_CUT: + return "CUT"; + case MetricsEvent.ACTION_TEXT_SELECTION_SHARE: + return "SHARE"; + case MetricsEvent.ACTION_TEXT_SELECTION_SMART_SHARE: + return "SMART_SHARE"; + case MetricsEvent.ACTION_TEXT_SELECTION_DRAG: + return "DRAG"; + case MetricsEvent.ACTION_TEXT_SELECTION_ABANDON: + return "ABANDON"; + case MetricsEvent.ACTION_TEXT_SELECTION_OTHER: + return "OTHER"; + case MetricsEvent.ACTION_TEXT_SELECTION_SELECT_ALL: + return "SELECT_ALL"; + case MetricsEvent.ACTION_TEXT_SELECTION_RESET: + return "RESET"; + case MetricsEvent.ACTION_TEXT_SELECTION_START: + return "SELECTION_STARTED"; + case MetricsEvent.ACTION_TEXT_SELECTION_MODIFY: + return "SELECTION_MODIFIED"; + case MetricsEvent.ACTION_TEXT_SELECTION_SMART_SINGLE: + return "SMART_SELECTION_SINGLE"; + case MetricsEvent.ACTION_TEXT_SELECTION_SMART_MULTI: + return "SMART_SELECTION_MULTI"; + case MetricsEvent.ACTION_TEXT_SELECTION_AUTO: + return "AUTO_SELECTION"; + default: + return UNKNOWN; + } + } + + private static String getLogSubTypeString(int logSubType) { + switch (logSubType) { + case MetricsEvent.TEXT_SELECTION_INVOCATION_MANUAL: + return "MANUAL"; + case MetricsEvent.TEXT_SELECTION_INVOCATION_LINK: + return "LINK"; + default: + return UNKNOWN; + } + } + + private static void debugLog(LogMaker log) { + if (!DEBUG_LOG_ENABLED) return; + + final String widgetType = Objects.toString(log.getTaggedData(WIDGET_TYPE), UNKNOWN); + final String widgetVersion = Objects.toString(log.getTaggedData(WIDGET_VERSION), ""); + final String widget = widgetVersion.isEmpty() + ? widgetType : widgetType + "-" + widgetVersion; + final int index = Integer.parseInt(Objects.toString(log.getTaggedData(INDEX), ZERO)); + if (log.getType() == MetricsEvent.ACTION_TEXT_SELECTION_START) { + String sessionId = Objects.toString(log.getTaggedData(SESSION_ID), ""); + sessionId = sessionId.substring(sessionId.lastIndexOf("-") + 1); + Log.d(LOG_TAG, String.format("New selection session: %s (%s)", widget, sessionId)); + } + + final String model = Objects.toString(log.getTaggedData(MODEL_NAME), UNKNOWN); + final String entity = Objects.toString(log.getTaggedData(ENTITY_TYPE), UNKNOWN); + final String type = getLogTypeString(log.getType()); + final String subType = getLogSubTypeString(log.getSubtype()); + final int smartStart = Integer.parseInt( + Objects.toString(log.getTaggedData(SMART_START), ZERO)); + final int smartEnd = Integer.parseInt( + Objects.toString(log.getTaggedData(SMART_END), ZERO)); + final int eventStart = Integer.parseInt( + Objects.toString(log.getTaggedData(EVENT_START), ZERO)); + final int eventEnd = Integer.parseInt( + Objects.toString(log.getTaggedData(EVENT_END), ZERO)); + + Log.d(LOG_TAG, String.format("%2d: %s/%s/%s, range=%d,%d - smart_range=%d,%d (%s/%s)", + index, type, subType, entity, eventStart, eventEnd, smartStart, smartEnd, widget, + model)); + } + + /** + * Creates a string id that may be used to identify a TextClassifier result. + */ + public static String createId( + String text, int start, int end, Context context, int modelVersion, + List<Locale> locales) { + Preconditions.checkNotNull(text); + Preconditions.checkNotNull(context); + Preconditions.checkNotNull(locales); + final StringJoiner localesJoiner = new StringJoiner(","); + for (Locale locale : locales) { + localesJoiner.add(locale.toLanguageTag()); + } + final String modelName = String.format(Locale.US, "%s_v%d", localesJoiner.toString(), + modelVersion); + final int hash = Objects.hash(text, start, end, context.getPackageName()); + return SignatureParser.createSignature(CLASSIFIER_ID, modelName, hash); + } + + /** + * Helper for creating and parsing string ids for + * {@link android.view.textclassifier.TextClassifierImpl}. + */ + @VisibleForTesting + public static final class SignatureParser { + + static String createSignature(String classifierId, String modelName, int hash) { + return String.format(Locale.US, "%s|%s|%d", classifierId, modelName, hash); + } + + static String getClassifierId(String signature) { + Preconditions.checkNotNull(signature); + final int end = signature.indexOf("|"); + if (end >= 0) { + return signature.substring(0, end); + } + return ""; + } + + static String getModelName(String signature) { + Preconditions.checkNotNull(signature); + final int start = signature.indexOf("|") + 1; + final int end = signature.indexOf("|", start); + if (start >= 1 && end >= start) { + return signature.substring(start, end); + } + return ""; + } + + static int getHash(String signature) { + Preconditions.checkNotNull(signature); + final int index1 = signature.indexOf("|"); + final int index2 = signature.indexOf("|", index1); + if (index2 > 0) { + return Integer.parseInt(signature.substring(index2)); + } + return 0; + } + } +} diff --git a/android/view/textclassifier/GenerateLinksLogger.java b/android/view/textclassifier/GenerateLinksLogger.java new file mode 100644 index 00000000..73cf43b8 --- /dev/null +++ b/android/view/textclassifier/GenerateLinksLogger.java @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.Nullable; +import android.metrics.LogMaker; +import android.util.ArrayMap; +import android.util.Log; + +import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.logging.MetricsLogger; +import com.android.internal.logging.nano.MetricsProto.MetricsEvent; +import com.android.internal.util.Preconditions; + +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.UUID; + +/** + * A helper for logging calls to generateLinks. + * @hide + */ +@VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE) +public final class GenerateLinksLogger { + + private static final String LOG_TAG = "GenerateLinksLogger"; + private static final String ZERO = "0"; + + private final MetricsLogger mMetricsLogger; + private final Random mRng; + private final int mSampleRate; + + /** + * @param sampleRate the rate at which log events are written. (e.g. 100 means there is a 0.01 + * chance that a call to logGenerateLinks results in an event being written). + * To write all events, pass 1. + */ + public GenerateLinksLogger(int sampleRate) { + mSampleRate = sampleRate; + mRng = new Random(System.nanoTime()); + mMetricsLogger = new MetricsLogger(); + } + + @VisibleForTesting + public GenerateLinksLogger(int sampleRate, MetricsLogger metricsLogger) { + mSampleRate = sampleRate; + mRng = new Random(System.nanoTime()); + mMetricsLogger = metricsLogger; + } + + /** Logs statistics about a call to generateLinks. */ + public void logGenerateLinks(CharSequence text, TextLinks links, String callingPackageName, + long latencyMs) { + Preconditions.checkNotNull(text); + Preconditions.checkNotNull(links); + Preconditions.checkNotNull(callingPackageName); + if (!shouldLog()) { + return; + } + + // Always populate the total stats, and per-entity stats for each entity type detected. + final LinkifyStats totalStats = new LinkifyStats(); + final Map<String, LinkifyStats> perEntityTypeStats = new ArrayMap<>(); + for (TextLinks.TextLink link : links.getLinks()) { + if (link.getEntityCount() == 0) continue; + final String entityType = link.getEntity(0); + if (entityType == null + || TextClassifier.TYPE_OTHER.equals(entityType) + || TextClassifier.TYPE_UNKNOWN.equals(entityType)) { + continue; + } + totalStats.countLink(link); + perEntityTypeStats.computeIfAbsent(entityType, k -> new LinkifyStats()).countLink(link); + } + + final String callId = UUID.randomUUID().toString(); + writeStats(callId, callingPackageName, null, totalStats, text, latencyMs); + for (Map.Entry<String, LinkifyStats> entry : perEntityTypeStats.entrySet()) { + writeStats(callId, callingPackageName, entry.getKey(), entry.getValue(), text, + latencyMs); + } + } + + /** + * Returns whether this particular event should be logged. + * + * Sampling is used to reduce the amount of logging data generated. + **/ + private boolean shouldLog() { + if (mSampleRate <= 1) { + return true; + } else { + return mRng.nextInt(mSampleRate) == 0; + } + } + + /** Writes a log event for the given stats. */ + private void writeStats(String callId, String callingPackageName, @Nullable String entityType, + LinkifyStats stats, CharSequence text, long latencyMs) { + final LogMaker log = new LogMaker(MetricsEvent.TEXT_CLASSIFIER_GENERATE_LINKS) + .setPackageName(callingPackageName) + .addTaggedData(MetricsEvent.FIELD_LINKIFY_CALL_ID, callId) + .addTaggedData(MetricsEvent.FIELD_LINKIFY_NUM_LINKS, stats.mNumLinks) + .addTaggedData(MetricsEvent.FIELD_LINKIFY_LINK_LENGTH, stats.mNumLinksTextLength) + .addTaggedData(MetricsEvent.FIELD_LINKIFY_TEXT_LENGTH, text.length()) + .addTaggedData(MetricsEvent.FIELD_LINKIFY_LATENCY, latencyMs); + if (entityType != null) { + log.addTaggedData(MetricsEvent.FIELD_LINKIFY_ENTITY_TYPE, entityType); + } + mMetricsLogger.write(log); + debugLog(log); + } + + private static void debugLog(LogMaker log) { + if (!Logger.DEBUG_LOG_ENABLED) return; + + final String callId = Objects.toString( + log.getTaggedData(MetricsEvent.FIELD_LINKIFY_CALL_ID), ""); + final String entityType = Objects.toString( + log.getTaggedData(MetricsEvent.FIELD_LINKIFY_ENTITY_TYPE), "ANY_ENTITY"); + final int numLinks = Integer.parseInt( + Objects.toString(log.getTaggedData(MetricsEvent.FIELD_LINKIFY_NUM_LINKS), ZERO)); + final int linkLength = Integer.parseInt( + Objects.toString(log.getTaggedData(MetricsEvent.FIELD_LINKIFY_LINK_LENGTH), ZERO)); + final int textLength = Integer.parseInt( + Objects.toString(log.getTaggedData(MetricsEvent.FIELD_LINKIFY_TEXT_LENGTH), ZERO)); + final int latencyMs = Integer.parseInt( + Objects.toString(log.getTaggedData(MetricsEvent.FIELD_LINKIFY_LATENCY), ZERO)); + + Log.d(LOG_TAG, String.format("%s:%s %d links (%d/%d chars) %dms %s", callId, entityType, + numLinks, linkLength, textLength, latencyMs, log.getPackageName())); + } + + /** Helper class for storing per-entity type statistics. */ + private static final class LinkifyStats { + int mNumLinks; + int mNumLinksTextLength; + + void countLink(TextLinks.TextLink link) { + mNumLinks += 1; + mNumLinksTextLength += link.getEnd() - link.getStart(); + } + } +} diff --git a/android/view/textclassifier/LinksInfo.java b/android/view/textclassifier/LinksInfo.java deleted file mode 100644 index 754c9e90..00000000 --- a/android/view/textclassifier/LinksInfo.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.textclassifier; - -import android.annotation.NonNull; - -/** - * Link information that can be applied to text. See: {@link #apply(CharSequence)}. - * Typical implementations of this interface will annotate spannable text with e.g - * {@link android.text.style.ClickableSpan}s or other annotations. - * @hide - */ -public interface LinksInfo { - - /** - * @hide - */ - LinksInfo NO_OP = text -> false; - - /** - * Applies link annotations to the specified text. - * These annotations are not guaranteed to be applied. For example, the annotations may not be - * applied if the text has changed from what it was when the link spec was generated for it. - * - * @return Whether or not the link annotations were successfully applied. - */ - boolean apply(@NonNull CharSequence text); -} diff --git a/android/view/textclassifier/Log.java b/android/view/textclassifier/Log.java index 83ca15df..ef19ee56 100644 --- a/android/view/textclassifier/Log.java +++ b/android/view/textclassifier/Log.java @@ -35,6 +35,10 @@ final class Log { Slog.d(tag, msg); } + public static void w(String tag, String msg) { + Slog.w(tag, msg); + } + public static void e(String tag, String msg, Throwable tr) { if (ENABLE_FULL_LOGGING) { Slog.e(tag, msg, tr); diff --git a/android/view/textclassifier/Logger.java b/android/view/textclassifier/Logger.java new file mode 100644 index 00000000..f03906a0 --- /dev/null +++ b/android/view/textclassifier/Logger.java @@ -0,0 +1,397 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.Context; + +import com.android.internal.util.Preconditions; + +import java.text.BreakIterator; +import java.util.Locale; +import java.util.Objects; + +/** + * A helper for logging TextClassifier related events. + * @hide + */ +public abstract class Logger { + + private static final String LOG_TAG = "Logger"; + /* package */ static final boolean DEBUG_LOG_ENABLED = true; + + private @SelectionEvent.InvocationMethod int mInvocationMethod; + private SelectionEvent mPrevEvent; + private SelectionEvent mSmartEvent; + private SelectionEvent mStartEvent; + + /** + * Logger that does not log anything. + * @hide + */ + public static final Logger DISABLED = new Logger() { + @Override + public void writeEvent(SelectionEvent event) {} + }; + + @Nullable + private final Config mConfig; + + public Logger(Config config) { + mConfig = Preconditions.checkNotNull(config); + } + + private Logger() { + mConfig = null; + } + + /** + * Writes the selection event to a log. + */ + public abstract void writeEvent(@NonNull SelectionEvent event); + + /** + * Returns true if the resultId matches that of a smart selection event (i.e. + * {@link SelectionEvent#EVENT_SMART_SELECTION_SINGLE} or + * {@link SelectionEvent#EVENT_SMART_SELECTION_MULTI}). + * Returns false otherwise. + */ + public boolean isSmartSelection(@NonNull String resultId) { + return false; + } + + /** + * Returns a token iterator for tokenizing text for logging purposes. + */ + public BreakIterator getTokenIterator(@NonNull Locale locale) { + return BreakIterator.getWordInstance(Preconditions.checkNotNull(locale)); + } + + /** + * Logs a "selection started" event. + * + * @param invocationMethod the way the selection was triggered + * @param start the token index of the selected token + */ + public final void logSelectionStartedEvent( + @SelectionEvent.InvocationMethod int invocationMethod, int start) { + if (mConfig == null) { + return; + } + + mInvocationMethod = invocationMethod; + logEvent(new SelectionEvent( + start, start + 1, SelectionEvent.EVENT_SELECTION_STARTED, + TextClassifier.TYPE_UNKNOWN, mInvocationMethod, null, mConfig)); + } + + /** + * Logs a "selection modified" event. + * Use when the user modifies the selection. + * + * @param start the start token (inclusive) index of the selection + * @param end the end token (exclusive) index of the selection + */ + public final void logSelectionModifiedEvent(int start, int end) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + + if (mConfig == null) { + return; + } + + logEvent(new SelectionEvent( + start, end, SelectionEvent.EVENT_SELECTION_MODIFIED, + TextClassifier.TYPE_UNKNOWN, mInvocationMethod, null, mConfig)); + } + + /** + * Logs a "selection modified" event. + * Use when the user modifies the selection and the selection's entity type is known. + * + * @param start the start token (inclusive) index of the selection + * @param end the end token (exclusive) index of the selection + * @param classification the TextClassification object returned by the TextClassifier that + * classified the selected text + */ + public final void logSelectionModifiedEvent( + int start, int end, @NonNull TextClassification classification) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + Preconditions.checkNotNull(classification); + + if (mConfig == null) { + return; + } + + final String entityType = classification.getEntityCount() > 0 + ? classification.getEntity(0) + : TextClassifier.TYPE_UNKNOWN; + logEvent(new SelectionEvent( + start, end, SelectionEvent.EVENT_SELECTION_MODIFIED, + entityType, mInvocationMethod, classification.getId(), mConfig)); + } + + /** + * Logs a "selection modified" event. + * Use when a TextClassifier modifies the selection. + * + * @param start the start token (inclusive) index of the selection + * @param end the end token (exclusive) index of the selection + * @param selection the TextSelection object returned by the TextClassifier for the + * specified selection + */ + public final void logSelectionModifiedEvent( + int start, int end, @NonNull TextSelection selection) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + Preconditions.checkNotNull(selection); + + if (mConfig == null) { + return; + } + + final int eventType; + if (isSmartSelection(selection.getId())) { + eventType = end - start > 1 + ? SelectionEvent.EVENT_SMART_SELECTION_MULTI + : SelectionEvent.EVENT_SMART_SELECTION_SINGLE; + + } else { + eventType = SelectionEvent.EVENT_AUTO_SELECTION; + } + final String entityType = selection.getEntityCount() > 0 + ? selection.getEntity(0) + : TextClassifier.TYPE_UNKNOWN; + logEvent(new SelectionEvent(start, end, eventType, entityType, mInvocationMethod, + selection.getId(), mConfig)); + } + + /** + * Logs an event specifying an action taken on a selection. + * Use when the user clicks on an action to act on the selected text. + * + * @param start the start token (inclusive) index of the selection + * @param end the end token (exclusive) index of the selection + * @param actionType the action that was performed on the selection + */ + public final void logSelectionActionEvent( + int start, int end, @SelectionEvent.ActionType int actionType) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + checkActionType(actionType); + + if (mConfig == null) { + return; + } + + logEvent(new SelectionEvent( + start, end, actionType, TextClassifier.TYPE_UNKNOWN, mInvocationMethod, + null, mConfig)); + } + + /** + * Logs an event specifying an action taken on a selection. + * Use when the user clicks on an action to act on the selected text and the selection's + * entity type is known. + * + * @param start the start token (inclusive) index of the selection + * @param end the end token (exclusive) index of the selection + * @param actionType the action that was performed on the selection + * @param classification the TextClassification object returned by the TextClassifier that + * classified the selected text + * + * @throws IllegalArgumentException If actionType is not a valid SelectionEvent actionType + */ + public final void logSelectionActionEvent( + int start, int end, @SelectionEvent.ActionType int actionType, + @NonNull TextClassification classification) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + Preconditions.checkNotNull(classification); + checkActionType(actionType); + + if (mConfig == null) { + return; + } + + final String entityType = classification.getEntityCount() > 0 + ? classification.getEntity(0) + : TextClassifier.TYPE_UNKNOWN; + logEvent(new SelectionEvent(start, end, actionType, entityType, mInvocationMethod, + classification.getId(), mConfig)); + } + + private void logEvent(@NonNull SelectionEvent event) { + Preconditions.checkNotNull(event); + + if (event.getEventType() != SelectionEvent.EVENT_SELECTION_STARTED + && mStartEvent == null) { + if (DEBUG_LOG_ENABLED) { + Log.d(LOG_TAG, "Selection session not yet started. Ignoring event"); + } + return; + } + + final long now = System.currentTimeMillis(); + switch (event.getEventType()) { + case SelectionEvent.EVENT_SELECTION_STARTED: + Preconditions.checkArgument(event.getAbsoluteEnd() == event.getAbsoluteStart() + 1); + event.setSessionId(startNewSession()); + mStartEvent = event; + break; + case SelectionEvent.EVENT_SMART_SELECTION_SINGLE: // fall through + case SelectionEvent.EVENT_SMART_SELECTION_MULTI: + mSmartEvent = event; + break; + case SelectionEvent.EVENT_SELECTION_MODIFIED: // fall through + case SelectionEvent.EVENT_AUTO_SELECTION: + if (mPrevEvent != null + && mPrevEvent.getAbsoluteStart() == event.getAbsoluteStart() + && mPrevEvent.getAbsoluteEnd() == event.getAbsoluteEnd()) { + // Selection did not change. Ignore event. + return; + } + break; + default: + // do nothing. + } + + event.setEventTime(now); + if (mStartEvent != null) { + event.setSessionId(mStartEvent.getSessionId()) + .setDurationSinceSessionStart(now - mStartEvent.getEventTime()) + .setStart(event.getAbsoluteStart() - mStartEvent.getAbsoluteStart()) + .setEnd(event.getAbsoluteEnd() - mStartEvent.getAbsoluteStart()); + } + if (mSmartEvent != null) { + event.setResultId(mSmartEvent.getResultId()) + .setSmartStart(mSmartEvent.getAbsoluteStart() - mStartEvent.getAbsoluteStart()) + .setSmartEnd(mSmartEvent.getAbsoluteEnd() - mStartEvent.getAbsoluteStart()); + } + if (mPrevEvent != null) { + event.setDurationSincePreviousEvent(now - mPrevEvent.getEventTime()) + .setEventIndex(mPrevEvent.getEventIndex() + 1); + } + writeEvent(event); + mPrevEvent = event; + + if (event.isTerminal()) { + endSession(); + } + } + + private TextClassificationSessionId startNewSession() { + endSession(); + return new TextClassificationSessionId(); + } + + private void endSession() { + mPrevEvent = null; + mSmartEvent = null; + mStartEvent = null; + } + + /** + * @throws IllegalArgumentException If eventType is not an {@link SelectionEvent.ActionType} + */ + private static void checkActionType(@SelectionEvent.EventType int eventType) + throws IllegalArgumentException { + switch (eventType) { + case SelectionEvent.ACTION_OVERTYPE: // fall through + case SelectionEvent.ACTION_COPY: // fall through + case SelectionEvent.ACTION_PASTE: // fall through + case SelectionEvent.ACTION_CUT: // fall through + case SelectionEvent.ACTION_SHARE: // fall through + case SelectionEvent.ACTION_SMART_SHARE: // fall through + case SelectionEvent.ACTION_DRAG: // fall through + case SelectionEvent.ACTION_ABANDON: // fall through + case SelectionEvent.ACTION_SELECT_ALL: // fall through + case SelectionEvent.ACTION_RESET: // fall through + return; + default: + throw new IllegalArgumentException( + String.format(Locale.US, "%d is not an eventType", eventType)); + } + } + + + /** + * A Logger config. + */ + public static final class Config { + + private final String mPackageName; + private final String mWidgetType; + @Nullable private final String mWidgetVersion; + + /** + * @param context Context of the widget the logger logs for + * @param widgetType a name for the widget being logged for. e.g. + * {@link TextClassifier#WIDGET_TYPE_TEXTVIEW} + * @param widgetVersion a string version info for the widget the logger logs for + */ + public Config( + @NonNull Context context, + @TextClassifier.WidgetType String widgetType, + @Nullable String widgetVersion) { + mPackageName = Preconditions.checkNotNull(context).getPackageName(); + mWidgetType = widgetType; + mWidgetVersion = widgetVersion; + } + + /** + * Returns the package name of the application the logger logs for. + */ + public String getPackageName() { + return mPackageName; + } + + /** + * Returns the name for the widget being logged for. e.g. + * {@link TextClassifier#WIDGET_TYPE_TEXTVIEW}. + */ + public String getWidgetType() { + return mWidgetType; + } + + /** + * Returns string version info for the logger. This is specific to the text classifier. + */ + @Nullable + public String getWidgetVersion() { + return mWidgetVersion; + } + + @Override + public int hashCode() { + return Objects.hash(mPackageName, mWidgetType, mWidgetVersion); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (!(obj instanceof Config)) { + return false; + } + + final Config other = (Config) obj; + return Objects.equals(mPackageName, other.mPackageName) + && Objects.equals(mWidgetType, other.mWidgetType) + && Objects.equals(mWidgetVersion, other.mWidgetType); + } + } +} diff --git a/android/view/textclassifier/SelectionEvent.java b/android/view/textclassifier/SelectionEvent.java new file mode 100644 index 00000000..1e978ccf --- /dev/null +++ b/android/view/textclassifier/SelectionEvent.java @@ -0,0 +1,670 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.IntDef; +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.os.Parcel; +import android.os.Parcelable; +import android.view.textclassifier.TextClassifier.EntityType; +import android.view.textclassifier.TextClassifier.WidgetType; + +import com.android.internal.util.Preconditions; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.util.Locale; +import java.util.Objects; + +/** + * A selection event. + * Specify index parameters as word token indices. + */ +public final class SelectionEvent implements Parcelable { + + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef({ACTION_OVERTYPE, ACTION_COPY, ACTION_PASTE, ACTION_CUT, + ACTION_SHARE, ACTION_SMART_SHARE, ACTION_DRAG, ACTION_ABANDON, + ACTION_OTHER, ACTION_SELECT_ALL, ACTION_RESET}) + // NOTE: ActionType values should not be lower than 100 to avoid colliding with the other + // EventTypes declared below. + public @interface ActionType { + /* + * Terminal event types range: [100,200). + * Non-terminal event types range: [200,300). + */ + } + + /** User typed over the selection. */ + public static final int ACTION_OVERTYPE = 100; + /** User copied the selection. */ + public static final int ACTION_COPY = 101; + /** User pasted over the selection. */ + public static final int ACTION_PASTE = 102; + /** User cut the selection. */ + public static final int ACTION_CUT = 103; + /** User shared the selection. */ + public static final int ACTION_SHARE = 104; + /** User clicked the textAssist menu item. */ + public static final int ACTION_SMART_SHARE = 105; + /** User dragged+dropped the selection. */ + public static final int ACTION_DRAG = 106; + /** User abandoned the selection. */ + public static final int ACTION_ABANDON = 107; + /** User performed an action on the selection. */ + public static final int ACTION_OTHER = 108; + + // Non-terminal actions. + /** User activated Select All */ + public static final int ACTION_SELECT_ALL = 200; + /** User reset the smart selection. */ + public static final int ACTION_RESET = 201; + + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef({ACTION_OVERTYPE, ACTION_COPY, ACTION_PASTE, ACTION_CUT, + ACTION_SHARE, ACTION_SMART_SHARE, ACTION_DRAG, ACTION_ABANDON, + ACTION_OTHER, ACTION_SELECT_ALL, ACTION_RESET, + EVENT_SELECTION_STARTED, EVENT_SELECTION_MODIFIED, + EVENT_SMART_SELECTION_SINGLE, EVENT_SMART_SELECTION_MULTI, + EVENT_AUTO_SELECTION}) + // NOTE: EventTypes declared here must be less than 100 to avoid colliding with the + // ActionTypes declared above. + public @interface EventType { + /* + * Range: 1 -> 99. + */ + } + + /** User started a new selection. */ + public static final int EVENT_SELECTION_STARTED = 1; + /** User modified an existing selection. */ + public static final int EVENT_SELECTION_MODIFIED = 2; + /** Smart selection triggered for a single token (word). */ + public static final int EVENT_SMART_SELECTION_SINGLE = 3; + /** Smart selection triggered spanning multiple tokens (words). */ + public static final int EVENT_SMART_SELECTION_MULTI = 4; + /** Something else other than User or the default TextClassifier triggered a selection. */ + public static final int EVENT_AUTO_SELECTION = 5; + + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef({INVOCATION_MANUAL, INVOCATION_LINK, INVOCATION_UNKNOWN}) + public @interface InvocationMethod {} + + /** Selection was invoked by the user long pressing, double tapping, or dragging to select. */ + public static final int INVOCATION_MANUAL = 1; + /** Selection was invoked by the user tapping on a link. */ + public static final int INVOCATION_LINK = 2; + /** Unknown invocation method */ + public static final int INVOCATION_UNKNOWN = 0; + + private static final String NO_SIGNATURE = ""; + + private final int mAbsoluteStart; + private final int mAbsoluteEnd; + private final @EntityType String mEntityType; + + private @EventType int mEventType; + private String mPackageName = ""; + private String mWidgetType = TextClassifier.WIDGET_TYPE_UNKNOWN; + private @InvocationMethod int mInvocationMethod; + @Nullable private String mWidgetVersion; + @Nullable private String mResultId; + private long mEventTime; + private long mDurationSinceSessionStart; + private long mDurationSincePreviousEvent; + private int mEventIndex; + @Nullable private TextClassificationSessionId mSessionId; + private int mStart; + private int mEnd; + private int mSmartStart; + private int mSmartEnd; + + SelectionEvent( + int start, int end, + @EventType int eventType, @EntityType String entityType, + @InvocationMethod int invocationMethod, @Nullable String resultId) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + mAbsoluteStart = start; + mAbsoluteEnd = end; + mEventType = eventType; + mEntityType = Preconditions.checkNotNull(entityType); + mResultId = resultId; + mInvocationMethod = invocationMethod; + } + + SelectionEvent( + int start, int end, + @EventType int eventType, @EntityType String entityType, + @InvocationMethod int invocationMethod, @Nullable String resultId, + Logger.Config config) { + this(start, end, eventType, entityType, invocationMethod, resultId); + Preconditions.checkNotNull(config); + setTextClassificationSessionContext( + new TextClassificationContext.Builder( + config.getPackageName(), config.getWidgetType()) + .setWidgetVersion(config.getWidgetVersion()) + .build()); + } + + private SelectionEvent(Parcel in) { + mAbsoluteStart = in.readInt(); + mAbsoluteEnd = in.readInt(); + mEventType = in.readInt(); + mEntityType = in.readString(); + mWidgetVersion = in.readInt() > 0 ? in.readString() : null; + mPackageName = in.readString(); + mWidgetType = in.readString(); + mInvocationMethod = in.readInt(); + mResultId = in.readString(); + mEventTime = in.readLong(); + mDurationSinceSessionStart = in.readLong(); + mDurationSincePreviousEvent = in.readLong(); + mEventIndex = in.readInt(); + mSessionId = in.readInt() > 0 + ? TextClassificationSessionId.CREATOR.createFromParcel(in) : null; + mStart = in.readInt(); + mEnd = in.readInt(); + mSmartStart = in.readInt(); + mSmartEnd = in.readInt(); + } + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeInt(mAbsoluteStart); + dest.writeInt(mAbsoluteEnd); + dest.writeInt(mEventType); + dest.writeString(mEntityType); + dest.writeInt(mWidgetVersion != null ? 1 : 0); + if (mWidgetVersion != null) { + dest.writeString(mWidgetVersion); + } + dest.writeString(mPackageName); + dest.writeString(mWidgetType); + dest.writeInt(mInvocationMethod); + dest.writeString(mResultId); + dest.writeLong(mEventTime); + dest.writeLong(mDurationSinceSessionStart); + dest.writeLong(mDurationSincePreviousEvent); + dest.writeInt(mEventIndex); + dest.writeInt(mSessionId != null ? 1 : 0); + if (mSessionId != null) { + mSessionId.writeToParcel(dest, flags); + } + dest.writeInt(mStart); + dest.writeInt(mEnd); + dest.writeInt(mSmartStart); + dest.writeInt(mSmartEnd); + } + + @Override + public int describeContents() { + return 0; + } + + /** + * Creates a "selection started" event. + * + * @param invocationMethod the way the selection was triggered + * @param start the index of the selected text + */ + @NonNull + public static SelectionEvent createSelectionStartedEvent( + @SelectionEvent.InvocationMethod int invocationMethod, int start) { + return new SelectionEvent( + start, start + 1, SelectionEvent.EVENT_SELECTION_STARTED, + TextClassifier.TYPE_UNKNOWN, invocationMethod, NO_SIGNATURE); + } + + /** + * Creates a "selection modified" event. + * Use when the user modifies the selection. + * + * @param start the start (inclusive) index of the selection + * @param end the end (exclusive) index of the selection + * + * @throws IllegalArgumentException if end is less than start + */ + @NonNull + public static SelectionEvent createSelectionModifiedEvent(int start, int end) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + return new SelectionEvent( + start, end, SelectionEvent.EVENT_SELECTION_MODIFIED, + TextClassifier.TYPE_UNKNOWN, INVOCATION_UNKNOWN, NO_SIGNATURE); + } + + /** + * Creates a "selection modified" event. + * Use when the user modifies the selection and the selection's entity type is known. + * + * @param start the start (inclusive) index of the selection + * @param end the end (exclusive) index of the selection + * @param classification the TextClassification object returned by the TextClassifier that + * classified the selected text + * + * @throws IllegalArgumentException if end is less than start + */ + @NonNull + public static SelectionEvent createSelectionModifiedEvent( + int start, int end, @NonNull TextClassification classification) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + Preconditions.checkNotNull(classification); + final String entityType = classification.getEntityCount() > 0 + ? classification.getEntity(0) + : TextClassifier.TYPE_UNKNOWN; + return new SelectionEvent( + start, end, SelectionEvent.EVENT_SELECTION_MODIFIED, + entityType, INVOCATION_UNKNOWN, classification.getId()); + } + + /** + * Creates a "selection modified" event. + * Use when a TextClassifier modifies the selection. + * + * @param start the start (inclusive) index of the selection + * @param end the end (exclusive) index of the selection + * @param selection the TextSelection object returned by the TextClassifier for the + * specified selection + * + * @throws IllegalArgumentException if end is less than start + */ + @NonNull + public static SelectionEvent createSelectionModifiedEvent( + int start, int end, @NonNull TextSelection selection) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + Preconditions.checkNotNull(selection); + final String entityType = selection.getEntityCount() > 0 + ? selection.getEntity(0) + : TextClassifier.TYPE_UNKNOWN; + return new SelectionEvent( + start, end, SelectionEvent.EVENT_AUTO_SELECTION, + entityType, INVOCATION_UNKNOWN, selection.getId()); + } + + /** + * Creates an event specifying an action taken on a selection. + * Use when the user clicks on an action to act on the selected text. + * + * @param start the start (inclusive) index of the selection + * @param end the end (exclusive) index of the selection + * @param actionType the action that was performed on the selection + * + * @throws IllegalArgumentException if end is less than start + */ + @NonNull + public static SelectionEvent createSelectionActionEvent( + int start, int end, @SelectionEvent.ActionType int actionType) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + checkActionType(actionType); + return new SelectionEvent( + start, end, actionType, TextClassifier.TYPE_UNKNOWN, INVOCATION_UNKNOWN, + NO_SIGNATURE); + } + + /** + * Creates an event specifying an action taken on a selection. + * Use when the user clicks on an action to act on the selected text and the selection's + * entity type is known. + * + * @param start the start (inclusive) index of the selection + * @param end the end (exclusive) index of the selection + * @param actionType the action that was performed on the selection + * @param classification the TextClassification object returned by the TextClassifier that + * classified the selected text + * + * @throws IllegalArgumentException if end is less than start + * @throws IllegalArgumentException If actionType is not a valid SelectionEvent actionType + */ + @NonNull + public static SelectionEvent createSelectionActionEvent( + int start, int end, @SelectionEvent.ActionType int actionType, + @NonNull TextClassification classification) { + Preconditions.checkArgument(end >= start, "end cannot be less than start"); + Preconditions.checkNotNull(classification); + checkActionType(actionType); + final String entityType = classification.getEntityCount() > 0 + ? classification.getEntity(0) + : TextClassifier.TYPE_UNKNOWN; + return new SelectionEvent(start, end, actionType, entityType, INVOCATION_UNKNOWN, + classification.getId()); + } + + /** + * @throws IllegalArgumentException If eventType is not an {@link SelectionEvent.ActionType} + */ + private static void checkActionType(@SelectionEvent.EventType int eventType) + throws IllegalArgumentException { + switch (eventType) { + case SelectionEvent.ACTION_OVERTYPE: // fall through + case SelectionEvent.ACTION_COPY: // fall through + case SelectionEvent.ACTION_PASTE: // fall through + case SelectionEvent.ACTION_CUT: // fall through + case SelectionEvent.ACTION_SHARE: // fall through + case SelectionEvent.ACTION_SMART_SHARE: // fall through + case SelectionEvent.ACTION_DRAG: // fall through + case SelectionEvent.ACTION_ABANDON: // fall through + case SelectionEvent.ACTION_SELECT_ALL: // fall through + case SelectionEvent.ACTION_RESET: // fall through + return; + default: + throw new IllegalArgumentException( + String.format(Locale.US, "%d is not an eventType", eventType)); + } + } + + int getAbsoluteStart() { + return mAbsoluteStart; + } + + int getAbsoluteEnd() { + return mAbsoluteEnd; + } + + /** + * Returns the type of event that was triggered. e.g. {@link #ACTION_COPY}. + */ + @EventType + public int getEventType() { + return mEventType; + } + + /** + * Sets the event type. + */ + void setEventType(@EventType int eventType) { + mEventType = eventType; + } + + /** + * Returns the type of entity that is associated with this event. e.g. + * {@link android.view.textclassifier.TextClassifier#TYPE_EMAIL}. + */ + @EntityType + @NonNull + public String getEntityType() { + return mEntityType; + } + + /** + * Returns the package name of the app that this event originated in. + */ + @NonNull + public String getPackageName() { + return mPackageName; + } + + /** + * Returns the type of widget that was involved in triggering this event. + */ + @WidgetType + @NonNull + public String getWidgetType() { + return mWidgetType; + } + + /** + * Returns a string version info for the widget this event was triggered in. + */ + @Nullable + public String getWidgetVersion() { + return mWidgetVersion; + } + + /** + * Sets the {@link TextClassificationContext} for this event. + */ + void setTextClassificationSessionContext(TextClassificationContext context) { + mPackageName = context.getPackageName(); + mWidgetType = context.getWidgetType(); + mWidgetVersion = context.getWidgetVersion(); + } + + /** + * Returns the way the selection mode was invoked. + */ + public @InvocationMethod int getInvocationMethod() { + return mInvocationMethod; + } + + /** + * Sets the invocationMethod for this event. + */ + void setInvocationMethod(@InvocationMethod int invocationMethod) { + mInvocationMethod = invocationMethod; + } + + /** + * Returns the id of the text classifier result associated with this event. + */ + @Nullable + public String getResultId() { + return mResultId; + } + + SelectionEvent setResultId(@Nullable String resultId) { + mResultId = resultId; + return this; + } + + /** + * Returns the time this event was triggered. + */ + public long getEventTime() { + return mEventTime; + } + + SelectionEvent setEventTime(long timeMs) { + mEventTime = timeMs; + return this; + } + + /** + * Returns the duration in ms between when this event was triggered and when the first event in + * the selection session was triggered. + */ + public long getDurationSinceSessionStart() { + return mDurationSinceSessionStart; + } + + SelectionEvent setDurationSinceSessionStart(long durationMs) { + mDurationSinceSessionStart = durationMs; + return this; + } + + /** + * Returns the duration in ms between when this event was triggered and when the previous event + * in the selection session was triggered. + */ + public long getDurationSincePreviousEvent() { + return mDurationSincePreviousEvent; + } + + SelectionEvent setDurationSincePreviousEvent(long durationMs) { + this.mDurationSincePreviousEvent = durationMs; + return this; + } + + /** + * Returns the index (e.g. 1st event, 2nd event, etc.) of this event in the selection session. + */ + public int getEventIndex() { + return mEventIndex; + } + + SelectionEvent setEventIndex(int index) { + mEventIndex = index; + return this; + } + + /** + * Returns the selection session id. + */ + @Nullable + public TextClassificationSessionId getSessionId() { + return mSessionId; + } + + SelectionEvent setSessionId(TextClassificationSessionId id) { + mSessionId = id; + return this; + } + + /** + * Returns the start index of this events relative to the index of the start selection + * event in the selection session. + */ + public int getStart() { + return mStart; + } + + SelectionEvent setStart(int start) { + mStart = start; + return this; + } + + /** + * Returns the end index of this events relative to the index of the start selection + * event in the selection session. + */ + public int getEnd() { + return mEnd; + } + + SelectionEvent setEnd(int end) { + mEnd = end; + return this; + } + + /** + * Returns the start index of this events relative to the index of the smart selection + * event in the selection session. + */ + public int getSmartStart() { + return mSmartStart; + } + + SelectionEvent setSmartStart(int start) { + this.mSmartStart = start; + return this; + } + + /** + * Returns the end index of this events relative to the index of the smart selection + * event in the selection session. + */ + public int getSmartEnd() { + return mSmartEnd; + } + + SelectionEvent setSmartEnd(int end) { + mSmartEnd = end; + return this; + } + + boolean isTerminal() { + return isTerminal(mEventType); + } + + /** + * Returns true if the eventType is a terminal event type. Otherwise returns false. + * A terminal event is an event that ends a selection interaction. + */ + public static boolean isTerminal(@EventType int eventType) { + switch (eventType) { + case ACTION_OVERTYPE: // fall through + case ACTION_COPY: // fall through + case ACTION_PASTE: // fall through + case ACTION_CUT: // fall through + case ACTION_SHARE: // fall through + case ACTION_SMART_SHARE: // fall through + case ACTION_DRAG: // fall through + case ACTION_ABANDON: // fall through + case ACTION_OTHER: // fall through + return true; + default: + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(mAbsoluteStart, mAbsoluteEnd, mEventType, mEntityType, + mWidgetVersion, mPackageName, mWidgetType, mInvocationMethod, mResultId, + mEventTime, mDurationSinceSessionStart, mDurationSincePreviousEvent, + mEventIndex, mSessionId, mStart, mEnd, mSmartStart, mSmartEnd); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SelectionEvent)) { + return false; + } + + final SelectionEvent other = (SelectionEvent) obj; + return mAbsoluteStart == other.mAbsoluteStart + && mAbsoluteEnd == other.mAbsoluteEnd + && mEventType == other.mEventType + && Objects.equals(mEntityType, other.mEntityType) + && Objects.equals(mWidgetVersion, other.mWidgetVersion) + && Objects.equals(mPackageName, other.mPackageName) + && Objects.equals(mWidgetType, other.mWidgetType) + && mInvocationMethod == other.mInvocationMethod + && Objects.equals(mResultId, other.mResultId) + && mEventTime == other.mEventTime + && mDurationSinceSessionStart == other.mDurationSinceSessionStart + && mDurationSincePreviousEvent == other.mDurationSincePreviousEvent + && mEventIndex == other.mEventIndex + && Objects.equals(mSessionId, other.mSessionId) + && mStart == other.mStart + && mEnd == other.mEnd + && mSmartStart == other.mSmartStart + && mSmartEnd == other.mSmartEnd; + } + + @Override + public String toString() { + return String.format(Locale.US, + "SelectionEvent {absoluteStart=%d, absoluteEnd=%d, eventType=%d, entityType=%s, " + + "widgetVersion=%s, packageName=%s, widgetType=%s, invocationMethod=%s, " + + "resultId=%s, eventTime=%d, durationSinceSessionStart=%d, " + + "durationSincePreviousEvent=%d, eventIndex=%d," + + "sessionId=%s, start=%d, end=%d, smartStart=%d, smartEnd=%d}", + mAbsoluteStart, mAbsoluteEnd, mEventType, mEntityType, + mWidgetVersion, mPackageName, mWidgetType, mInvocationMethod, + mResultId, mEventTime, mDurationSinceSessionStart, + mDurationSincePreviousEvent, mEventIndex, + mSessionId, mStart, mEnd, mSmartStart, mSmartEnd); + } + + public static final Creator<SelectionEvent> CREATOR = new Creator<SelectionEvent>() { + @Override + public SelectionEvent createFromParcel(Parcel in) { + return new SelectionEvent(in); + } + + @Override + public SelectionEvent[] newArray(int size) { + return new SelectionEvent[size]; + } + }; +}
\ No newline at end of file diff --git a/android/view/textclassifier/SmartSelection.java b/android/view/textclassifier/SmartSelection.java deleted file mode 100644 index 2c93a19b..00000000 --- a/android/view/textclassifier/SmartSelection.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.textclassifier; - -import android.content.res.AssetFileDescriptor; - -/** - * Java wrapper for SmartSelection native library interface. - * This library is used for detecting entities in text. - */ -final class SmartSelection { - - static { - System.loadLibrary("textclassifier"); - } - - /** Hints the classifier that this may be a url. */ - static final int HINT_FLAG_URL = 0x01; - /** Hints the classifier that this may be an email. */ - static final int HINT_FLAG_EMAIL = 0x02; - - private final long mCtx; - - /** - * Creates a new instance of SmartSelect predictor, using the provided model image, - * given as a file descriptor. - */ - SmartSelection(int fd) { - mCtx = nativeNew(fd); - } - - /** - * Creates a new instance of SmartSelect predictor, using the provided model image, given as a - * file path. - */ - SmartSelection(String path) { - mCtx = nativeNewFromPath(path); - } - - /** - * Creates a new instance of SmartSelect predictor, using the provided model image, given as an - * AssetFileDescriptor. - */ - SmartSelection(AssetFileDescriptor afd) { - mCtx = nativeNewFromAssetFileDescriptor(afd, afd.getStartOffset(), afd.getLength()); - if (mCtx == 0L) { - throw new IllegalArgumentException( - "Couldn't initialize TC from given AssetFileDescriptor"); - } - } - - /** - * Given a string context and current selection, computes the SmartSelection suggestion. - * - * The begin and end are character indices into the context UTF8 string. selectionBegin is the - * character index where the selection begins, and selectionEnd is the index of one character - * past the selection span. - * - * The return value is an array of two ints: suggested selection beginning and end, with the - * same semantics as the input selectionBeginning and selectionEnd. - */ - public int[] suggest(String context, int selectionBegin, int selectionEnd) { - return nativeSuggest(mCtx, context, selectionBegin, selectionEnd); - } - - /** - * Given a string context and current selection, classifies the type of the selected text. - * - * The begin and end params are character indices in the context string. - * - * Returns an array of ClassificationResult objects with the probability - * scores for different collections. - */ - public ClassificationResult[] classifyText( - String context, int selectionBegin, int selectionEnd, int hintFlags) { - return nativeClassifyText(mCtx, context, selectionBegin, selectionEnd, hintFlags); - } - - /** - * Annotates given input text. Every word of the input is a part of some annotation. - * The annotations are sorted by their position in the context string. - * The annotations do not overlap. - */ - public AnnotatedSpan[] annotate(String text) { - return nativeAnnotate(mCtx, text); - } - - /** - * Frees up the allocated memory. - */ - public void close() { - nativeClose(mCtx); - } - - /** - * Returns the language of the model. - */ - public static String getLanguage(int fd) { - return nativeGetLanguage(fd); - } - - /** - * Returns the version of the model. - */ - public static int getVersion(int fd) { - return nativeGetVersion(fd); - } - - private static native long nativeNew(int fd); - - private static native long nativeNewFromPath(String path); - - private static native long nativeNewFromAssetFileDescriptor(AssetFileDescriptor afd, - long offset, long size); - - private static native int[] nativeSuggest( - long context, String text, int selectionBegin, int selectionEnd); - - private static native ClassificationResult[] nativeClassifyText( - long context, String text, int selectionBegin, int selectionEnd, int hintFlags); - - private static native AnnotatedSpan[] nativeAnnotate(long context, String text); - - private static native void nativeClose(long context); - - private static native String nativeGetLanguage(int fd); - - private static native int nativeGetVersion(int fd); - - /** Classification result for classifyText method. */ - static final class ClassificationResult { - final String mCollection; - /** float range: 0 - 1 */ - final float mScore; - - ClassificationResult(String collection, float score) { - mCollection = collection; - mScore = score; - } - } - - /** Represents a result of Annotate call. */ - public static final class AnnotatedSpan { - final int mStartIndex; - final int mEndIndex; - final ClassificationResult[] mClassification; - - AnnotatedSpan(int startIndex, int endIndex, ClassificationResult[] classification) { - mStartIndex = startIndex; - mEndIndex = endIndex; - mClassification = classification; - } - - public int getStartIndex() { - return mStartIndex; - } - - public int getEndIndex() { - return mEndIndex; - } - - public ClassificationResult[] getClassification() { - return mClassification; - } - } -} diff --git a/android/view/textclassifier/SystemTextClassifier.java b/android/view/textclassifier/SystemTextClassifier.java new file mode 100644 index 00000000..45fd6bfb --- /dev/null +++ b/android/view/textclassifier/SystemTextClassifier.java @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.annotation.WorkerThread; +import android.content.Context; +import android.os.Looper; +import android.os.RemoteException; +import android.os.ServiceManager; +import android.service.textclassifier.ITextClassificationCallback; +import android.service.textclassifier.ITextClassifierService; +import android.service.textclassifier.ITextLinksCallback; +import android.service.textclassifier.ITextSelectionCallback; + +import com.android.internal.annotations.GuardedBy; +import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.annotations.VisibleForTesting.Visibility; +import com.android.internal.util.Preconditions; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Proxy to the system's default TextClassifier. + * @hide + */ +@VisibleForTesting(visibility = Visibility.PACKAGE) +public final class SystemTextClassifier implements TextClassifier { + + private static final String LOG_TAG = "SystemTextClassifier"; + + private final ITextClassifierService mManagerService; + private final TextClassificationConstants mSettings; + private final TextClassifier mFallback; + private final String mPackageName; + + private final Object mLoggerLock = new Object(); + @GuardedBy("mLoggerLock") + private Logger.Config mLoggerConfig; + @GuardedBy("mLoggerLock") + private Logger mLogger; + @GuardedBy("mLoggerLock") + private TextClassificationSessionId mSessionId; + + public SystemTextClassifier(Context context, TextClassificationConstants settings) + throws ServiceManager.ServiceNotFoundException { + mManagerService = ITextClassifierService.Stub.asInterface( + ServiceManager.getServiceOrThrow(Context.TEXT_CLASSIFICATION_SERVICE)); + mSettings = Preconditions.checkNotNull(settings); + mFallback = new TextClassifierImpl(context, settings); + mPackageName = Preconditions.checkNotNull(context.getPackageName()); + } + + /** + * @inheritDoc + */ + @Override + @WorkerThread + public TextSelection suggestSelection(TextSelection.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); + try { + final TextSelectionCallback callback = new TextSelectionCallback(); + mManagerService.onSuggestSelection(mSessionId, request, callback); + final TextSelection selection = callback.mReceiver.get(); + if (selection != null) { + return selection; + } + } catch (RemoteException | InterruptedException e) { + Log.e(LOG_TAG, "Error suggesting selection for text. Using fallback.", e); + } + return mFallback.suggestSelection(request); + } + + /** + * @inheritDoc + */ + @Override + @WorkerThread + public TextClassification classifyText(TextClassification.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); + try { + final TextClassificationCallback callback = new TextClassificationCallback(); + mManagerService.onClassifyText(mSessionId, request, callback); + final TextClassification classification = callback.mReceiver.get(); + if (classification != null) { + return classification; + } + } catch (RemoteException | InterruptedException e) { + Log.e(LOG_TAG, "Error classifying text. Using fallback.", e); + } + return mFallback.classifyText(request); + } + + /** + * @inheritDoc + */ + @Override + @WorkerThread + public TextLinks generateLinks(@NonNull TextLinks.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); + + if (!mSettings.isSmartLinkifyEnabled() && request.isLegacyFallback()) { + return Utils.generateLegacyLinks(request); + } + + try { + request.setCallingPackageName(mPackageName); + final TextLinksCallback callback = new TextLinksCallback(); + mManagerService.onGenerateLinks(mSessionId, request, callback); + final TextLinks links = callback.mReceiver.get(); + if (links != null) { + return links; + } + } catch (RemoteException | InterruptedException e) { + Log.e(LOG_TAG, "Error generating links. Using fallback.", e); + } + return mFallback.generateLinks(request); + } + + /** + * @inheritDoc + */ + @Override + @WorkerThread + public int getMaxGenerateLinksTextLength() { + // TODO: retrieve this from the bound service. + return mFallback.getMaxGenerateLinksTextLength(); + } + + @Override + public Logger getLogger(@NonNull Logger.Config config) { + Preconditions.checkNotNull(config); + synchronized (mLoggerLock) { + if (mLogger == null || !config.equals(mLoggerConfig)) { + mLoggerConfig = config; + mLogger = new Logger(config) { + @Override + public void writeEvent(SelectionEvent event) { + try { + mManagerService.onSelectionEvent(mSessionId, event); + } catch (RemoteException e) { + Log.e(LOG_TAG, "Error reporting selection event.", e); + } + } + }; + } + } + return mLogger; + } + + @Override + public void destroy() { + try { + if (mSessionId != null) { + mManagerService.onDestroyTextClassificationSession(mSessionId); + } + } catch (RemoteException e) { + Log.e(LOG_TAG, "Error destroying classification session.", e); + } + } + + /** + * Attempts to initialize a new classification session. + * + * @param classificationContext the classification context + * @param sessionId the session's id + */ + void initializeRemoteSession( + @NonNull TextClassificationContext classificationContext, + @NonNull TextClassificationSessionId sessionId) { + mSessionId = Preconditions.checkNotNull(sessionId); + try { + mManagerService.onCreateTextClassificationSession(classificationContext, mSessionId); + } catch (RemoteException e) { + Log.e(LOG_TAG, "Error starting a new classification session.", e); + } + } + + private static final class TextSelectionCallback extends ITextSelectionCallback.Stub { + + final ResponseReceiver<TextSelection> mReceiver = new ResponseReceiver<>(); + + @Override + public void onSuccess(TextSelection selection) { + mReceiver.onSuccess(selection); + } + + @Override + public void onFailure() { + mReceiver.onFailure(); + } + } + + private static final class TextClassificationCallback extends ITextClassificationCallback.Stub { + + final ResponseReceiver<TextClassification> mReceiver = new ResponseReceiver<>(); + + @Override + public void onSuccess(TextClassification classification) { + mReceiver.onSuccess(classification); + } + + @Override + public void onFailure() { + mReceiver.onFailure(); + } + } + + private static final class TextLinksCallback extends ITextLinksCallback.Stub { + + final ResponseReceiver<TextLinks> mReceiver = new ResponseReceiver<>(); + + @Override + public void onSuccess(TextLinks links) { + mReceiver.onSuccess(links); + } + + @Override + public void onFailure() { + mReceiver.onFailure(); + } + } + + private static final class ResponseReceiver<T> { + + private final CountDownLatch mLatch = new CountDownLatch(1); + + private T mResponse; + + public void onSuccess(T response) { + mResponse = response; + mLatch.countDown(); + } + + public void onFailure() { + Log.e(LOG_TAG, "Request failed.", null); + mLatch.countDown(); + } + + @Nullable + public T get() throws InterruptedException { + // If this is running on the main thread, do not block for a response. + // The response will unfortunately be null and the TextClassifier should depend on its + // fallback. + // NOTE that TextClassifier calls should preferably always be called on a worker thread. + if (Looper.myLooper() != Looper.getMainLooper()) { + mLatch.await(2, TimeUnit.SECONDS); + } + return mResponse; + } + } +} diff --git a/android/view/textclassifier/TextClassification.java b/android/view/textclassifier/TextClassification.java index 7089677d..37a5d9a1 100644 --- a/android/view/textclassifier/TextClassification.java +++ b/android/view/textclassifier/TextClassification.java @@ -17,11 +17,17 @@ package android.view.textclassifier; import android.annotation.FloatRange; +import android.annotation.IntDef; import android.annotation.IntRange; import android.annotation.NonNull; import android.annotation.Nullable; +import android.app.PendingIntent; +import android.app.RemoteAction; import android.content.Context; import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; +import android.content.res.Resources; import android.graphics.Bitmap; import android.graphics.Canvas; import android.graphics.drawable.BitmapDrawable; @@ -32,10 +38,15 @@ import android.os.Parcelable; import android.util.ArrayMap; import android.view.View.OnClickListener; import android.view.textclassifier.TextClassifier.EntityType; +import android.view.textclassifier.TextClassifier.Utils; import com.android.internal.util.Preconditions; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -70,25 +81,16 @@ import java.util.Map; * view.startActionMode(new ActionMode.Callback() { * * public boolean onCreateActionMode(ActionMode mode, Menu menu) { - * // Add the "primary" action. - * if (thisAppHasPermissionToInvokeIntent(classification.getIntent())) { - * menu.add(Menu.NONE, 0, 20, classification.getLabel()) - * .setIcon(classification.getIcon()) - * .setIntent(classification.getIntent()); - * } - * // Add the "secondary" actions. - * for (int i = 0; i < classification.getSecondaryActionsCount(); i++) { - * if (thisAppHasPermissionToInvokeIntent(classification.getSecondaryIntent(i))) { - * menu.add(Menu.NONE, i + 1, 20, classification.getSecondaryLabel(i)) - * .setIcon(classification.getSecondaryIcon(i)) - * .setIntent(classification.getSecondaryIntent(i)); - * } + * for (int i = 0; i < classification.getActions().size(); ++i) { + * RemoteAction action = classification.getActions().get(i); + * menu.add(Menu.NONE, i, 20, action.getTitle()) + * .setIcon(action.getIcon()); * } * return true; * } * * public boolean onActionItemClicked(ActionMode mode, MenuItem item) { - * context.startActivity(item.getIntent()); + * classification.getActions().get(item.getItemId()).getActionIntent().send(); * return true; * } * @@ -96,51 +98,51 @@ import java.util.Map; * }); * }</pre> */ -public final class TextClassification { +public final class TextClassification implements Parcelable { /** * @hide */ static final TextClassification EMPTY = new TextClassification.Builder().build(); + private static final String LOG_TAG = "TextClassification"; // TODO(toki): investigate a way to derive this based on device properties. - private static final int MAX_PRIMARY_ICON_SIZE = 192; - private static final int MAX_SECONDARY_ICON_SIZE = 144; + private static final int MAX_LEGACY_ICON_SIZE = 192; + + @Retention(RetentionPolicy.SOURCE) + @IntDef(value = {IntentType.UNSUPPORTED, IntentType.ACTIVITY, IntentType.SERVICE}) + private @interface IntentType { + int UNSUPPORTED = -1; + int ACTIVITY = 0; + int SERVICE = 1; + } @NonNull private final String mText; - @Nullable private final Drawable mPrimaryIcon; - @Nullable private final String mPrimaryLabel; - @Nullable private final Intent mPrimaryIntent; - @Nullable private final OnClickListener mPrimaryOnClickListener; - @NonNull private final List<Drawable> mSecondaryIcons; - @NonNull private final List<String> mSecondaryLabels; - @NonNull private final List<Intent> mSecondaryIntents; + @Nullable private final Drawable mLegacyIcon; + @Nullable private final String mLegacyLabel; + @Nullable private final Intent mLegacyIntent; + @Nullable private final OnClickListener mLegacyOnClickListener; + @NonNull private final List<RemoteAction> mActions; @NonNull private final EntityConfidence mEntityConfidence; - @NonNull private final String mSignature; + @Nullable private final String mId; private TextClassification( @Nullable String text, - @Nullable Drawable primaryIcon, - @Nullable String primaryLabel, - @Nullable Intent primaryIntent, - @Nullable OnClickListener primaryOnClickListener, - @NonNull List<Drawable> secondaryIcons, - @NonNull List<String> secondaryLabels, - @NonNull List<Intent> secondaryIntents, + @Nullable Drawable legacyIcon, + @Nullable String legacyLabel, + @Nullable Intent legacyIntent, + @Nullable OnClickListener legacyOnClickListener, + @NonNull List<RemoteAction> actions, @NonNull Map<String, Float> entityConfidence, - @NonNull String signature) { - Preconditions.checkArgument(secondaryLabels.size() == secondaryIntents.size()); - Preconditions.checkArgument(secondaryIcons.size() == secondaryIntents.size()); + @Nullable String id) { mText = text; - mPrimaryIcon = primaryIcon; - mPrimaryLabel = primaryLabel; - mPrimaryIntent = primaryIntent; - mPrimaryOnClickListener = primaryOnClickListener; - mSecondaryIcons = secondaryIcons; - mSecondaryLabels = secondaryLabels; - mSecondaryIntents = secondaryIntents; + mLegacyIcon = legacyIcon; + mLegacyLabel = legacyLabel; + mLegacyIntent = legacyIntent; + mLegacyOnClickListener = legacyOnClickListener; + mActions = Collections.unmodifiableList(actions); mEntityConfidence = new EntityConfidence(entityConfidence); - mSignature = signature; + mId = id; } /** @@ -182,181 +184,143 @@ public final class TextClassification { } /** - * Returns the number of <i>secondary</i> actions that are available to act on the classified - * text. - * - * <p><strong>Note: </strong> that there may or may not be a <i>primary</i> action. - * - * @see #getSecondaryIntent(int) - * @see #getSecondaryLabel(int) - * @see #getSecondaryIcon(int) - */ - @IntRange(from = 0) - public int getSecondaryActionsCount() { - return mSecondaryIntents.size(); - } - - /** - * Returns one of the <i>secondary</i> icons that maybe rendered on a widget used to act on the - * classified text. - * - * @param index Index of the action to get the icon for. - * @throws IndexOutOfBoundsException if the specified index is out of range. - * @see #getSecondaryActionsCount() for the number of actions available. - * @see #getSecondaryIntent(int) - * @see #getSecondaryLabel(int) - * @see #getIcon() + * Returns a list of actions that may be performed on the text. The list is ordered based on + * the likelihood that a user will use the action, with the most likely action appearing first. */ - @Nullable - public Drawable getSecondaryIcon(int index) { - return mSecondaryIcons.get(index); + public List<RemoteAction> getActions() { + return mActions; } /** - * Returns an icon for the <i>primary</i> intent that may be rendered on a widget used to act - * on the classified text. + * Returns an icon that may be rendered on a widget used to act on the classified text. * - * @see #getSecondaryIcon(int) + * @deprecated Use {@link #getActions()} instead. */ + @Deprecated @Nullable public Drawable getIcon() { - return mPrimaryIcon; - } - - /** - * Returns one of the <i>secondary</i> labels that may be rendered on a widget used to act on - * the classified text. - * - * @param index Index of the action to get the label for. - * @throws IndexOutOfBoundsException if the specified index is out of range. - * @see #getSecondaryActionsCount() - * @see #getSecondaryIntent(int) - * @see #getSecondaryIcon(int) - * @see #getLabel() - */ - @Nullable - public CharSequence getSecondaryLabel(int index) { - return mSecondaryLabels.get(index); + return mLegacyIcon; } /** - * Returns a label for the <i>primary</i> intent that may be rendered on a widget used to act - * on the classified text. + * Returns a label that may be rendered on a widget used to act on the classified text. * - * @see #getSecondaryLabel(int) + * @deprecated Use {@link #getActions()} instead. */ + @Deprecated @Nullable public CharSequence getLabel() { - return mPrimaryLabel; + return mLegacyLabel; } /** - * Returns one of the <i>secondary</i> intents that may be fired to act on the classified text. + * Returns an intent that may be fired to act on the classified text. * - * @param index Index of the action to get the intent for. - * @throws IndexOutOfBoundsException if the specified index is out of range. - * @see #getSecondaryActionsCount() - * @see #getSecondaryLabel(int) - * @see #getSecondaryIcon(int) - * @see #getIntent() - */ - @Nullable - public Intent getSecondaryIntent(int index) { - return mSecondaryIntents.get(index); - } - - /** - * Returns the <i>primary</i> intent that may be fired to act on the classified text. - * - * @see #getSecondaryIntent(int) + * @deprecated Use {@link #getActions()} instead. */ + @Deprecated @Nullable public Intent getIntent() { - return mPrimaryIntent; + return mLegacyIntent; } /** - * Returns the <i>primary</i> OnClickListener that may be triggered to act on the classified - * text. This field is not parcelable and will be null for all objects read from a parcel. - * Instead, call Context#startActivity(Intent) with the result of #getSecondaryIntent(int). - * Note that this may fail if the activity doesn't have permission to send the intent. + * Returns the OnClickListener that may be triggered to act on the classified text. This field + * is not parcelable and will be null for all objects read from a parcel. Instead, call + * Context#startActivity(Intent) with the result of #getSecondaryIntent(int). Note that this may + * fail if the activity doesn't have permission to send the intent. + * + * @deprecated Use {@link #getActions()} instead. */ @Nullable public OnClickListener getOnClickListener() { - return mPrimaryOnClickListener; + return mLegacyOnClickListener; } /** - * Returns the signature for this object. - * The TextClassifier that generates this object may use it as a way to internally identify - * this object. + * Returns the id, if one exists, for this object. */ - @NonNull - public String getSignature() { - return mSignature; + @Nullable + public String getId() { + return mId; } @Override public String toString() { - return String.format(Locale.US, "TextClassification {" - + "text=%s, entities=%s, " - + "primaryLabel=%s, secondaryLabels=%s, " - + "primaryIntent=%s, secondaryIntents=%s, " - + "signature=%s}", - mText, mEntityConfidence, - mPrimaryLabel, mSecondaryLabels, - mPrimaryIntent, mSecondaryIntents, - mSignature); + return String.format(Locale.US, + "TextClassification {text=%s, entities=%s, actions=%s, id=%s}", + mText, mEntityConfidence, mActions, mId); } - /** Helper for parceling via #ParcelableWrapper. */ - private void writeToParcel(Parcel dest, int flags) { - dest.writeString(mText); - final Bitmap primaryIconBitmap = drawableToBitmap(mPrimaryIcon, MAX_PRIMARY_ICON_SIZE); - dest.writeInt(primaryIconBitmap != null ? 1 : 0); - if (primaryIconBitmap != null) { - primaryIconBitmap.writeToParcel(dest, flags); - } - dest.writeString(mPrimaryLabel); - dest.writeInt(mPrimaryIntent != null ? 1 : 0); - if (mPrimaryIntent != null) { - mPrimaryIntent.writeToParcel(dest, flags); - } - // mPrimaryOnClickListener is not parcelable. - dest.writeTypedList(drawablesToBitmaps(mSecondaryIcons, MAX_SECONDARY_ICON_SIZE)); - dest.writeStringList(mSecondaryLabels); - dest.writeTypedList(mSecondaryIntents); - mEntityConfidence.writeToParcel(dest, flags); - dest.writeString(mSignature); - } - - /** Helper for unparceling via #ParcelableWrapper. */ - private TextClassification(Parcel in) { - mText = in.readString(); - mPrimaryIcon = in.readInt() == 0 - ? null : new BitmapDrawable(null, Bitmap.CREATOR.createFromParcel(in)); - mPrimaryLabel = in.readString(); - mPrimaryIntent = in.readInt() == 0 ? null : Intent.CREATOR.createFromParcel(in); - mPrimaryOnClickListener = null; // not parcelable - mSecondaryIcons = bitmapsToDrawables(in.createTypedArrayList(Bitmap.CREATOR)); - mSecondaryLabels = in.createStringArrayList(); - mSecondaryIntents = in.createTypedArrayList(Intent.CREATOR); - mEntityConfidence = EntityConfidence.CREATOR.createFromParcel(in); - mSignature = in.readString(); + /** + * Creates an OnClickListener that triggers the specified PendingIntent. + * + * @hide + */ + public static OnClickListener createIntentOnClickListener(@NonNull final PendingIntent intent) { + Preconditions.checkNotNull(intent); + return v -> { + try { + intent.send(); + } catch (PendingIntent.CanceledException e) { + Log.e(LOG_TAG, "Error sending PendingIntent", e); + } + }; } /** - * Creates an OnClickListener that starts an activity with the specified intent. + * Creates a PendingIntent for the specified intent. + * Returns null if the intent is not supported for the specified context. * * @throws IllegalArgumentException if context or intent is null * @hide */ - @NonNull - public static OnClickListener createStartActivityOnClickListener( + @Nullable + public static PendingIntent createPendingIntent( @NonNull final Context context, @NonNull final Intent intent) { + switch (getIntentType(intent, context)) { + case IntentType.ACTIVITY: + return PendingIntent.getActivity(context, 0, intent, 0); + case IntentType.SERVICE: + return PendingIntent.getService(context, 0, intent, 0); + default: + return null; + } + } + + @IntentType + private static int getIntentType(@NonNull Intent intent, @NonNull Context context) { Preconditions.checkArgument(context != null); Preconditions.checkArgument(intent != null); - return v -> context.startActivity(intent); + + final ResolveInfo activityRI = context.getPackageManager().resolveActivity(intent, 0); + if (activityRI != null) { + if (context.getPackageName().equals(activityRI.activityInfo.packageName)) { + return IntentType.ACTIVITY; + } + final boolean exported = activityRI.activityInfo.exported; + if (exported && hasPermission(context, activityRI.activityInfo.permission)) { + return IntentType.ACTIVITY; + } + } + + final ResolveInfo serviceRI = context.getPackageManager().resolveService(intent, 0); + if (serviceRI != null) { + if (context.getPackageName().equals(serviceRI.serviceInfo.packageName)) { + return IntentType.SERVICE; + } + final boolean exported = serviceRI.serviceInfo.exported; + if (exported && hasPermission(context, serviceRI.serviceInfo.permission)) { + return IntentType.SERVICE; + } + } + + return IntentType.UNSUPPORTED; + } + + private static boolean hasPermission(@NonNull Context context, @NonNull String permission) { + return permission == null + || context.checkSelfPermission(permission) == PackageManager.PERMISSION_GRANTED; } /** @@ -395,33 +359,6 @@ public final class TextClassification { } /** - * Returns a list of drawables converted to Bitmaps - * - * @param drawables The drawables to convert. - * @param maxDims The maximum edge length of the resulting bitmaps (in pixels). - */ - private static List<Bitmap> drawablesToBitmaps(List<Drawable> drawables, int maxDims) { - final List<Bitmap> bitmaps = new ArrayList<>(drawables.size()); - for (Drawable drawable : drawables) { - bitmaps.add(drawableToBitmap(drawable, maxDims)); - } - return bitmaps; - } - - /** Returns a list of drawable wrappers for a list of bitmaps. */ - private static List<Drawable> bitmapsToDrawables(List<Bitmap> bitmaps) { - final List<Drawable> drawables = new ArrayList<>(bitmaps.size()); - for (Bitmap bitmap : bitmaps) { - if (bitmap != null) { - drawables.add(new BitmapDrawable(null, bitmap)); - } else { - drawables.add(null); - } - } - return drawables; - } - - /** * Builder for building {@link TextClassification} objects. * * <p>e.g. @@ -431,28 +368,26 @@ public final class TextClassification { * .setText(classifiedText) * .setEntityType(TextClassifier.TYPE_EMAIL, 0.9) * .setEntityType(TextClassifier.TYPE_OTHER, 0.1) - * .setPrimaryAction(intent, label, icon) - * .addSecondaryAction(intent1, label1, icon1) - * .addSecondaryAction(intent2, label2, icon2) + * .addAction(remoteAction1) + * .addAction(remoteAction2) * .build(); * }</pre> */ public static final class Builder { @NonNull private String mText; - @NonNull private final List<Drawable> mSecondaryIcons = new ArrayList<>(); - @NonNull private final List<String> mSecondaryLabels = new ArrayList<>(); - @NonNull private final List<Intent> mSecondaryIntents = new ArrayList<>(); + @NonNull private List<RemoteAction> mActions = new ArrayList<>(); @NonNull private final Map<String, Float> mEntityConfidence = new ArrayMap<>(); - @Nullable Drawable mPrimaryIcon; - @Nullable String mPrimaryLabel; - @Nullable Intent mPrimaryIntent; - @Nullable OnClickListener mPrimaryOnClickListener; - @NonNull private String mSignature = ""; + @Nullable Drawable mLegacyIcon; + @Nullable String mLegacyLabel; + @Nullable Intent mLegacyIntent; + @Nullable OnClickListener mLegacyOnClickListener; + @Nullable private String mId; /** * Sets the classified text. */ + @NonNull public Builder setText(@Nullable String text) { mText = text; return this; @@ -467,6 +402,7 @@ public final class TextClassification { * 0 implies the entity does not exist for the classified text. * Values greater than 1 are clamped to 1. */ + @NonNull public Builder setEntityType( @NonNull @EntityType String type, @FloatRange(from = 0.0, to = 1.0) float confidenceScore) { @@ -475,60 +411,27 @@ public final class TextClassification { } /** - * Adds an <i>secondary</i> action that may be performed on the classified text. - * Secondary actions are in addition to the <i>primary</i> action which may or may not - * exist. - * - * <p>The label and icon are used for rendering of widgets that offer the intent. - * Actions should be added in order of priority. - * - * <p><stong>Note: </stong> If all input parameters are set to null, this method will be a - * no-op. - * - * @see #setPrimaryAction(Intent, String, Drawable) + * Adds an action that may be performed on the classified text. Actions should be added in + * order of likelihood that the user will use them, with the most likely action being added + * first. */ - public Builder addSecondaryAction( - @Nullable Intent intent, @Nullable String label, @Nullable Drawable icon) { - if (intent != null || label != null || icon != null) { - mSecondaryIntents.add(intent); - mSecondaryLabels.add(label); - mSecondaryIcons.add(icon); - } - return this; - } - - /** - * Removes all the <i>secondary</i> actions. - */ - public Builder clearSecondaryActions() { - mSecondaryIntents.clear(); - mSecondaryLabels.clear(); - mSecondaryIcons.clear(); + @NonNull + public Builder addAction(@NonNull RemoteAction action) { + Preconditions.checkArgument(action != null); + mActions.add(action); return this; } /** - * Sets the <i>primary</i> action that may be performed on the classified text. This is - * equivalent to calling {@code setIntent(intent).setLabel(label).setIcon(icon)}. - * - * <p><strong>Note: </strong>If all input parameters are null, there will be no - * <i>primary</i> action but there may still be <i>secondary</i> actions. - * - * @see #addSecondaryAction(Intent, String, Drawable) - */ - public Builder setPrimaryAction( - @Nullable Intent intent, @Nullable String label, @Nullable Drawable icon) { - return setIntent(intent).setLabel(label).setIcon(icon); - } - - /** * Sets the icon for the <i>primary</i> action that may be rendered on a widget used to act * on the classified text. * - * @see #setPrimaryAction(Intent, String, Drawable) + * @deprecated Use {@link #addAction(RemoteAction)} instead. */ + @Deprecated + @NonNull public Builder setIcon(@Nullable Drawable icon) { - mPrimaryIcon = icon; + mLegacyIcon = icon; return this; } @@ -536,10 +439,12 @@ public final class TextClassification { * Sets the label for the <i>primary</i> action that may be rendered on a widget used to * act on the classified text. * - * @see #setPrimaryAction(Intent, String, Drawable) + * @deprecated Use {@link #addAction(RemoteAction)} instead. */ + @Deprecated + @NonNull public Builder setLabel(@Nullable String label) { - mPrimaryLabel = label; + mLegacyLabel = label; return this; } @@ -547,10 +452,12 @@ public final class TextClassification { * Sets the intent for the <i>primary</i> action that may be fired to act on the classified * text. * - * @see #setPrimaryAction(Intent, String, Drawable) + * @deprecated Use {@link #addAction(RemoteAction)} instead. */ + @Deprecated + @NonNull public Builder setIntent(@Nullable Intent intent) { - mPrimaryIntent = intent; + mLegacyIntent = intent; return this; } @@ -558,51 +465,82 @@ public final class TextClassification { * Sets the OnClickListener for the <i>primary</i> action that may be triggered to act on * the classified text. This field is not parcelable and will always be null when the * object is read from a parcel. + * + * @deprecated Use {@link #addAction(RemoteAction)} instead. */ + @Deprecated + @NonNull public Builder setOnClickListener(@Nullable OnClickListener onClickListener) { - mPrimaryOnClickListener = onClickListener; + mLegacyOnClickListener = onClickListener; return this; } /** - * Sets a signature for the TextClassification object. - * The TextClassifier that generates the TextClassification object may use it as a way to - * internally identify the TextClassification object. + * Sets an id for the TextClassification object. */ - public Builder setSignature(@NonNull String signature) { - mSignature = Preconditions.checkNotNull(signature); + @NonNull + public Builder setId(@Nullable String id) { + mId = id; return this; } /** * Builds and returns a {@link TextClassification} object. */ + @NonNull public TextClassification build() { - return new TextClassification( - mText, - mPrimaryIcon, mPrimaryLabel, mPrimaryIntent, mPrimaryOnClickListener, - mSecondaryIcons, mSecondaryLabels, mSecondaryIntents, - mEntityConfidence, mSignature); + return new TextClassification(mText, mLegacyIcon, mLegacyLabel, mLegacyIntent, + mLegacyOnClickListener, mActions, mEntityConfidence, mId); } } /** - * Optional input parameters for generating TextClassification. + * A request object for generating TextClassification. */ - public static final class Options implements Parcelable { + public static final class Request implements Parcelable { + + private final CharSequence mText; + private final int mStartIndex; + private final int mEndIndex; + @Nullable private final LocaleList mDefaultLocales; + @Nullable private final ZonedDateTime mReferenceTime; + + private Request( + CharSequence text, + int startIndex, + int endIndex, + LocaleList defaultLocales, + ZonedDateTime referenceTime) { + mText = text; + mStartIndex = startIndex; + mEndIndex = endIndex; + mDefaultLocales = defaultLocales; + mReferenceTime = referenceTime; + } - private @Nullable LocaleList mDefaultLocales; + /** + * Returns the text providing context for the text to classify (which is specified + * by the sub sequence starting at startIndex and ending at endIndex) + */ + @NonNull + public CharSequence getText() { + return mText; + } - public Options() {} + /** + * Returns start index of the text to classify. + */ + @IntRange(from = 0) + public int getStartIndex() { + return mStartIndex; + } /** - * @param defaultLocales ordered list of locale preferences that may be used to disambiguate - * the provided text. If no locale preferences exist, set this to null or an empty - * locale list. + * Returns end index of the text to classify. */ - public Options setDefaultLocales(@Nullable LocaleList defaultLocales) { - mDefaultLocales = defaultLocales; - return this; + @IntRange(from = 0) + public int getEndIndex() { + return mEndIndex; } /** @@ -614,6 +552,78 @@ public final class TextClassification { return mDefaultLocales; } + /** + * @return reference time based on which relative dates (e.g. "tomorrow") should be + * interpreted. + */ + @Nullable + public ZonedDateTime getReferenceTime() { + return mReferenceTime; + } + + /** + * A builder for building TextClassification requests. + */ + public static final class Builder { + + private final CharSequence mText; + private final int mStartIndex; + private final int mEndIndex; + + @Nullable private LocaleList mDefaultLocales; + @Nullable private ZonedDateTime mReferenceTime; + + /** + * @param text text providing context for the text to classify (which is specified + * by the sub sequence starting at startIndex and ending at endIndex) + * @param startIndex start index of the text to classify + * @param endIndex end index of the text to classify + */ + public Builder( + @NonNull CharSequence text, + @IntRange(from = 0) int startIndex, + @IntRange(from = 0) int endIndex) { + Utils.checkArgument(text, startIndex, endIndex); + mText = text; + mStartIndex = startIndex; + mEndIndex = endIndex; + } + + /** + * @param defaultLocales ordered list of locale preferences that may be used to + * disambiguate the provided text. If no locale preferences exist, set this to null + * or an empty locale list. + * + * @return this builder + */ + @NonNull + public Builder setDefaultLocales(@Nullable LocaleList defaultLocales) { + mDefaultLocales = defaultLocales; + return this; + } + + /** + * @param referenceTime reference time based on which relative dates (e.g. "tomorrow" + * should be interpreted. This should usually be the time when the text was + * originally composed. If no reference time is set, now is used. + * + * @return this builder + */ + @NonNull + public Builder setReferenceTime(@Nullable ZonedDateTime referenceTime) { + mReferenceTime = referenceTime; + return this; + } + + /** + * Builds and returns the request object. + */ + @NonNull + public Request build() { + return new Request(mText, mStartIndex, mEndIndex, mDefaultLocales, mReferenceTime); + } + } + @Override public int describeContents() { return 0; @@ -621,72 +631,94 @@ public final class TextClassification { @Override public void writeToParcel(Parcel dest, int flags) { + dest.writeString(mText.toString()); + dest.writeInt(mStartIndex); + dest.writeInt(mEndIndex); dest.writeInt(mDefaultLocales != null ? 1 : 0); if (mDefaultLocales != null) { mDefaultLocales.writeToParcel(dest, flags); } + dest.writeInt(mReferenceTime != null ? 1 : 0); + if (mReferenceTime != null) { + dest.writeString(mReferenceTime.toString()); + } } - public static final Parcelable.Creator<Options> CREATOR = - new Parcelable.Creator<Options>() { + public static final Parcelable.Creator<Request> CREATOR = + new Parcelable.Creator<Request>() { @Override - public Options createFromParcel(Parcel in) { - return new Options(in); + public Request createFromParcel(Parcel in) { + return new Request(in); } @Override - public Options[] newArray(int size) { - return new Options[size]; + public Request[] newArray(int size) { + return new Request[size]; } }; - private Options(Parcel in) { - if (in.readInt() > 0) { - mDefaultLocales = LocaleList.CREATOR.createFromParcel(in); - } + private Request(Parcel in) { + mText = in.readString(); + mStartIndex = in.readInt(); + mEndIndex = in.readInt(); + mDefaultLocales = in.readInt() == 0 ? null : LocaleList.CREATOR.createFromParcel(in); + mReferenceTime = in.readInt() == 0 ? null : ZonedDateTime.parse(in.readString()); } } - /** - * Parcelable wrapper for TextClassification objects. - * @hide - */ - public static final class ParcelableWrapper implements Parcelable { - - @NonNull private TextClassification mTextClassification; - - public ParcelableWrapper(@NonNull TextClassification textClassification) { - Preconditions.checkNotNull(textClassification); - mTextClassification = textClassification; - } - - @NonNull - public TextClassification getTextClassification() { - return mTextClassification; - } + @Override + public int describeContents() { + return 0; + } - @Override - public int describeContents() { - return 0; + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeString(mText); + final Bitmap legacyIconBitmap = drawableToBitmap(mLegacyIcon, MAX_LEGACY_ICON_SIZE); + dest.writeInt(legacyIconBitmap != null ? 1 : 0); + if (legacyIconBitmap != null) { + legacyIconBitmap.writeToParcel(dest, flags); } - - @Override - public void writeToParcel(Parcel dest, int flags) { - mTextClassification.writeToParcel(dest, flags); + dest.writeString(mLegacyLabel); + dest.writeInt(mLegacyIntent != null ? 1 : 0); + if (mLegacyIntent != null) { + mLegacyIntent.writeToParcel(dest, flags); } + // mOnClickListener is not parcelable. + dest.writeTypedList(mActions); + mEntityConfidence.writeToParcel(dest, flags); + dest.writeString(mId); + } - public static final Parcelable.Creator<ParcelableWrapper> CREATOR = - new Parcelable.Creator<ParcelableWrapper>() { - @Override - public ParcelableWrapper createFromParcel(Parcel in) { - return new ParcelableWrapper(new TextClassification(in)); - } + public static final Parcelable.Creator<TextClassification> CREATOR = + new Parcelable.Creator<TextClassification>() { + @Override + public TextClassification createFromParcel(Parcel in) { + return new TextClassification(in); + } - @Override - public ParcelableWrapper[] newArray(int size) { - return new ParcelableWrapper[size]; - } - }; + @Override + public TextClassification[] newArray(int size) { + return new TextClassification[size]; + } + }; + private TextClassification(Parcel in) { + mText = in.readString(); + mLegacyIcon = in.readInt() == 0 + ? null + : new BitmapDrawable(Resources.getSystem(), Bitmap.CREATOR.createFromParcel(in)); + mLegacyLabel = in.readString(); + if (in.readInt() == 0) { + mLegacyIntent = null; + } else { + mLegacyIntent = Intent.CREATOR.createFromParcel(in); + mLegacyIntent.removeFlags( + Intent.FLAG_GRANT_READ_URI_PERMISSION | Intent.FLAG_GRANT_WRITE_URI_PERMISSION); + } + mLegacyOnClickListener = null; // not parcelable + mActions = in.createTypedArrayList(RemoteAction.CREATOR); + mEntityConfidence = EntityConfidence.CREATOR.createFromParcel(in); + mId = in.readString(); } } diff --git a/android/view/textclassifier/TextClassificationConstants.java b/android/view/textclassifier/TextClassificationConstants.java new file mode 100644 index 00000000..21b56031 --- /dev/null +++ b/android/view/textclassifier/TextClassificationConstants.java @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.Nullable; +import android.util.KeyValueListParser; +import android.util.Slog; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.StringJoiner; + +/** + * TextClassifier specific settings. + * This is encoded as a key=value list, separated by commas. Ex: + * + * <pre> + * smart_linkify_enabled (boolean) + * system_textclassifier_enabled (boolean) + * model_dark_launch_enabled (boolean) + * smart_selection_enabled (boolean) + * smart_text_share_enabled (boolean) + * smart_linkify_enabled (boolean) + * smart_select_animation_enabled (boolean) + * suggest_selection_max_range_length (int) + * classify_text_max_range_length (int) + * generate_links_max_text_length (int) + * generate_links_log_sample_rate (int) + * entity_list_default (String[]) + * entity_list_not_editable (String[]) + * entity_list_editable (String[]) + * </pre> + * + * <p> + * Type: string + * see also android.provider.Settings.Global.TEXT_CLASSIFIER_CONSTANTS + * + * Example of setting the values for testing. + * adb shell settings put global text_classifier_constants \ + * model_dark_launch_enabled=true,smart_selection_enabled=true,\ + * entity_list_default=phone:address + * @hide + */ +public final class TextClassificationConstants { + + private static final String LOG_TAG = "TextClassificationConstants"; + + private static final String LOCAL_TEXT_CLASSIFIER_ENABLED = + "local_textclassifier_enabled"; + private static final String SYSTEM_TEXT_CLASSIFIER_ENABLED = + "system_textclassifier_enabled"; + private static final String MODEL_DARK_LAUNCH_ENABLED = + "model_dark_launch_enabled"; + private static final String SMART_SELECTION_ENABLED = + "smart_selection_enabled"; + private static final String SMART_TEXT_SHARE_ENABLED = + "smart_text_share_enabled"; + private static final String SMART_LINKIFY_ENABLED = + "smart_linkify_enabled"; + private static final String SMART_SELECT_ANIMATION_ENABLED = + "smart_select_animation_enabled"; + private static final String SUGGEST_SELECTION_MAX_RANGE_LENGTH = + "suggest_selection_max_range_length"; + private static final String CLASSIFY_TEXT_MAX_RANGE_LENGTH = + "classify_text_max_range_length"; + private static final String GENERATE_LINKS_MAX_TEXT_LENGTH = + "generate_links_max_text_length"; + private static final String GENERATE_LINKS_LOG_SAMPLE_RATE = + "generate_links_log_sample_rate"; + private static final String ENTITY_LIST_DEFAULT = + "entity_list_default"; + private static final String ENTITY_LIST_NOT_EDITABLE = + "entity_list_not_editable"; + private static final String ENTITY_LIST_EDITABLE = + "entity_list_editable"; + + private static final boolean LOCAL_TEXT_CLASSIFIER_ENABLED_DEFAULT = true; + private static final boolean SYSTEM_TEXT_CLASSIFIER_ENABLED_DEFAULT = true; + private static final boolean MODEL_DARK_LAUNCH_ENABLED_DEFAULT = false; + private static final boolean SMART_SELECTION_ENABLED_DEFAULT = true; + private static final boolean SMART_TEXT_SHARE_ENABLED_DEFAULT = true; + private static final boolean SMART_LINKIFY_ENABLED_DEFAULT = true; + private static final boolean SMART_SELECT_ANIMATION_ENABLED_DEFAULT = true; + private static final int SUGGEST_SELECTION_MAX_RANGE_LENGTH_DEFAULT = 10 * 1000; + private static final int CLASSIFY_TEXT_MAX_RANGE_LENGTH_DEFAULT = 10 * 1000; + private static final int GENERATE_LINKS_MAX_TEXT_LENGTH_DEFAULT = 100 * 1000; + private static final int GENERATE_LINKS_LOG_SAMPLE_RATE_DEFAULT = 100; + private static final String ENTITY_LIST_DELIMITER = ":"; + private static final String ENTITY_LIST_DEFAULT_VALUE = new StringJoiner(ENTITY_LIST_DELIMITER) + .add(TextClassifier.TYPE_ADDRESS) + .add(TextClassifier.TYPE_EMAIL) + .add(TextClassifier.TYPE_PHONE) + .add(TextClassifier.TYPE_URL) + .add(TextClassifier.TYPE_DATE) + .add(TextClassifier.TYPE_DATE_TIME) + .add(TextClassifier.TYPE_FLIGHT_NUMBER).toString(); + + private final boolean mSystemTextClassifierEnabled; + private final boolean mLocalTextClassifierEnabled; + private final boolean mModelDarkLaunchEnabled; + private final boolean mSmartSelectionEnabled; + private final boolean mSmartTextShareEnabled; + private final boolean mSmartLinkifyEnabled; + private final boolean mSmartSelectionAnimationEnabled; + private final int mSuggestSelectionMaxRangeLength; + private final int mClassifyTextMaxRangeLength; + private final int mGenerateLinksMaxTextLength; + private final int mGenerateLinksLogSampleRate; + private final List<String> mEntityListDefault; + private final List<String> mEntityListNotEditable; + private final List<String> mEntityListEditable; + + private TextClassificationConstants(@Nullable String settings) { + final KeyValueListParser parser = new KeyValueListParser(','); + try { + parser.setString(settings); + } catch (IllegalArgumentException e) { + // Failed to parse the settings string, log this and move on with defaults. + Slog.e(LOG_TAG, "Bad TextClassifier settings: " + settings); + } + mSystemTextClassifierEnabled = parser.getBoolean( + SYSTEM_TEXT_CLASSIFIER_ENABLED, + SYSTEM_TEXT_CLASSIFIER_ENABLED_DEFAULT); + mLocalTextClassifierEnabled = parser.getBoolean( + LOCAL_TEXT_CLASSIFIER_ENABLED, + LOCAL_TEXT_CLASSIFIER_ENABLED_DEFAULT); + mModelDarkLaunchEnabled = parser.getBoolean( + MODEL_DARK_LAUNCH_ENABLED, + MODEL_DARK_LAUNCH_ENABLED_DEFAULT); + mSmartSelectionEnabled = parser.getBoolean( + SMART_SELECTION_ENABLED, + SMART_SELECTION_ENABLED_DEFAULT); + mSmartTextShareEnabled = parser.getBoolean( + SMART_TEXT_SHARE_ENABLED, + SMART_TEXT_SHARE_ENABLED_DEFAULT); + mSmartLinkifyEnabled = parser.getBoolean( + SMART_LINKIFY_ENABLED, + SMART_LINKIFY_ENABLED_DEFAULT); + mSmartSelectionAnimationEnabled = parser.getBoolean( + SMART_SELECT_ANIMATION_ENABLED, + SMART_SELECT_ANIMATION_ENABLED_DEFAULT); + mSuggestSelectionMaxRangeLength = parser.getInt( + SUGGEST_SELECTION_MAX_RANGE_LENGTH, + SUGGEST_SELECTION_MAX_RANGE_LENGTH_DEFAULT); + mClassifyTextMaxRangeLength = parser.getInt( + CLASSIFY_TEXT_MAX_RANGE_LENGTH, + CLASSIFY_TEXT_MAX_RANGE_LENGTH_DEFAULT); + mGenerateLinksMaxTextLength = parser.getInt( + GENERATE_LINKS_MAX_TEXT_LENGTH, + GENERATE_LINKS_MAX_TEXT_LENGTH_DEFAULT); + mGenerateLinksLogSampleRate = parser.getInt( + GENERATE_LINKS_LOG_SAMPLE_RATE, + GENERATE_LINKS_LOG_SAMPLE_RATE_DEFAULT); + mEntityListDefault = parseEntityList(parser.getString( + ENTITY_LIST_DEFAULT, + ENTITY_LIST_DEFAULT_VALUE)); + mEntityListNotEditable = parseEntityList(parser.getString( + ENTITY_LIST_NOT_EDITABLE, + ENTITY_LIST_DEFAULT_VALUE)); + mEntityListEditable = parseEntityList(parser.getString( + ENTITY_LIST_EDITABLE, + ENTITY_LIST_DEFAULT_VALUE)); + } + + /** Load from a settings string. */ + public static TextClassificationConstants loadFromString(String settings) { + return new TextClassificationConstants(settings); + } + + public boolean isLocalTextClassifierEnabled() { + return mLocalTextClassifierEnabled; + } + + public boolean isSystemTextClassifierEnabled() { + return mSystemTextClassifierEnabled; + } + + public boolean isModelDarkLaunchEnabled() { + return mModelDarkLaunchEnabled; + } + + public boolean isSmartSelectionEnabled() { + return mSmartSelectionEnabled; + } + + public boolean isSmartTextShareEnabled() { + return mSmartTextShareEnabled; + } + + public boolean isSmartLinkifyEnabled() { + return mSmartLinkifyEnabled; + } + + public boolean isSmartSelectionAnimationEnabled() { + return mSmartSelectionAnimationEnabled; + } + + public int getSuggestSelectionMaxRangeLength() { + return mSuggestSelectionMaxRangeLength; + } + + public int getClassifyTextMaxRangeLength() { + return mClassifyTextMaxRangeLength; + } + + public int getGenerateLinksMaxTextLength() { + return mGenerateLinksMaxTextLength; + } + + public int getGenerateLinksLogSampleRate() { + return mGenerateLinksLogSampleRate; + } + + public List<String> getEntityListDefault() { + return mEntityListDefault; + } + + public List<String> getEntityListNotEditable() { + return mEntityListNotEditable; + } + + public List<String> getEntityListEditable() { + return mEntityListEditable; + } + + private static List<String> parseEntityList(String listStr) { + return Collections.unmodifiableList(Arrays.asList(listStr.split(ENTITY_LIST_DELIMITER))); + } +} diff --git a/android/view/textclassifier/TextClassificationContext.java b/android/view/textclassifier/TextClassificationContext.java new file mode 100644 index 00000000..a15411f0 --- /dev/null +++ b/android/view/textclassifier/TextClassificationContext.java @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.os.Parcel; +import android.os.Parcelable; +import android.view.textclassifier.TextClassifier.WidgetType; + +import com.android.internal.util.Preconditions; + +import java.util.Locale; + +/** + * A representation of the context in which text classification would be performed. + * @see TextClassificationManager#createTextClassificationSession(TextClassificationContext) + */ +public final class TextClassificationContext implements Parcelable { + + private final String mPackageName; + private final String mWidgetType; + @Nullable private final String mWidgetVersion; + + private TextClassificationContext( + String packageName, + String widgetType, + String widgetVersion) { + mPackageName = Preconditions.checkNotNull(packageName); + mWidgetType = Preconditions.checkNotNull(widgetType); + mWidgetVersion = widgetVersion; + } + + /** + * Returns the package name for the calling package. + */ + @NonNull + public String getPackageName() { + return mPackageName; + } + + /** + * Returns the widget type for this classification context. + */ + @NonNull + @WidgetType + public String getWidgetType() { + return mWidgetType; + } + + /** + * Returns a custom version string for the widget type. + * + * @see #getWidgetType() + */ + @Nullable + public String getWidgetVersion() { + return mWidgetVersion; + } + + @Override + public String toString() { + return String.format(Locale.US, "TextClassificationContext{" + + "packageName=%s, widgetType=%s, widgetVersion=%s}", + mPackageName, mWidgetType, mWidgetVersion); + } + + /** + * A builder for building a TextClassification context. + */ + public static final class Builder { + + private final String mPackageName; + private final String mWidgetType; + + @Nullable private String mWidgetVersion; + + /** + * Initializes a new builder for text classification context objects. + * + * @param packageName the name of the calling package + * @param widgetType the type of widget e.g. {@link TextClassifier#WIDGET_TYPE_TEXTVIEW} + * + * @return this builder + */ + public Builder(@NonNull String packageName, @NonNull @WidgetType String widgetType) { + mPackageName = Preconditions.checkNotNull(packageName); + mWidgetType = Preconditions.checkNotNull(widgetType); + } + + /** + * Sets an optional custom version string for the widget type. + * + * @return this builder + */ + public Builder setWidgetVersion(@Nullable String widgetVersion) { + mWidgetVersion = widgetVersion; + return this; + } + + /** + * Builds the text classification context object. + * + * @return the built TextClassificationContext object + */ + @NonNull + public TextClassificationContext build() { + return new TextClassificationContext(mPackageName, mWidgetType, mWidgetVersion); + } + } + + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(Parcel parcel, int flags) { + parcel.writeString(mPackageName); + parcel.writeString(mWidgetType); + parcel.writeString(mWidgetVersion); + } + + private TextClassificationContext(Parcel in) { + mPackageName = in.readString(); + mWidgetType = in.readString(); + mWidgetVersion = in.readString(); + } + + public static final Parcelable.Creator<TextClassificationContext> CREATOR = + new Parcelable.Creator<TextClassificationContext>() { + @Override + public TextClassificationContext createFromParcel(Parcel parcel) { + return new TextClassificationContext(parcel); + } + + @Override + public TextClassificationContext[] newArray(int size) { + return new TextClassificationContext[size]; + } + }; +} diff --git a/android/view/textclassifier/TextClassificationManager.java b/android/view/textclassifier/TextClassificationManager.java index d7b07761..262d9b85 100644 --- a/android/view/textclassifier/TextClassificationManager.java +++ b/android/view/textclassifier/TextClassificationManager.java @@ -16,10 +16,16 @@ package android.view.textclassifier; +import android.annotation.NonNull; import android.annotation.Nullable; import android.annotation.SystemService; import android.content.Context; +import android.os.ServiceManager; +import android.provider.Settings; +import android.service.textclassifier.TextClassifierService; +import android.view.textclassifier.TextClassifier.TextClassifierType; +import com.android.internal.annotations.GuardedBy; import com.android.internal.util.Preconditions; /** @@ -28,23 +34,49 @@ import com.android.internal.util.Preconditions; @SystemService(Context.TEXT_CLASSIFICATION_SERVICE) public final class TextClassificationManager { - private final Object mTextClassifierLock = new Object(); + private static final String LOG_TAG = "TextClassificationManager"; + + private final Object mLock = new Object(); + private final TextClassificationSessionFactory mDefaultSessionFactory = + classificationContext -> new TextClassificationSession( + classificationContext, getTextClassifier()); private final Context mContext; + private final TextClassificationConstants mSettings; + + @GuardedBy("mLock") private TextClassifier mTextClassifier; + @GuardedBy("mLock") + private TextClassifier mLocalTextClassifier; + @GuardedBy("mLock") + private TextClassifier mSystemTextClassifier; + @GuardedBy("mLock") + private TextClassificationSessionFactory mSessionFactory; /** @hide */ public TextClassificationManager(Context context) { mContext = Preconditions.checkNotNull(context); + mSettings = TextClassificationConstants.loadFromString(Settings.Global.getString( + context.getContentResolver(), Settings.Global.TEXT_CLASSIFIER_CONSTANTS)); + mSessionFactory = mDefaultSessionFactory; } /** - * Returns the text classifier. + * Returns the text classifier that was set via {@link #setTextClassifier(TextClassifier)}. + * If this is null, this method returns a default text classifier (i.e. either the system text + * classifier if one exists, or a local text classifier running in this app.) + * + * @see #setTextClassifier(TextClassifier) */ + @NonNull public TextClassifier getTextClassifier() { - synchronized (mTextClassifierLock) { + synchronized (mLock) { if (mTextClassifier == null) { - mTextClassifier = new TextClassifierImpl(mContext); + if (isSystemTextClassifierEnabled()) { + mTextClassifier = getSystemTextClassifier(); + } else { + mTextClassifier = getLocalTextClassifier(); + } } return mTextClassifier; } @@ -56,8 +88,134 @@ public final class TextClassificationManager { * Set to {@link TextClassifier#NO_OP} to disable text classifier features. */ public void setTextClassifier(@Nullable TextClassifier textClassifier) { - synchronized (mTextClassifierLock) { + synchronized (mLock) { mTextClassifier = textClassifier; } } + + /** + * Returns a specific type of text classifier. + * If the specified text classifier cannot be found, this returns {@link TextClassifier#NO_OP}. + * + * @see TextClassifier#LOCAL + * @see TextClassifier#SYSTEM + * @hide + */ + public TextClassifier getTextClassifier(@TextClassifierType int type) { + switch (type) { + case TextClassifier.LOCAL: + return getLocalTextClassifier(); + default: + return getSystemTextClassifier(); + } + } + + /** @hide */ + public TextClassificationConstants getSettings() { + return mSettings; + } + + /** + * Call this method to start a text classification session with the given context. + * A session is created with a context helping the classifier better understand + * what the user needs and consists of queries and feedback events. The queries + * are directly related to providing useful functionality to the user and the events + * are a feedback loop back to the classifier helping it learn and better serve + * future queries. + * + * <p> All interactions with the returned classifier are considered part of a single + * session and are logically grouped. For example, when a text widget is focused + * all user interactions around text editing (selection, editing, etc) can be + * grouped together to allow the classifier get better. + * + * @param classificationContext The context in which classification would occur + * + * @return An instance to perform classification in the given context + */ + @NonNull + public TextClassifier createTextClassificationSession( + @NonNull TextClassificationContext classificationContext) { + Preconditions.checkNotNull(classificationContext); + final TextClassifier textClassifier = + mSessionFactory.createTextClassificationSession(classificationContext); + Preconditions.checkNotNull(textClassifier, "Session Factory should never return null"); + return textClassifier; + } + + /** + * @see #createTextClassificationSession(TextClassificationContext, TextClassifier) + * @hide + */ + public TextClassifier createTextClassificationSession( + TextClassificationContext classificationContext, TextClassifier textClassifier) { + Preconditions.checkNotNull(classificationContext); + Preconditions.checkNotNull(textClassifier); + return new TextClassificationSession(classificationContext, textClassifier); + } + + /** + * Sets a TextClassificationSessionFactory to be used to create session-aware TextClassifiers. + * + * @param factory the textClassification session factory. If this is null, the default factory + * will be used. + */ + public void setTextClassificationSessionFactory( + @Nullable TextClassificationSessionFactory factory) { + synchronized (mLock) { + if (factory != null) { + mSessionFactory = factory; + } else { + mSessionFactory = mDefaultSessionFactory; + } + } + } + + private TextClassifier getSystemTextClassifier() { + synchronized (mLock) { + if (mSystemTextClassifier == null && isSystemTextClassifierEnabled()) { + try { + mSystemTextClassifier = new SystemTextClassifier(mContext, mSettings); + Log.d(LOG_TAG, "Initialized SystemTextClassifier"); + } catch (ServiceManager.ServiceNotFoundException e) { + Log.e(LOG_TAG, "Could not initialize SystemTextClassifier", e); + } + } + } + if (mSystemTextClassifier != null) { + return mSystemTextClassifier; + } + return TextClassifier.NO_OP; + } + + private TextClassifier getLocalTextClassifier() { + synchronized (mLock) { + if (mLocalTextClassifier == null) { + if (mSettings.isLocalTextClassifierEnabled()) { + mLocalTextClassifier = new TextClassifierImpl(mContext, mSettings); + } else { + Log.d(LOG_TAG, "Local TextClassifier disabled"); + mLocalTextClassifier = TextClassifierImpl.NO_OP; + } + } + return mLocalTextClassifier; + } + } + + private boolean isSystemTextClassifierEnabled() { + return mSettings.isSystemTextClassifierEnabled() + && TextClassifierService.getServiceComponentName(mContext) != null; + } + + /** @hide */ + public static TextClassificationConstants getSettings(Context context) { + Preconditions.checkNotNull(context); + final TextClassificationManager tcm = + context.getSystemService(TextClassificationManager.class); + if (tcm != null) { + return tcm.mSettings; + } else { + return TextClassificationConstants.loadFromString(Settings.Global.getString( + context.getContentResolver(), Settings.Global.TEXT_CLASSIFIER_CONSTANTS)); + } + } } diff --git a/android/view/textclassifier/TextClassificationSession.java b/android/view/textclassifier/TextClassificationSession.java new file mode 100644 index 00000000..e8e300a9 --- /dev/null +++ b/android/view/textclassifier/TextClassificationSession.java @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.WorkerThread; +import android.view.textclassifier.DefaultLogger.SignatureParser; +import android.view.textclassifier.SelectionEvent.InvocationMethod; + +import com.android.internal.util.Preconditions; + +/** + * Session-aware TextClassifier. + */ +@WorkerThread +final class TextClassificationSession implements TextClassifier { + + /* package */ static final boolean DEBUG_LOG_ENABLED = true; + private static final String LOG_TAG = "TextClassificationSession"; + + private final TextClassifier mDelegate; + private final SelectionEventHelper mEventHelper; + private final TextClassificationSessionId mSessionId; + private final TextClassificationContext mClassificationContext; + + private boolean mDestroyed; + + TextClassificationSession(TextClassificationContext context, TextClassifier delegate) { + mClassificationContext = Preconditions.checkNotNull(context); + mDelegate = Preconditions.checkNotNull(delegate); + mSessionId = new TextClassificationSessionId(); + mEventHelper = new SelectionEventHelper(mSessionId, mClassificationContext); + initializeRemoteSession(); + } + + @Override + public TextSelection suggestSelection(TextSelection.Request request) { + checkDestroyed(); + return mDelegate.suggestSelection(request); + } + + private void initializeRemoteSession() { + if (mDelegate instanceof SystemTextClassifier) { + ((SystemTextClassifier) mDelegate).initializeRemoteSession( + mClassificationContext, mSessionId); + } + } + + @Override + public TextClassification classifyText(TextClassification.Request request) { + checkDestroyed(); + return mDelegate.classifyText(request); + } + + @Override + public TextLinks generateLinks(TextLinks.Request request) { + checkDestroyed(); + return mDelegate.generateLinks(request); + } + + @Override + public void onSelectionEvent(SelectionEvent event) { + checkDestroyed(); + Preconditions.checkNotNull(event); + if (mEventHelper.sanitizeEvent(event)) { + mDelegate.onSelectionEvent(event); + } + } + + @Override + public void destroy() { + mEventHelper.endSession(); + mDelegate.destroy(); + mDestroyed = true; + } + + @Override + public boolean isDestroyed() { + return mDestroyed; + } + + /** + * @throws IllegalStateException if this TextClassification session has been destroyed. + * @see #isDestroyed() + * @see #destroy() + */ + private void checkDestroyed() { + if (mDestroyed) { + throw new IllegalStateException("This TextClassification session has been destroyed"); + } + } + + /** + * Helper class for updating SelectionEvent fields. + */ + private static final class SelectionEventHelper { + + private final TextClassificationSessionId mSessionId; + private final TextClassificationContext mContext; + + @InvocationMethod + private int mInvocationMethod = SelectionEvent.INVOCATION_UNKNOWN; + private SelectionEvent mPrevEvent; + private SelectionEvent mSmartEvent; + private SelectionEvent mStartEvent; + + SelectionEventHelper( + TextClassificationSessionId sessionId, TextClassificationContext context) { + mSessionId = Preconditions.checkNotNull(sessionId); + mContext = Preconditions.checkNotNull(context); + } + + /** + * Updates the necessary fields in the event for the current session. + * + * @return true if the event should be reported. false if the event should be ignored + */ + boolean sanitizeEvent(SelectionEvent event) { + updateInvocationMethod(event); + modifyAutoSelectionEventType(event); + + if (event.getEventType() != SelectionEvent.EVENT_SELECTION_STARTED + && mStartEvent == null) { + if (DEBUG_LOG_ENABLED) { + Log.d(LOG_TAG, "Selection session not yet started. Ignoring event"); + } + return false; + } + + final long now = System.currentTimeMillis(); + switch (event.getEventType()) { + case SelectionEvent.EVENT_SELECTION_STARTED: + Preconditions.checkArgument( + event.getAbsoluteEnd() == event.getAbsoluteStart() + 1); + event.setSessionId(mSessionId); + mStartEvent = event; + break; + case SelectionEvent.EVENT_SMART_SELECTION_SINGLE: // fall through + case SelectionEvent.EVENT_SMART_SELECTION_MULTI: + mSmartEvent = event; + break; + case SelectionEvent.EVENT_SELECTION_MODIFIED: // fall through + case SelectionEvent.EVENT_AUTO_SELECTION: + if (mPrevEvent != null + && mPrevEvent.getAbsoluteStart() == event.getAbsoluteStart() + && mPrevEvent.getAbsoluteEnd() == event.getAbsoluteEnd()) { + // Selection did not change. Ignore event. + return false; + } + break; + default: + // do nothing. + } + + event.setEventTime(now); + if (mStartEvent != null) { + event.setSessionId(mStartEvent.getSessionId()) + .setDurationSinceSessionStart(now - mStartEvent.getEventTime()) + .setStart(event.getAbsoluteStart() - mStartEvent.getAbsoluteStart()) + .setEnd(event.getAbsoluteEnd() - mStartEvent.getAbsoluteStart()); + } + if (mSmartEvent != null) { + event.setResultId(mSmartEvent.getResultId()) + .setSmartStart( + mSmartEvent.getAbsoluteStart() - mStartEvent.getAbsoluteStart()) + .setSmartEnd(mSmartEvent.getAbsoluteEnd() - mStartEvent.getAbsoluteStart()); + } + if (mPrevEvent != null) { + event.setDurationSincePreviousEvent(now - mPrevEvent.getEventTime()) + .setEventIndex(mPrevEvent.getEventIndex() + 1); + } + mPrevEvent = event; + return true; + } + + void endSession() { + mPrevEvent = null; + mSmartEvent = null; + mStartEvent = null; + } + + private void updateInvocationMethod(SelectionEvent event) { + event.setTextClassificationSessionContext(mContext); + if (event.getInvocationMethod() == SelectionEvent.INVOCATION_UNKNOWN) { + event.setInvocationMethod(mInvocationMethod); + } else { + mInvocationMethod = event.getInvocationMethod(); + } + } + + private void modifyAutoSelectionEventType(SelectionEvent event) { + switch (event.getEventType()) { + case SelectionEvent.EVENT_SMART_SELECTION_SINGLE: // fall through + case SelectionEvent.EVENT_SMART_SELECTION_MULTI: // fall through + case SelectionEvent.EVENT_AUTO_SELECTION: + if (isPlatformLocalTextClassifierSmartSelection(event.getResultId())) { + if (event.getAbsoluteEnd() - event.getAbsoluteStart() > 1) { + event.setEventType(SelectionEvent.EVENT_SMART_SELECTION_MULTI); + } else { + event.setEventType(SelectionEvent.EVENT_SMART_SELECTION_SINGLE); + } + } else { + event.setEventType(SelectionEvent.EVENT_AUTO_SELECTION); + } + return; + default: + return; + } + } + + private static boolean isPlatformLocalTextClassifierSmartSelection(String signature) { + return DefaultLogger.CLASSIFIER_ID.equals(SignatureParser.getClassifierId(signature)); + } + } +} diff --git a/android/view/textclassifier/TextClassificationSessionFactory.java b/android/view/textclassifier/TextClassificationSessionFactory.java new file mode 100644 index 00000000..c0914b6c --- /dev/null +++ b/android/view/textclassifier/TextClassificationSessionFactory.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; + +/** + * An interface for creating a session-aware TextClassifier. + * + * @see TextClassificationManager#createTextClassificationSession(TextClassificationContext) + */ +public interface TextClassificationSessionFactory { + + /** + * Creates and returns a session-aware TextClassifier. + * + * @param classificationContext the classification context + */ + @NonNull + TextClassifier createTextClassificationSession( + @NonNull TextClassificationContext classificationContext); +} diff --git a/android/view/textclassifier/TextClassificationSessionId.java b/android/view/textclassifier/TextClassificationSessionId.java new file mode 100644 index 00000000..1378bd9c --- /dev/null +++ b/android/view/textclassifier/TextClassificationSessionId.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; +import android.os.Parcel; +import android.os.Parcelable; + +import com.android.internal.util.Preconditions; + +import java.util.Locale; +import java.util.UUID; + +/** + * This class represents the id of a text classification session. + */ +public final class TextClassificationSessionId implements Parcelable { + private final @NonNull String mValue; + + /** + * Creates a new instance. + * + * @hide + */ + public TextClassificationSessionId() { + this(UUID.randomUUID().toString()); + } + + /** + * Creates a new instance. + * + * @param value The internal value. + * + * @hide + */ + public TextClassificationSessionId(@NonNull String value) { + mValue = value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + mValue.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + TextClassificationSessionId other = (TextClassificationSessionId) obj; + if (!mValue.equals(other.mValue)) { + return false; + } + return true; + } + + @Override + public String toString() { + return String.format(Locale.US, "TextClassificationSessionId {%s}", mValue); + } + + @Override + public void writeToParcel(Parcel parcel, int flags) { + parcel.writeString(mValue); + } + + @Override + public int describeContents() { + return 0; + } + + /** + * Flattens this id to a string. + * + * @return The flattened id. + * + * @hide + */ + public @NonNull String flattenToString() { + return mValue; + } + + /** + * Unflattens a print job id from a string. + * + * @param string The string. + * @return The unflattened id, or null if the string is malformed. + * + * @hide + */ + public static @NonNull TextClassificationSessionId unflattenFromString(@NonNull String string) { + return new TextClassificationSessionId(string); + } + + public static final Parcelable.Creator<TextClassificationSessionId> CREATOR = + new Parcelable.Creator<TextClassificationSessionId>() { + @Override + public TextClassificationSessionId createFromParcel(Parcel parcel) { + return new TextClassificationSessionId( + Preconditions.checkNotNull(parcel.readString())); + } + + @Override + public TextClassificationSessionId[] newArray(int size) { + return new TextClassificationSessionId[size]; + } + }; +} diff --git a/android/view/textclassifier/TextClassifier.java b/android/view/textclassifier/TextClassifier.java index e9715c51..54261be3 100644 --- a/android/view/textclassifier/TextClassifier.java +++ b/android/view/textclassifier/TextClassifier.java @@ -23,8 +23,15 @@ import android.annotation.Nullable; import android.annotation.StringDef; import android.annotation.WorkerThread; import android.os.LocaleList; +import android.os.Looper; import android.os.Parcel; import android.os.Parcelable; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.URLSpan; +import android.text.util.Linkify; +import android.text.util.Linkify.LinkifyMask; +import android.util.ArrayMap; import android.util.ArraySet; import com.android.internal.util.Preconditions; @@ -35,24 +42,49 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; /** * Interface for providing text classification related features. * - * <p>Unless otherwise stated, methods of this interface are blocking operations. - * Avoid calling them on the UI thread. + * <p><strong>NOTE: </strong>Unless otherwise stated, methods of this interface are blocking + * operations. Call on a worker thread. */ public interface TextClassifier { /** @hide */ String DEFAULT_LOG_TAG = "androidtc"; + + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef(value = {LOCAL, SYSTEM}) + @interface TextClassifierType {} // TODO: Expose as system APIs. + /** Specifies a TextClassifier that runs locally in the app's process. @hide */ + int LOCAL = 0; + /** Specifies a TextClassifier that runs in the system process and serves all apps. @hide */ + int SYSTEM = 1; + + /** The TextClassifier failed to run. */ String TYPE_UNKNOWN = ""; + /** The classifier ran, but didn't recognize a known entity. */ String TYPE_OTHER = "other"; + /** E-mail address (e.g. "noreply@android.com"). */ String TYPE_EMAIL = "email"; + /** Phone number (e.g. "555-123 456"). */ String TYPE_PHONE = "phone"; + /** Physical address. */ String TYPE_ADDRESS = "address"; + /** Web URL. */ String TYPE_URL = "url"; + /** Time reference that is no more specific than a date. May be absolute such as "01/01/2000" or + * relative like "tomorrow". **/ + String TYPE_DATE = "date"; + /** Time reference that includes a specific time. May be absolute such as "01/01/2000 5:30pm" or + * relative like "tomorrow at 5:30pm". **/ + String TYPE_DATE_TIME = "datetime"; + /** Flight number in IATA format. */ + String TYPE_FLIGHT_NUMBER = "flight"; /** @hide */ @Retention(RetentionPolicy.SOURCE) @@ -63,22 +95,53 @@ public interface TextClassifier { TYPE_PHONE, TYPE_ADDRESS, TYPE_URL, + TYPE_DATE, + TYPE_DATE_TIME, + TYPE_FLIGHT_NUMBER, }) @interface EntityType {} - /** Designates that the TextClassifier should identify all entity types it can. **/ - int ENTITY_PRESET_ALL = 0; - /** Designates that the TextClassifier should identify no entities. **/ - int ENTITY_PRESET_NONE = 1; - /** Designates that the TextClassifier should identify a base set of entities determined by the - * TextClassifier. **/ - int ENTITY_PRESET_BASE = 2; + /** Designates that the text in question is editable. **/ + String HINT_TEXT_IS_EDITABLE = "android.text_is_editable"; + /** Designates that the text in question is not editable. **/ + String HINT_TEXT_IS_NOT_EDITABLE = "android.text_is_not_editable"; /** @hide */ @Retention(RetentionPolicy.SOURCE) - @IntDef(prefix = { "ENTITY_CONFIG_" }, - value = {ENTITY_PRESET_ALL, ENTITY_PRESET_NONE, ENTITY_PRESET_BASE}) - @interface EntityPreset {} + @StringDef(prefix = { "HINT_" }, value = {HINT_TEXT_IS_EDITABLE, HINT_TEXT_IS_NOT_EDITABLE}) + @interface Hints {} + + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @StringDef({WIDGET_TYPE_TEXTVIEW, WIDGET_TYPE_WEBVIEW, WIDGET_TYPE_EDITTEXT, + WIDGET_TYPE_EDIT_WEBVIEW, WIDGET_TYPE_CUSTOM_TEXTVIEW, WIDGET_TYPE_CUSTOM_EDITTEXT, + WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW, WIDGET_TYPE_UNKNOWN}) + @interface WidgetType {} + + /** The widget involved in the text classification session is a standard + * {@link android.widget.TextView}. */ + String WIDGET_TYPE_TEXTVIEW = "textview"; + /** The widget involved in the text classification session is a standard + * {@link android.widget.EditText}. */ + String WIDGET_TYPE_EDITTEXT = "edittext"; + /** The widget involved in the text classification session is a standard non-selectable + * {@link android.widget.TextView}. */ + String WIDGET_TYPE_UNSELECTABLE_TEXTVIEW = "nosel-textview"; + /** The widget involved in the text classification session is a standard + * {@link android.webkit.WebView}. */ + String WIDGET_TYPE_WEBVIEW = "webview"; + /** The widget involved in the text classification session is a standard editable + * {@link android.webkit.WebView}. */ + String WIDGET_TYPE_EDIT_WEBVIEW = "edit-webview"; + /** The widget involved in the text classification session is a custom text widget. */ + String WIDGET_TYPE_CUSTOM_TEXTVIEW = "customview"; + /** The widget involved in the text classification session is a custom editable text widget. */ + String WIDGET_TYPE_CUSTOM_EDITTEXT = "customedit"; + /** The widget involved in the text classification session is a custom non-selectable text + * widget. */ + String WIDGET_TYPE_CUSTOM_UNSELECTABLE_TEXTVIEW = "nosel-customview"; + /** The widget involved in the text classification session is of an unknown/unspecified type. */ + String WIDGET_TYPE_UNKNOWN = "unknown"; /** * No-op TextClassifier. @@ -90,63 +153,46 @@ public interface TextClassifier { * Returns suggested text selection start and end indices, recognized entity types, and their * associated confidence scores. The entity types are ordered from highest to lowest scoring. * - * @param text text providing context for the selected text (which is specified - * by the sub sequence starting at selectionStartIndex and ending at selectionEndIndex) - * @param selectionStartIndex start index of the selected part of text - * @param selectionEndIndex end index of the selected part of text - * @param options optional input parameters + * <p><strong>NOTE: </strong>Call on a worker thread. * - * @throws IllegalArgumentException if text is null; selectionStartIndex is negative; - * selectionEndIndex is greater than text.length() or not greater than selectionStartIndex + * <p><strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. * - * @see #suggestSelection(CharSequence, int, int) + * @param request the text selection request */ @WorkerThread @NonNull - default TextSelection suggestSelection( - @NonNull CharSequence text, - @IntRange(from = 0) int selectionStartIndex, - @IntRange(from = 0) int selectionEndIndex, - @Nullable TextSelection.Options options) { - Utils.validateInput(text, selectionStartIndex, selectionEndIndex); - return new TextSelection.Builder(selectionStartIndex, selectionEndIndex).build(); + default TextSelection suggestSelection(@NonNull TextSelection.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); + return new TextSelection.Builder(request.getStartIndex(), request.getEndIndex()).build(); } /** * Returns suggested text selection start and end indices, recognized entity types, and their * associated confidence scores. The entity types are ordered from highest to lowest scoring. * + * <p><strong>NOTE: </strong>Call on a worker thread. + * + * <p><strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. + * * <p><b>NOTE:</b> Do not implement. The default implementation of this method calls - * {@link #suggestSelection(CharSequence, int, int, TextSelection.Options)}. If that method - * calls this method, a stack overflow error will happen. + * {@link #suggestSelection(TextSelection.Request)}. If that method calls this method, + * a stack overflow error will happen. * * @param text text providing context for the selected text (which is specified * by the sub sequence starting at selectionStartIndex and ending at selectionEndIndex) * @param selectionStartIndex start index of the selected part of text * @param selectionEndIndex end index of the selected part of text + * @param defaultLocales ordered list of locale preferences that may be used to + * disambiguate the provided text. If no locale preferences exist, set this to null + * or an empty locale list. * * @throws IllegalArgumentException if text is null; selectionStartIndex is negative; * selectionEndIndex is greater than text.length() or not greater than selectionStartIndex * - * @see #suggestSelection(CharSequence, int, int, TextSelection.Options) - */ - @WorkerThread - @NonNull - default TextSelection suggestSelection( - @NonNull CharSequence text, - @IntRange(from = 0) int selectionStartIndex, - @IntRange(from = 0) int selectionEndIndex) { - return suggestSelection(text, selectionStartIndex, selectionEndIndex, - (TextSelection.Options) null); - } - - /** - * See {@link #suggestSelection(CharSequence, int, int)} or - * {@link #suggestSelection(CharSequence, int, int, TextSelection.Options)}. - * - * <p><b>NOTE:</b> Do not implement. The default implementation of this method calls - * {@link #suggestSelection(CharSequence, int, int, TextSelection.Options)}. If that method - * calls this method, a stack overflow error will happen. + * @see #suggestSelection(TextSelection.Request) */ @WorkerThread @NonNull @@ -155,35 +201,29 @@ public interface TextClassifier { @IntRange(from = 0) int selectionStartIndex, @IntRange(from = 0) int selectionEndIndex, @Nullable LocaleList defaultLocales) { - final TextSelection.Options options = (defaultLocales != null) - ? new TextSelection.Options().setDefaultLocales(defaultLocales) - : null; - return suggestSelection(text, selectionStartIndex, selectionEndIndex, options); + final TextSelection.Request request = new TextSelection.Request.Builder( + text, selectionStartIndex, selectionEndIndex) + .setDefaultLocales(defaultLocales) + .build(); + return suggestSelection(request); } /** * Classifies the specified text and returns a {@link TextClassification} object that can be * used to generate a widget for handling the classified text. * - * @param text text providing context for the text to classify (which is specified - * by the sub sequence starting at startIndex and ending at endIndex) - * @param startIndex start index of the text to classify - * @param endIndex end index of the text to classify - * @param options optional input parameters + * <p><strong>NOTE: </strong>Call on a worker thread. * - * @throws IllegalArgumentException if text is null; startIndex is negative; - * endIndex is greater than text.length() or not greater than startIndex + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. * - * @see #classifyText(CharSequence, int, int) + * @param request the text classification request */ @WorkerThread @NonNull - default TextClassification classifyText( - @NonNull CharSequence text, - @IntRange(from = 0) int startIndex, - @IntRange(from = 0) int endIndex, - @Nullable TextClassification.Options options) { - Utils.validateInput(text, startIndex, endIndex); + default TextClassification classifyText(@NonNull TextClassification.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); return TextClassification.EMPTY; } @@ -191,115 +231,117 @@ public interface TextClassifier { * Classifies the specified text and returns a {@link TextClassification} object that can be * used to generate a widget for handling the classified text. * + * <p><strong>NOTE: </strong>Call on a worker thread. + * * <p><b>NOTE:</b> Do not implement. The default implementation of this method calls - * {@link #classifyText(CharSequence, int, int, TextClassification.Options)}. If that method - * calls this method, a stack overflow error will happen. + * {@link #classifyText(TextClassification.Request)}. If that method calls this method, + * a stack overflow error will happen. + * + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. * * @param text text providing context for the text to classify (which is specified * by the sub sequence starting at startIndex and ending at endIndex) * @param startIndex start index of the text to classify * @param endIndex end index of the text to classify + * @param defaultLocales ordered list of locale preferences that may be used to + * disambiguate the provided text. If no locale preferences exist, set this to null + * or an empty locale list. * * @throws IllegalArgumentException if text is null; startIndex is negative; * endIndex is greater than text.length() or not greater than startIndex * - * @see #classifyText(CharSequence, int, int, TextClassification.Options) + * @see #classifyText(TextClassification.Request) */ @WorkerThread @NonNull default TextClassification classifyText( @NonNull CharSequence text, @IntRange(from = 0) int startIndex, - @IntRange(from = 0) int endIndex) { - return classifyText(text, startIndex, endIndex, (TextClassification.Options) null); + @IntRange(from = 0) int endIndex, + @Nullable LocaleList defaultLocales) { + final TextClassification.Request request = new TextClassification.Request.Builder( + text, startIndex, endIndex) + .setDefaultLocales(defaultLocales) + .build(); + return classifyText(request); } /** - * See {@link #classifyText(CharSequence, int, int, TextClassification.Options)} or - * {@link #classifyText(CharSequence, int, int)}. + * Generates and returns a {@link TextLinks} that may be applied to the text to annotate it with + * links information. * - * <p><b>NOTE:</b> Do not implement. The default implementation of this method calls - * {@link #classifyText(CharSequence, int, int, TextClassification.Options)}. If that method - * calls this method, a stack overflow error will happen. + * <p><strong>NOTE: </strong>Call on a worker thread. + * + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. + * + * @param request the text links request + * + * @see #getMaxGenerateLinksTextLength() */ @WorkerThread @NonNull - default TextClassification classifyText( - @NonNull CharSequence text, - @IntRange(from = 0) int startIndex, - @IntRange(from = 0) int endIndex, - @Nullable LocaleList defaultLocales) { - final TextClassification.Options options = (defaultLocales != null) - ? new TextClassification.Options().setDefaultLocales(defaultLocales) - : null; - return classifyText(text, startIndex, endIndex, options); + default TextLinks generateLinks(@NonNull TextLinks.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); + return new TextLinks.Builder(request.getText().toString()).build(); } /** - * Returns a {@link TextLinks} that may be applied to the text to annotate it with links - * information. - * - * If no options are supplied, default values will be used, determined by the TextClassifier. + * Returns the maximal length of text that can be processed by generateLinks. * - * @param text the text to generate annotations for - * @param options configuration for link generation + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. * - * @throws IllegalArgumentException if text is null - * - * @see #generateLinks(CharSequence) + * @see #generateLinks(TextLinks.Request) */ @WorkerThread - default TextLinks generateLinks( - @NonNull CharSequence text, @Nullable TextLinks.Options options) { - Utils.validateInput(text); - return new TextLinks.Builder(text.toString()).build(); + default int getMaxGenerateLinksTextLength() { + return Integer.MAX_VALUE; } /** - * Returns a {@link TextLinks} that may be applied to the text to annotate it with links - * information. - * - * <p><b>NOTE:</b> Do not implement. The default implementation of this method calls - * {@link #generateLinks(CharSequence, TextLinks.Options)}. If that method calls this method, - * a stack overflow error will happen. - * - * @param text the text to generate annotations for + * Returns a helper for logging TextClassifier related events. * - * @throws IllegalArgumentException if text is null - * - * @see #generateLinks(CharSequence, TextLinks.Options) + * @param config logger configuration + * @hide */ @WorkerThread - default TextLinks generateLinks(@NonNull CharSequence text) { - return generateLinks(text, null); + default Logger getLogger(@NonNull Logger.Config config) { + Preconditions.checkNotNull(config); + return Logger.DISABLED; } /** - * Returns a {@link Collection} of the entity types in the specified preset. + * Reports a selection event. * - * @see #ENTITY_PRESET_ALL - * @see #ENTITY_PRESET_NONE + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to this method should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. */ - default Collection<String> getEntitiesForPreset(@EntityPreset int entityPreset) { - return Collections.EMPTY_LIST; - } + default void onSelectionEvent(@NonNull SelectionEvent event) {} /** - * Logs a TextClassifier event. + * Destroys this TextClassifier. * - * @param source the text classifier used to generate this event - * @param event the text classifier related event - * @hide + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, calls to its methods should + * throw an {@link IllegalStateException}. See {@link #isDestroyed()}. + * + * <p>Subsequent calls to this method are no-ops. */ - @WorkerThread - default void logEvent(String source, String event) {} + default void destroy() {} /** - * Returns this TextClassifier's settings. - * @hide + * Returns whether or not this TextClassifier has been destroyed. + * + * <strong>NOTE: </strong>If a TextClassifier has been destroyed, caller should not interact + * with the classifier and an attempt to do so would throw an {@link IllegalStateException}. + * However, this method should never throw an {@link IllegalStateException}. + * + * @see #destroy() */ - default TextClassifierConstants getSettings() { - return TextClassifierConstants.DEFAULT; + default boolean isDestroyed() { + return false; } /** @@ -308,54 +350,93 @@ public interface TextClassifier { * Configs are initially based on a predefined preset, and can be modified from there. */ final class EntityConfig implements Parcelable { - private final @TextClassifier.EntityPreset int mEntityPreset; + private final Collection<String> mHints; private final Collection<String> mExcludedEntityTypes; private final Collection<String> mIncludedEntityTypes; + private final boolean mUseHints; + + private EntityConfig(boolean useHints, Collection<String> hints, + Collection<String> includedEntityTypes, Collection<String> excludedEntityTypes) { + mHints = hints == null + ? Collections.EMPTY_LIST + : Collections.unmodifiableCollection(new ArraySet<>(hints)); + mExcludedEntityTypes = excludedEntityTypes == null + ? Collections.EMPTY_LIST : new ArraySet<>(excludedEntityTypes); + mIncludedEntityTypes = includedEntityTypes == null + ? Collections.EMPTY_LIST : new ArraySet<>(includedEntityTypes); + mUseHints = useHints; + } - public EntityConfig(@TextClassifier.EntityPreset int mEntityPreset) { - this.mEntityPreset = mEntityPreset; - mExcludedEntityTypes = new ArraySet<>(); - mIncludedEntityTypes = new ArraySet<>(); + /** + * Creates an EntityConfig. + * + * @param hints Hints for the TextClassifier to determine what types of entities to find. + */ + public static EntityConfig createWithHints(@Nullable Collection<String> hints) { + return new EntityConfig(/* useHints */ true, hints, + /* includedEntityTypes */null, /* excludedEntityTypes */ null); } /** - * Specifies an entity to include in addition to any specified by the enity preset. + * Creates an EntityConfig. + * + * @param hints Hints for the TextClassifier to determine what types of entities to find + * @param includedEntityTypes Entity types, e.g. {@link #TYPE_EMAIL}, to explicitly include + * @param excludedEntityTypes Entity types, e.g. {@link #TYPE_PHONE}, to explicitly exclude + * * * Note that if an entity has been excluded, the exclusion will take precedence. */ - public EntityConfig includeEntities(String... entities) { - for (String entity : entities) { - mIncludedEntityTypes.add(entity); - } - return this; + public static EntityConfig create(@Nullable Collection<String> hints, + @Nullable Collection<String> includedEntityTypes, + @Nullable Collection<String> excludedEntityTypes) { + return new EntityConfig(/* useHints */ true, hints, + includedEntityTypes, excludedEntityTypes); } /** - * Specifies an entity to be excluded. + * Creates an EntityConfig with an explicit entity list. + * + * @param entityTypes Complete set of entities, e.g. {@link #TYPE_URL} to find. + * */ - public EntityConfig excludeEntities(String... entities) { - for (String entity : entities) { - mExcludedEntityTypes.add(entity); - } - return this; + public static EntityConfig createWithExplicitEntityList( + @Nullable Collection<String> entityTypes) { + return new EntityConfig(/* useHints */ false, /* hints */ null, + /* includedEntityTypes */ entityTypes, /* excludedEntityTypes */ null); } /** - * Returns an unmodifiable list of the final set of entities to find. + * Returns a list of the final set of entities to find. + * + * @param entities Entities we think should be found before factoring in includes/excludes + * + * This method is intended for use by TextClassifier implementations. */ - public List<String> getEntities(TextClassifier textClassifier) { - ArrayList<String> entities = new ArrayList<>(); - for (String entity : textClassifier.getEntitiesForPreset(mEntityPreset)) { - if (!mExcludedEntityTypes.contains(entity)) { - entities.add(entity); + public List<String> resolveEntityListModifications(@NonNull Collection<String> entities) { + final ArrayList<String> finalList = new ArrayList<>(); + if (mUseHints) { + for (String entity : entities) { + if (!mExcludedEntityTypes.contains(entity)) { + finalList.add(entity); + } } } for (String entity : mIncludedEntityTypes) { - if (!mExcludedEntityTypes.contains(entity) && !entities.contains(entity)) { - entities.add(entity); + if (!mExcludedEntityTypes.contains(entity) && !finalList.contains(entity)) { + finalList.add(entity); } } - return Collections.unmodifiableList(entities); + return finalList; + } + + /** + * Retrieves the list of hints. + * + * @return An unmodifiable collection of the hints. + */ + public Collection<String> getHints() { + return mHints; } @Override @@ -365,9 +446,10 @@ public interface TextClassifier { @Override public void writeToParcel(Parcel dest, int flags) { - dest.writeInt(mEntityPreset); + dest.writeStringList(new ArrayList<>(mHints)); dest.writeStringList(new ArrayList<>(mExcludedEntityTypes)); dest.writeStringList(new ArrayList<>(mIncludedEntityTypes)); + dest.writeInt(mUseHints ? 1 : 0); } public static final Parcelable.Creator<EntityConfig> CREATOR = @@ -384,9 +466,10 @@ public interface TextClassifier { }; private EntityConfig(Parcel in) { - mEntityPreset = in.readInt(); + mHints = new ArraySet<>(in.createStringArrayList()); mExcludedEntityTypes = new ArraySet<>(in.createStringArrayList()); mIncludedEntityTypes = new ArraySet<>(in.createStringArrayList()); + mUseHints = in.readInt() == 1; } } @@ -407,19 +490,79 @@ public interface TextClassifier { * endIndex is greater than text.length() or is not greater than startIndex; * options is null */ - static void validateInput( - @NonNull CharSequence text, int startIndex, int endIndex) { + static void checkArgument(@NonNull CharSequence text, int startIndex, int endIndex) { Preconditions.checkArgument(text != null); Preconditions.checkArgument(startIndex >= 0); Preconditions.checkArgument(endIndex <= text.length()); Preconditions.checkArgument(endIndex > startIndex); } + static void checkTextLength(CharSequence text, int maxLength) { + Preconditions.checkArgumentInRange(text.length(), 0, maxLength, "text.length()"); + } + /** - * @throws IllegalArgumentException if text is null or options is null + * Generates links using legacy {@link Linkify}. */ - static void validateInput(@NonNull CharSequence text) { - Preconditions.checkArgument(text != null); + public static TextLinks generateLegacyLinks(@NonNull TextLinks.Request request) { + final String string = request.getText().toString(); + final TextLinks.Builder links = new TextLinks.Builder(string); + + final List<String> entities = request.getEntityConfig() + .resolveEntityListModifications(Collections.emptyList()); + if (entities.contains(TextClassifier.TYPE_URL)) { + addLinks(links, string, TextClassifier.TYPE_URL); + } + if (entities.contains(TextClassifier.TYPE_PHONE)) { + addLinks(links, string, TextClassifier.TYPE_PHONE); + } + if (entities.contains(TextClassifier.TYPE_EMAIL)) { + addLinks(links, string, TextClassifier.TYPE_EMAIL); + } + // NOTE: Do not support MAP_ADDRESSES. Legacy version does not work well. + return links.build(); + } + + private static void addLinks( + TextLinks.Builder links, String string, @EntityType String entityType) { + final Spannable spannable = new SpannableString(string); + if (Linkify.addLinks(spannable, linkMask(entityType))) { + final URLSpan[] spans = spannable.getSpans(0, spannable.length(), URLSpan.class); + for (URLSpan urlSpan : spans) { + links.addLink( + spannable.getSpanStart(urlSpan), + spannable.getSpanEnd(urlSpan), + entityScores(entityType), + urlSpan); + } + } + } + + @LinkifyMask + private static int linkMask(@EntityType String entityType) { + switch (entityType) { + case TextClassifier.TYPE_URL: + return Linkify.WEB_URLS; + case TextClassifier.TYPE_PHONE: + return Linkify.PHONE_NUMBERS; + case TextClassifier.TYPE_EMAIL: + return Linkify.EMAIL_ADDRESSES; + default: + // NOTE: Do not support MAP_ADDRESSES. Legacy version does not work well. + return 0; + } + } + + private static Map<String, Float> entityScores(@EntityType String entityType) { + final Map<String, Float> scores = new ArrayMap<>(); + scores.put(entityType, 1f); + return scores; + } + + static void checkMainThread() { + if (Looper.myLooper() == Looper.getMainLooper()) { + Log.w(DEFAULT_LOG_TAG, "TextClassifier called on main thread"); + } } } } diff --git a/android/view/textclassifier/TextClassifierConstants.java b/android/view/textclassifier/TextClassifierConstants.java deleted file mode 100644 index 00695b79..00000000 --- a/android/view/textclassifier/TextClassifierConstants.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package android.view.textclassifier; - -import android.annotation.Nullable; -import android.util.KeyValueListParser; -import android.util.Slog; - -/** - * TextClassifier specific settings. - * This is encoded as a key=value list, separated by commas. Ex: - * - * <pre> - * smart_selection_dark_launch (boolean) - * smart_selection_enabled_for_edit_text (boolean) - * </pre> - * - * <p> - * Type: string - * see also android.provider.Settings.Global.TEXT_CLASSIFIER_CONSTANTS - * - * Example of setting the values for testing. - * adb shell settings put global text_classifier_constants smart_selection_dark_launch=true,smart_selection_enabled_for_edit_text=true - * @hide - */ -public final class TextClassifierConstants { - - private static final String LOG_TAG = "TextClassifierConstants"; - - private static final String SMART_SELECTION_DARK_LAUNCH = - "smart_selection_dark_launch"; - private static final String SMART_SELECTION_ENABLED_FOR_EDIT_TEXT = - "smart_selection_enabled_for_edit_text"; - private static final String SMART_LINKIFY_ENABLED = - "smart_linkify_enabled"; - - private static final boolean SMART_SELECTION_DARK_LAUNCH_DEFAULT = false; - private static final boolean SMART_SELECTION_ENABLED_FOR_EDIT_TEXT_DEFAULT = true; - private static final boolean SMART_LINKIFY_ENABLED_DEFAULT = true; - - /** Default settings. */ - static final TextClassifierConstants DEFAULT = new TextClassifierConstants(); - - private final boolean mDarkLaunch; - private final boolean mSuggestSelectionEnabledForEditableText; - private final boolean mSmartLinkifyEnabled; - - private TextClassifierConstants() { - mDarkLaunch = SMART_SELECTION_DARK_LAUNCH_DEFAULT; - mSuggestSelectionEnabledForEditableText = SMART_SELECTION_ENABLED_FOR_EDIT_TEXT_DEFAULT; - mSmartLinkifyEnabled = SMART_LINKIFY_ENABLED_DEFAULT; - } - - private TextClassifierConstants(@Nullable String settings) { - final KeyValueListParser parser = new KeyValueListParser(','); - try { - parser.setString(settings); - } catch (IllegalArgumentException e) { - // Failed to parse the settings string, log this and move on with defaults. - Slog.e(LOG_TAG, "Bad TextClassifier settings: " + settings); - } - mDarkLaunch = parser.getBoolean( - SMART_SELECTION_DARK_LAUNCH, - SMART_SELECTION_DARK_LAUNCH_DEFAULT); - mSuggestSelectionEnabledForEditableText = parser.getBoolean( - SMART_SELECTION_ENABLED_FOR_EDIT_TEXT, - SMART_SELECTION_ENABLED_FOR_EDIT_TEXT_DEFAULT); - mSmartLinkifyEnabled = parser.getBoolean( - SMART_LINKIFY_ENABLED, - SMART_LINKIFY_ENABLED_DEFAULT); - } - - static TextClassifierConstants loadFromString(String settings) { - return new TextClassifierConstants(settings); - } - - public boolean isDarkLaunch() { - return mDarkLaunch; - } - - public boolean isSuggestSelectionEnabledForEditableText() { - return mSuggestSelectionEnabledForEditableText; - } - - public boolean isSmartLinkifyEnabled() { - return mSmartLinkifyEnabled; - } -} diff --git a/android/view/textclassifier/TextClassifierImpl.java b/android/view/textclassifier/TextClassifierImpl.java index 7db0e76d..7e3748ae 100644 --- a/android/view/textclassifier/TextClassifierImpl.java +++ b/android/view/textclassifier/TextClassifierImpl.java @@ -16,30 +16,39 @@ package android.view.textclassifier; +import static java.time.temporal.ChronoUnit.MILLIS; + import android.annotation.NonNull; import android.annotation.Nullable; +import android.annotation.WorkerThread; +import android.app.RemoteAction; +import android.app.SearchManager; import android.content.ComponentName; +import android.content.ContentUris; import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; import android.content.pm.ResolveInfo; -import android.graphics.drawable.Drawable; +import android.graphics.drawable.Icon; import android.net.Uri; +import android.os.Bundle; import android.os.LocaleList; import android.os.ParcelFileDescriptor; +import android.os.UserManager; import android.provider.Browser; +import android.provider.CalendarContract; import android.provider.ContactsContract; -import android.provider.Settings; -import android.text.util.Linkify; -import android.util.Patterns; import com.android.internal.annotations.GuardedBy; -import com.android.internal.logging.MetricsLogger; import com.android.internal.util.Preconditions; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.time.Instant; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -49,6 +58,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.StringJoiner; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -61,82 +72,86 @@ import java.util.regex.Pattern; * * @hide */ -final class TextClassifierImpl implements TextClassifier { +public final class TextClassifierImpl implements TextClassifier { private static final String LOG_TAG = DEFAULT_LOG_TAG; private static final String MODEL_DIR = "/etc/textclassifier/"; - private static final String MODEL_FILE_REGEX = "textclassifier\\.smartselection\\.(.*)\\.model"; + private static final String MODEL_FILE_REGEX = "textclassifier\\.(.*)\\.model"; private static final String UPDATED_MODEL_FILE_PATH = - "/data/misc/textclassifier/textclassifier.smartselection.model"; - private static final List<String> ENTITY_TYPES_ALL = - Collections.unmodifiableList(Arrays.asList( - TextClassifier.TYPE_ADDRESS, - TextClassifier.TYPE_EMAIL, - TextClassifier.TYPE_PHONE, - TextClassifier.TYPE_URL)); - private static final List<String> ENTITY_TYPES_BASE = - Collections.unmodifiableList(Arrays.asList( - TextClassifier.TYPE_ADDRESS, - TextClassifier.TYPE_EMAIL, - TextClassifier.TYPE_PHONE, - TextClassifier.TYPE_URL)); + "/data/misc/textclassifier/textclassifier.model"; private final Context mContext; - - private final MetricsLogger mMetricsLogger = new MetricsLogger(); - - private final Object mSmartSelectionLock = new Object(); - @GuardedBy("mSmartSelectionLock") // Do not access outside this lock. - private Map<Locale, String> mModelFilePaths; - @GuardedBy("mSmartSelectionLock") // Do not access outside this lock. - private Locale mLocale; - @GuardedBy("mSmartSelectionLock") // Do not access outside this lock. - private int mVersion; - @GuardedBy("mSmartSelectionLock") // Do not access outside this lock. - private SmartSelection mSmartSelection; - - private TextClassifierConstants mSettings; - - TextClassifierImpl(Context context) { + private final TextClassifier mFallback; + private final GenerateLinksLogger mGenerateLinksLogger; + + private final Object mLock = new Object(); + @GuardedBy("mLock") // Do not access outside this lock. + private List<ModelFile> mAllModelFiles; + @GuardedBy("mLock") // Do not access outside this lock. + private ModelFile mModel; + @GuardedBy("mLock") // Do not access outside this lock. + private TextClassifierImplNative mNative; + + private final Object mLoggerLock = new Object(); + @GuardedBy("mLoggerLock") // Do not access outside this lock. + private Logger.Config mLoggerConfig; + @GuardedBy("mLoggerLock") // Do not access outside this lock. + private Logger mLogger; + @GuardedBy("mLoggerLock") // Do not access outside this lock. + private Logger mLogger2; // This is the new logger. Will replace mLogger. + + private final TextClassificationConstants mSettings; + + public TextClassifierImpl(Context context, TextClassificationConstants settings) { mContext = Preconditions.checkNotNull(context); + mFallback = TextClassifier.NO_OP; + mSettings = Preconditions.checkNotNull(settings); + mGenerateLinksLogger = new GenerateLinksLogger(mSettings.getGenerateLinksLogSampleRate()); } + /** @inheritDoc */ @Override - public TextSelection suggestSelection( - @NonNull CharSequence text, int selectionStartIndex, int selectionEndIndex, - @NonNull TextSelection.Options options) { - Utils.validateInput(text, selectionStartIndex, selectionEndIndex); + @WorkerThread + public TextSelection suggestSelection(TextSelection.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); try { - if (text.length() > 0) { - final LocaleList locales = (options == null) ? null : options.getDefaultLocales(); - final SmartSelection smartSelection = getSmartSelection(locales); - final String string = text.toString(); + final int rangeLength = request.getEndIndex() - request.getStartIndex(); + final String string = request.getText().toString(); + if (string.length() > 0 + && rangeLength <= mSettings.getSuggestSelectionMaxRangeLength()) { + final String localesString = concatenateLocales(request.getDefaultLocales()); + final ZonedDateTime refTime = ZonedDateTime.now(); + final TextClassifierImplNative nativeImpl = getNative(request.getDefaultLocales()); final int start; final int end; - if (getSettings().isDarkLaunch() && !options.isDarkLaunchAllowed()) { - start = selectionStartIndex; - end = selectionEndIndex; + if (mSettings.isModelDarkLaunchEnabled() && !request.isDarkLaunchAllowed()) { + start = request.getStartIndex(); + end = request.getEndIndex(); } else { - final int[] startEnd = smartSelection.suggest( - string, selectionStartIndex, selectionEndIndex); + final int[] startEnd = nativeImpl.suggestSelection( + string, request.getStartIndex(), request.getEndIndex(), + new TextClassifierImplNative.SelectionOptions(localesString)); start = startEnd[0]; end = startEnd[1]; } - if (start <= end + if (start < end && start >= 0 && end <= string.length() - && start <= selectionStartIndex && end >= selectionEndIndex) { + && start <= request.getStartIndex() && end >= request.getEndIndex()) { final TextSelection.Builder tsBuilder = new TextSelection.Builder(start, end); - final SmartSelection.ClassificationResult[] results = - smartSelection.classifyText( + final TextClassifierImplNative.ClassificationResult[] results = + nativeImpl.classifyText( string, start, end, - getHintFlags(string, start, end)); + new TextClassifierImplNative.ClassificationOptions( + refTime.toInstant().toEpochMilli(), + refTime.getZone().getId(), + localesString)); final int size = results.length; for (int i = 0; i < size; i++) { - tsBuilder.setEntityType(results[i].mCollection, results[i].mScore); + tsBuilder.setEntityType(results[i].getCollection(), results[i].getScore()); } - return tsBuilder - .setSignature( - getSignature(string, selectionStartIndex, selectionEndIndex)) + return tsBuilder.setId(createId( + string, request.getStartIndex(), request.getEndIndex())) .build(); } else { // We can not trust the result. Log the issue and ignore the result. @@ -150,26 +165,34 @@ final class TextClassifierImpl implements TextClassifier { t); } // Getting here means something went wrong, return a NO_OP result. - return TextClassifier.NO_OP.suggestSelection( - text, selectionStartIndex, selectionEndIndex, options); + return mFallback.suggestSelection(request); } + /** @inheritDoc */ @Override - public TextClassification classifyText( - @NonNull CharSequence text, int startIndex, int endIndex, - @NonNull TextClassification.Options options) { - Utils.validateInput(text, startIndex, endIndex); + @WorkerThread + public TextClassification classifyText(TextClassification.Request request) { + Preconditions.checkNotNull(request); + Utils.checkMainThread(); try { - if (text.length() > 0) { - final String string = text.toString(); - final LocaleList locales = (options == null) ? null : options.getDefaultLocales(); - final SmartSelection.ClassificationResult[] results = getSmartSelection(locales) - .classifyText(string, startIndex, endIndex, - getHintFlags(string, startIndex, endIndex)); + final int rangeLength = request.getEndIndex() - request.getStartIndex(); + final String string = request.getText().toString(); + if (string.length() > 0 && rangeLength <= mSettings.getClassifyTextMaxRangeLength()) { + final String localesString = concatenateLocales(request.getDefaultLocales()); + final ZonedDateTime refTime = request.getReferenceTime() != null + ? request.getReferenceTime() : ZonedDateTime.now(); + final TextClassifierImplNative.ClassificationResult[] results = + getNative(request.getDefaultLocales()) + .classifyText( + string, request.getStartIndex(), request.getEndIndex(), + new TextClassifierImplNative.ClassificationOptions( + refTime.toInstant().toEpochMilli(), + refTime.getZone().getId(), + localesString)); if (results.length > 0) { - final TextClassification classificationResult = - createClassificationResult(results, string, startIndex, endIndex); - return classificationResult; + return createClassificationResult( + results, string, + request.getStartIndex(), request.getEndIndex(), refTime.toInstant()); } } } catch (Throwable t) { @@ -177,427 +200,606 @@ final class TextClassifierImpl implements TextClassifier { Log.e(LOG_TAG, "Error getting text classification info.", t); } // Getting here means something went wrong, return a NO_OP result. - return TextClassifier.NO_OP.classifyText(text, startIndex, endIndex, options); + return mFallback.classifyText(request); } + /** @inheritDoc */ @Override - public TextLinks generateLinks( - @NonNull CharSequence text, @Nullable TextLinks.Options options) { - Utils.validateInput(text); - final String textString = text.toString(); - final TextLinks.Builder builder = new TextLinks.Builder(textString); - - if (!getSettings().isSmartLinkifyEnabled()) { - return builder.build(); + @WorkerThread + public TextLinks generateLinks(@NonNull TextLinks.Request request) { + Preconditions.checkNotNull(request); + Utils.checkTextLength(request.getText(), getMaxGenerateLinksTextLength()); + Utils.checkMainThread(); + + if (!mSettings.isSmartLinkifyEnabled() && request.isLegacyFallback()) { + return Utils.generateLegacyLinks(request); } + final String textString = request.getText().toString(); + final TextLinks.Builder builder = new TextLinks.Builder(textString); + try { - final LocaleList defaultLocales = options != null ? options.getDefaultLocales() : null; - final Collection<String> entitiesToIdentify = - options != null && options.getEntityConfig() != null - ? options.getEntityConfig().getEntities(this) : ENTITY_TYPES_ALL; - final SmartSelection smartSelection = getSmartSelection(defaultLocales); - final SmartSelection.AnnotatedSpan[] annotations = smartSelection.annotate(textString); - for (SmartSelection.AnnotatedSpan span : annotations) { - final SmartSelection.ClassificationResult[] results = span.getClassification(); - if (results.length == 0 || !entitiesToIdentify.contains(results[0].mCollection)) { + final long startTimeMs = System.currentTimeMillis(); + final ZonedDateTime refTime = ZonedDateTime.now(); + final Collection<String> entitiesToIdentify = request.getEntityConfig() != null + ? request.getEntityConfig().resolveEntityListModifications( + getEntitiesForHints(request.getEntityConfig().getHints())) + : mSettings.getEntityListDefault(); + final TextClassifierImplNative nativeImpl = + getNative(request.getDefaultLocales()); + final TextClassifierImplNative.AnnotatedSpan[] annotations = + nativeImpl.annotate( + textString, + new TextClassifierImplNative.AnnotationOptions( + refTime.toInstant().toEpochMilli(), + refTime.getZone().getId(), + concatenateLocales(request.getDefaultLocales()))); + for (TextClassifierImplNative.AnnotatedSpan span : annotations) { + final TextClassifierImplNative.ClassificationResult[] results = + span.getClassification(); + if (results.length == 0 + || !entitiesToIdentify.contains(results[0].getCollection())) { continue; } final Map<String, Float> entityScores = new HashMap<>(); for (int i = 0; i < results.length; i++) { - entityScores.put(results[i].mCollection, results[i].mScore); + entityScores.put(results[i].getCollection(), results[i].getScore()); } - builder.addLink(new TextLinks.TextLink( - textString, span.getStartIndex(), span.getEndIndex(), entityScores)); + builder.addLink(span.getStartIndex(), span.getEndIndex(), entityScores); } + final TextLinks links = builder.build(); + final long endTimeMs = System.currentTimeMillis(); + final String callingPackageName = request.getCallingPackageName() == null + ? mContext.getPackageName() // local (in process) TC. + : request.getCallingPackageName(); + mGenerateLinksLogger.logGenerateLinks( + request.getText(), links, callingPackageName, endTimeMs - startTimeMs); + return links; } catch (Throwable t) { // Avoid throwing from this method. Log the error. Log.e(LOG_TAG, "Error getting links info.", t); } - return builder.build(); + return mFallback.generateLinks(request); } + /** @inheritDoc */ @Override - public Collection<String> getEntitiesForPreset(@TextClassifier.EntityPreset int entityPreset) { - switch (entityPreset) { - case TextClassifier.ENTITY_PRESET_NONE: - return Collections.emptyList(); - case TextClassifier.ENTITY_PRESET_BASE: - return ENTITY_TYPES_BASE; - case TextClassifier.ENTITY_PRESET_ALL: - // fall through - default: - return ENTITY_TYPES_ALL; + public int getMaxGenerateLinksTextLength() { + return mSettings.getGenerateLinksMaxTextLength(); + } + + private Collection<String> getEntitiesForHints(Collection<String> hints) { + final boolean editable = hints.contains(HINT_TEXT_IS_EDITABLE); + final boolean notEditable = hints.contains(HINT_TEXT_IS_NOT_EDITABLE); + + // Use the default if there is no hint, or conflicting ones. + final boolean useDefault = editable == notEditable; + if (useDefault) { + return mSettings.getEntityListDefault(); + } else if (editable) { + return mSettings.getEntityListEditable(); + } else { // notEditable + return mSettings.getEntityListNotEditable(); } } + /** @inheritDoc */ @Override - public void logEvent(String source, String event) { - if (LOG_TAG.equals(source)) { - mMetricsLogger.count(event, 1); + public Logger getLogger(@NonNull Logger.Config config) { + Preconditions.checkNotNull(config); + synchronized (mLoggerLock) { + if (mLogger == null || !config.equals(mLoggerConfig)) { + mLoggerConfig = config; + mLogger = new DefaultLogger(config); + } } + return mLogger; } @Override - public TextClassifierConstants getSettings() { - if (mSettings == null) { - mSettings = TextClassifierConstants.loadFromString(Settings.Global.getString( - mContext.getContentResolver(), Settings.Global.TEXT_CLASSIFIER_CONSTANTS)); + public void onSelectionEvent(SelectionEvent event) { + Preconditions.checkNotNull(event); + synchronized (mLoggerLock) { + if (mLogger2 == null) { + mLogger2 = new DefaultLogger( + new Logger.Config(mContext, WIDGET_TYPE_UNKNOWN, null)); + } + mLogger2.writeEvent(event); } - return mSettings; } - private SmartSelection getSmartSelection(LocaleList localeList) throws FileNotFoundException { - synchronized (mSmartSelectionLock) { + private TextClassifierImplNative getNative(LocaleList localeList) + throws FileNotFoundException { + synchronized (mLock) { localeList = localeList == null ? LocaleList.getEmptyLocaleList() : localeList; - final Locale locale = findBestSupportedLocaleLocked(localeList); - if (locale == null) { - throw new FileNotFoundException("No file for null locale"); + final ModelFile bestModel = findBestModelLocked(localeList); + if (bestModel == null) { + throw new FileNotFoundException("No model for " + localeList.toLanguageTags()); } - if (mSmartSelection == null || !Objects.equals(mLocale, locale)) { - destroySmartSelectionIfExistsLocked(); - final ParcelFileDescriptor fd = getFdLocked(locale); - final int modelFd = fd.getFd(); - mVersion = SmartSelection.getVersion(modelFd); - mSmartSelection = new SmartSelection(modelFd); + if (mNative == null || !Objects.equals(mModel, bestModel)) { + Log.d(DEFAULT_LOG_TAG, "Loading " + bestModel); + destroyNativeIfExistsLocked(); + final ParcelFileDescriptor fd = ParcelFileDescriptor.open( + new File(bestModel.getPath()), ParcelFileDescriptor.MODE_READ_ONLY); + mNative = new TextClassifierImplNative(fd.getFd()); closeAndLogError(fd); - mLocale = locale; + mModel = bestModel; } - return mSmartSelection; + return mNative; } } - private String getSignature(String text, int start, int end) { - synchronized (mSmartSelectionLock) { - final String versionInfo = (mLocale != null) - ? String.format(Locale.US, "%s_v%d", mLocale.toLanguageTag(), mVersion) - : ""; - final int hash = Objects.hash(text, start, end, mContext.getPackageName()); - return String.format(Locale.US, "%s|%s|%d", LOG_TAG, versionInfo, hash); + private String createId(String text, int start, int end) { + synchronized (mLock) { + return DefaultLogger.createId(text, start, end, mContext, mModel.getVersion(), + mModel.getSupportedLocales()); } } - @GuardedBy("mSmartSelectionLock") // Do not call outside this lock. - private ParcelFileDescriptor getFdLocked(Locale locale) throws FileNotFoundException { - ParcelFileDescriptor updateFd; - int updateVersion = -1; - try { - updateFd = ParcelFileDescriptor.open( - new File(UPDATED_MODEL_FILE_PATH), ParcelFileDescriptor.MODE_READ_ONLY); - if (updateFd != null) { - updateVersion = SmartSelection.getVersion(updateFd.getFd()); - } - } catch (FileNotFoundException e) { - updateFd = null; - } - ParcelFileDescriptor factoryFd; - int factoryVersion = -1; - try { - final String factoryModelFilePath = getFactoryModelFilePathsLocked().get(locale); - if (factoryModelFilePath != null) { - factoryFd = ParcelFileDescriptor.open( - new File(factoryModelFilePath), ParcelFileDescriptor.MODE_READ_ONLY); - if (factoryFd != null) { - factoryVersion = SmartSelection.getVersion(factoryFd.getFd()); - } - } else { - factoryFd = null; - } - } catch (FileNotFoundException e) { - factoryFd = null; - } - - if (updateFd == null) { - if (factoryFd != null) { - return factoryFd; - } else { - throw new FileNotFoundException( - String.format("No model file found for %s", locale)); - } - } - - final int updateFdInt = updateFd.getFd(); - final boolean localeMatches = Objects.equals( - locale.getLanguage().trim().toLowerCase(), - SmartSelection.getLanguage(updateFdInt).trim().toLowerCase()); - if (factoryFd == null) { - if (localeMatches) { - return updateFd; - } else { - closeAndLogError(updateFd); - throw new FileNotFoundException( - String.format("No model file found for %s", locale)); - } - } - - if (!localeMatches) { - closeAndLogError(updateFd); - return factoryFd; - } - - if (updateVersion > factoryVersion) { - closeAndLogError(factoryFd); - return updateFd; - } else { - closeAndLogError(updateFd); - return factoryFd; + @GuardedBy("mLock") // Do not call outside this lock. + private void destroyNativeIfExistsLocked() { + if (mNative != null) { + mNative.close(); + mNative = null; } } - @GuardedBy("mSmartSelectionLock") // Do not call outside this lock. - private void destroySmartSelectionIfExistsLocked() { - if (mSmartSelection != null) { - mSmartSelection.close(); - mSmartSelection = null; - } + private static String concatenateLocales(@Nullable LocaleList locales) { + return (locales == null) ? "" : locales.toLanguageTags(); } - @GuardedBy("mSmartSelectionLock") // Do not call outside this lock. + /** + * Finds the most appropriate model to use for the given target locale list. + * + * The basic logic is: we ignore all models that don't support any of the target locales. For + * the remaining candidates, we take the update model unless its version number is lower than + * the factory version. It's assumed that factory models do not have overlapping locale ranges + * and conflict resolution between these models hence doesn't matter. + */ + @GuardedBy("mLock") // Do not call outside this lock. @Nullable - private Locale findBestSupportedLocaleLocked(LocaleList localeList) { + private ModelFile findBestModelLocked(LocaleList localeList) { // Specified localeList takes priority over the system default, so it is listed first. final String languages = localeList.isEmpty() ? LocaleList.getDefault().toLanguageTags() : localeList.toLanguageTags() + "," + LocaleList.getDefault().toLanguageTags(); final List<Locale.LanguageRange> languageRangeList = Locale.LanguageRange.parse(languages); - final List<Locale> supportedLocales = - new ArrayList<>(getFactoryModelFilePathsLocked().keySet()); - final Locale updatedModelLocale = getUpdatedModelLocale(); - if (updatedModelLocale != null) { - supportedLocales.add(updatedModelLocale); + ModelFile bestModel = null; + for (ModelFile model : listAllModelsLocked()) { + if (model.isAnyLanguageSupported(languageRangeList)) { + if (model.isPreferredTo(bestModel)) { + bestModel = model; + } + } } - return Locale.lookup(languageRangeList, supportedLocales); + return bestModel; } - @GuardedBy("mSmartSelectionLock") // Do not call outside this lock. - private Map<Locale, String> getFactoryModelFilePathsLocked() { - if (mModelFilePaths == null) { - final Map<Locale, String> modelFilePaths = new HashMap<>(); + /** Returns a list of all model files available, in order of precedence. */ + @GuardedBy("mLock") // Do not call outside this lock. + private List<ModelFile> listAllModelsLocked() { + if (mAllModelFiles == null) { + final List<ModelFile> allModels = new ArrayList<>(); + // The update model has the highest precedence. + if (new File(UPDATED_MODEL_FILE_PATH).exists()) { + final ModelFile updatedModel = ModelFile.fromPath(UPDATED_MODEL_FILE_PATH); + if (updatedModel != null) { + allModels.add(updatedModel); + } + } + // Factory models should never have overlapping locales, so the order doesn't matter. final File modelsDir = new File(MODEL_DIR); if (modelsDir.exists() && modelsDir.isDirectory()) { - final File[] models = modelsDir.listFiles(); + final File[] modelFiles = modelsDir.listFiles(); final Pattern modelFilenamePattern = Pattern.compile(MODEL_FILE_REGEX); - final int size = models.length; - for (int i = 0; i < size; i++) { - final File modelFile = models[i]; + for (File modelFile : modelFiles) { final Matcher matcher = modelFilenamePattern.matcher(modelFile.getName()); if (matcher.matches() && modelFile.isFile()) { - final String language = matcher.group(1); - final Locale locale = Locale.forLanguageTag(language); - modelFilePaths.put(locale, modelFile.getAbsolutePath()); + final ModelFile model = ModelFile.fromPath(modelFile.getAbsolutePath()); + if (model != null) { + allModels.add(model); + } } } } - mModelFilePaths = modelFilePaths; - } - return mModelFilePaths; - } - - @Nullable - private Locale getUpdatedModelLocale() { - try { - final ParcelFileDescriptor updateFd = ParcelFileDescriptor.open( - new File(UPDATED_MODEL_FILE_PATH), ParcelFileDescriptor.MODE_READ_ONLY); - final Locale locale = Locale.forLanguageTag( - SmartSelection.getLanguage(updateFd.getFd())); - closeAndLogError(updateFd); - return locale; - } catch (FileNotFoundException e) { - return null; + mAllModelFiles = allModels; } + return mAllModelFiles; } private TextClassification createClassificationResult( - SmartSelection.ClassificationResult[] classifications, - String text, int start, int end) { + TextClassifierImplNative.ClassificationResult[] classifications, + String text, int start, int end, @Nullable Instant referenceTime) { final String classifiedText = text.substring(start, end); final TextClassification.Builder builder = new TextClassification.Builder() .setText(classifiedText); final int size = classifications.length; + TextClassifierImplNative.ClassificationResult highestScoringResult = null; + float highestScore = Float.MIN_VALUE; for (int i = 0; i < size; i++) { - builder.setEntityType(classifications[i].mCollection, classifications[i].mScore); + builder.setEntityType(classifications[i].getCollection(), + classifications[i].getScore()); + if (classifications[i].getScore() > highestScore) { + highestScoringResult = classifications[i]; + highestScore = classifications[i].getScore(); + } } - final String type = getHighestScoringType(classifications); - addActions(builder, IntentFactory.create(mContext, type, classifiedText)); + boolean isPrimaryAction = true; + for (LabeledIntent labeledIntent : IntentFactory.create( + mContext, referenceTime, highestScoringResult, classifiedText)) { + RemoteAction action = labeledIntent.asRemoteAction(mContext); + if (isPrimaryAction) { + // For O backwards compatibility, the first RemoteAction is also written to the + // legacy API fields. + builder.setIcon(action.getIcon().loadDrawable(mContext)); + builder.setLabel(action.getTitle().toString()); + builder.setIntent(labeledIntent.getIntent()); + builder.setOnClickListener(TextClassification.createIntentOnClickListener( + TextClassification.createPendingIntent(mContext, + labeledIntent.getIntent()))); + isPrimaryAction = false; + } + builder.addAction(action); + } - return builder.setSignature(getSignature(text, start, end)).build(); + return builder.setId(createId(text, start, end)).build(); } - /** Extends the classification with the intents that can be resolved. */ - private void addActions( - TextClassification.Builder builder, List<Intent> intents) { - final PackageManager pm = mContext.getPackageManager(); - final int size = intents.size(); - for (int i = 0; i < size; i++) { - final Intent intent = intents.get(i); - final ResolveInfo resolveInfo; - if (intent != null) { - resolveInfo = pm.resolveActivity(intent, 0); - } else { - resolveInfo = null; - } - if (resolveInfo != null && resolveInfo.activityInfo != null) { - final String packageName = resolveInfo.activityInfo.packageName; - CharSequence label; - Drawable icon; - if ("android".equals(packageName)) { - // Requires the chooser to find an activity to handle the intent. - label = IntentFactory.getLabel(mContext, intent); - icon = null; - } else { - // A default activity will handle the intent. - intent.setComponent( - new ComponentName(packageName, resolveInfo.activityInfo.name)); - icon = resolveInfo.activityInfo.loadIcon(pm); - if (icon == null) { - icon = resolveInfo.loadIcon(pm); - } - label = resolveInfo.activityInfo.loadLabel(pm); - if (label == null) { - label = resolveInfo.loadLabel(pm); - } + /** + * Closes the ParcelFileDescriptor and logs any errors that occur. + */ + private static void closeAndLogError(ParcelFileDescriptor fd) { + try { + fd.close(); + } catch (IOException e) { + Log.e(LOG_TAG, "Error closing file.", e); + } + } + + /** + * Describes TextClassifier model files on disk. + */ + private static final class ModelFile { + + private final String mPath; + private final String mName; + private final int mVersion; + private final List<Locale> mSupportedLocales; + private final boolean mLanguageIndependent; + + /** Returns null if the path did not point to a compatible model. */ + static @Nullable ModelFile fromPath(String path) { + final File file = new File(path); + try { + final ParcelFileDescriptor modelFd = ParcelFileDescriptor.open( + file, ParcelFileDescriptor.MODE_READ_ONLY); + final int version = TextClassifierImplNative.getVersion(modelFd.getFd()); + final String supportedLocalesStr = + TextClassifierImplNative.getLocales(modelFd.getFd()); + if (supportedLocalesStr.isEmpty()) { + Log.d(DEFAULT_LOG_TAG, "Ignoring " + file.getAbsolutePath()); + return null; } - final String labelString = (label != null) ? label.toString() : null; - if (i == 0) { - builder.setPrimaryAction(intent, labelString, icon); - } else { - builder.addSecondaryAction(intent, labelString, icon); + final boolean languageIndependent = supportedLocalesStr.equals("*"); + final List<Locale> supportedLocales = new ArrayList<>(); + for (String langTag : supportedLocalesStr.split(",")) { + supportedLocales.add(Locale.forLanguageTag(langTag)); } + closeAndLogError(modelFd); + return new ModelFile(path, file.getName(), version, supportedLocales, + languageIndependent); + } catch (FileNotFoundException e) { + Log.e(DEFAULT_LOG_TAG, "Failed to peek " + file.getAbsolutePath(), e); + return null; } } - } - private static int getHintFlags(CharSequence text, int start, int end) { - int flag = 0; - final CharSequence subText = text.subSequence(start, end); - if (Patterns.AUTOLINK_EMAIL_ADDRESS.matcher(subText).matches()) { - flag |= SmartSelection.HINT_FLAG_EMAIL; + /** The absolute path to the model file. */ + String getPath() { + return mPath; } - if (Patterns.AUTOLINK_WEB_URL.matcher(subText).matches() - && Linkify.sUrlMatchFilter.acceptMatch(text, start, end)) { - flag |= SmartSelection.HINT_FLAG_URL; + + /** A name to use for id generation. Effectively the name of the model file. */ + String getName() { + return mName; } - return flag; - } - private static String getHighestScoringType(SmartSelection.ClassificationResult[] types) { - if (types.length < 1) { - return ""; + /** Returns the version tag in the model's metadata. */ + int getVersion() { + return mVersion; } - String type = types[0].mCollection; - float highestScore = types[0].mScore; - final int size = types.length; - for (int i = 1; i < size; i++) { - if (types[i].mScore > highestScore) { - type = types[i].mCollection; - highestScore = types[i].mScore; + /** Returns whether the language supports any language in the given ranges. */ + boolean isAnyLanguageSupported(List<Locale.LanguageRange> languageRanges) { + return mLanguageIndependent || Locale.lookup(languageRanges, mSupportedLocales) != null; + } + + /** All locales supported by the model. */ + List<Locale> getSupportedLocales() { + return Collections.unmodifiableList(mSupportedLocales); + } + + public boolean isPreferredTo(ModelFile model) { + // A model is preferred to no model. + if (model == null) { + return true; + } + + // A language-specific model is preferred to a language independent + // model. + if (!mLanguageIndependent && model.mLanguageIndependent) { + return true; + } + + // A higher-version model is preferred. + if (getVersion() > model.getVersion()) { + return true; + } + return false; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other == null || !ModelFile.class.isAssignableFrom(other.getClass())) { + return false; + } else { + final ModelFile otherModel = (ModelFile) other; + return mPath.equals(otherModel.mPath); } } - return type; + + @Override + public String toString() { + final StringJoiner localesJoiner = new StringJoiner(","); + for (Locale locale : mSupportedLocales) { + localesJoiner.add(locale.toLanguageTag()); + } + return String.format(Locale.US, "ModelFile { path=%s name=%s version=%d locales=%s }", + mPath, mName, mVersion, localesJoiner.toString()); + } + + private ModelFile(String path, String name, int version, List<Locale> supportedLocales, + boolean languageIndependent) { + mPath = path; + mName = name; + mVersion = version; + mSupportedLocales = supportedLocales; + mLanguageIndependent = languageIndependent; + } } /** - * Closes the ParcelFileDescriptor and logs any errors that occur. + * Helper class to store the information from which RemoteActions are built. */ - private static void closeAndLogError(ParcelFileDescriptor fd) { - try { - fd.close(); - } catch (IOException e) { - Log.e(LOG_TAG, "Error closing file.", e); + private static final class LabeledIntent { + private String mTitle; + private String mDescription; + private Intent mIntent; + + LabeledIntent(String title, String description, Intent intent) { + mTitle = title; + mDescription = description; + mIntent = intent; + } + + String getTitle() { + return mTitle; + } + + String getDescription() { + return mDescription; + } + + Intent getIntent() { + return mIntent; + } + + RemoteAction asRemoteAction(Context context) { + final PackageManager pm = context.getPackageManager(); + final ResolveInfo resolveInfo = pm.resolveActivity(mIntent, 0); + final String packageName = resolveInfo != null && resolveInfo.activityInfo != null + ? resolveInfo.activityInfo.packageName : null; + Icon icon = null; + boolean shouldShowIcon = false; + if (packageName != null && !"android".equals(packageName)) { + // There is a default activity handling the intent. + mIntent.setComponent(new ComponentName(packageName, resolveInfo.activityInfo.name)); + if (resolveInfo.activityInfo.getIconResource() != 0) { + icon = Icon.createWithResource( + packageName, resolveInfo.activityInfo.getIconResource()); + shouldShowIcon = true; + } + } + if (icon == null) { + // RemoteAction requires that there be an icon. + icon = Icon.createWithResource("android", + com.android.internal.R.drawable.ic_more_items); + } + RemoteAction action = new RemoteAction(icon, mTitle, mDescription, + TextClassification.createPendingIntent(context, mIntent)); + action.setShouldShowIcon(shouldShowIcon); + return action; } } /** * Creates intents based on the classification type. */ - private static final class IntentFactory { + static final class IntentFactory { + + private static final long MIN_EVENT_FUTURE_MILLIS = TimeUnit.MINUTES.toMillis(5); + private static final long DEFAULT_EVENT_DURATION = TimeUnit.HOURS.toMillis(1); private IntentFactory() {} @NonNull - public static List<Intent> create(Context context, String type, String text) { - final List<Intent> intents = new ArrayList<>(); - type = type.trim().toLowerCase(Locale.ENGLISH); + public static List<LabeledIntent> create( + Context context, + @Nullable Instant referenceTime, + TextClassifierImplNative.ClassificationResult classification, + String text) { + final String type = classification.getCollection().trim().toLowerCase(Locale.ENGLISH); text = text.trim(); switch (type) { case TextClassifier.TYPE_EMAIL: - intents.add(new Intent(Intent.ACTION_SENDTO) - .setData(Uri.parse(String.format("mailto:%s", text)))); - intents.add(new Intent(Intent.ACTION_INSERT_OR_EDIT) - .setType(ContactsContract.Contacts.CONTENT_ITEM_TYPE) - .putExtra(ContactsContract.Intents.Insert.EMAIL, text)); - break; + return createForEmail(context, text); case TextClassifier.TYPE_PHONE: - intents.add(new Intent(Intent.ACTION_DIAL) - .setData(Uri.parse(String.format("tel:%s", text)))); - intents.add(new Intent(Intent.ACTION_INSERT_OR_EDIT) - .setType(ContactsContract.Contacts.CONTENT_ITEM_TYPE) - .putExtra(ContactsContract.Intents.Insert.PHONE, text)); - intents.add(new Intent(Intent.ACTION_SENDTO) - .setData(Uri.parse(String.format("smsto:%s", text)))); - break; + return createForPhone(context, text); case TextClassifier.TYPE_ADDRESS: - intents.add(new Intent(Intent.ACTION_VIEW) - .setData(Uri.parse(String.format("geo:0,0?q=%s", text)))); - break; + return createForAddress(context, text); case TextClassifier.TYPE_URL: - final String httpPrefix = "http://"; - final String httpsPrefix = "https://"; - if (text.toLowerCase().startsWith(httpPrefix)) { - text = httpPrefix + text.substring(httpPrefix.length()); - } else if (text.toLowerCase().startsWith(httpsPrefix)) { - text = httpsPrefix + text.substring(httpsPrefix.length()); + return createForUrl(context, text); + case TextClassifier.TYPE_DATE: + case TextClassifier.TYPE_DATE_TIME: + if (classification.getDatetimeResult() != null) { + final Instant parsedTime = Instant.ofEpochMilli( + classification.getDatetimeResult().getTimeMsUtc()); + return createForDatetime(context, type, referenceTime, parsedTime); } else { - text = httpPrefix + text; + return new ArrayList<>(); } - intents.add(new Intent(Intent.ACTION_VIEW, Uri.parse(text)) - .putExtra(Browser.EXTRA_APPLICATION_ID, context.getPackageName())); - break; + case TextClassifier.TYPE_FLIGHT_NUMBER: + return createForFlight(context, text); + default: + return new ArrayList<>(); } - return intents; } - @Nullable - public static String getLabel(Context context, @Nullable Intent intent) { - if (intent == null || intent.getAction() == null) { - return null; + @NonNull + private static List<LabeledIntent> createForEmail(Context context, String text) { + return Arrays.asList( + new LabeledIntent( + context.getString(com.android.internal.R.string.email), + context.getString(com.android.internal.R.string.email_desc), + new Intent(Intent.ACTION_SENDTO) + .setData(Uri.parse(String.format("mailto:%s", text)))), + new LabeledIntent( + context.getString(com.android.internal.R.string.add_contact), + context.getString(com.android.internal.R.string.add_contact_desc), + new Intent(Intent.ACTION_INSERT_OR_EDIT) + .setType(ContactsContract.Contacts.CONTENT_ITEM_TYPE) + .putExtra(ContactsContract.Intents.Insert.EMAIL, text))); + } + + @NonNull + private static List<LabeledIntent> createForPhone(Context context, String text) { + final List<LabeledIntent> actions = new ArrayList<>(); + final UserManager userManager = context.getSystemService(UserManager.class); + final Bundle userRestrictions = userManager != null + ? userManager.getUserRestrictions() : new Bundle(); + if (!userRestrictions.getBoolean(UserManager.DISALLOW_OUTGOING_CALLS, false)) { + actions.add(new LabeledIntent( + context.getString(com.android.internal.R.string.dial), + context.getString(com.android.internal.R.string.dial_desc), + new Intent(Intent.ACTION_DIAL).setData( + Uri.parse(String.format("tel:%s", text))))); } - switch (intent.getAction()) { - case Intent.ACTION_DIAL: - return context.getString(com.android.internal.R.string.dial); - case Intent.ACTION_SENDTO: - switch (intent.getScheme()) { - case "mailto": - return context.getString(com.android.internal.R.string.email); - case "smsto": - return context.getString(com.android.internal.R.string.sms); - default: - return null; - } - case Intent.ACTION_INSERT_OR_EDIT: - switch (intent.getDataString()) { - case ContactsContract.Contacts.CONTENT_ITEM_TYPE: - return context.getString(com.android.internal.R.string.add_contact); - default: - return null; - } - case Intent.ACTION_VIEW: - switch (intent.getScheme()) { - case "geo": - return context.getString(com.android.internal.R.string.map); - case "http": // fall through - case "https": - return context.getString(com.android.internal.R.string.browse); - default: - return null; - } - default: - return null; + actions.add(new LabeledIntent( + context.getString(com.android.internal.R.string.add_contact), + context.getString(com.android.internal.R.string.add_contact_desc), + new Intent(Intent.ACTION_INSERT_OR_EDIT) + .setType(ContactsContract.Contacts.CONTENT_ITEM_TYPE) + .putExtra(ContactsContract.Intents.Insert.PHONE, text))); + if (!userRestrictions.getBoolean(UserManager.DISALLOW_SMS, false)) { + actions.add(new LabeledIntent( + context.getString(com.android.internal.R.string.sms), + context.getString(com.android.internal.R.string.sms_desc), + new Intent(Intent.ACTION_SENDTO) + .setData(Uri.parse(String.format("smsto:%s", text))))); + } + return actions; + } + + @NonNull + private static List<LabeledIntent> createForAddress(Context context, String text) { + final List<LabeledIntent> actions = new ArrayList<>(); + try { + final String encText = URLEncoder.encode(text, "UTF-8"); + actions.add(new LabeledIntent( + context.getString(com.android.internal.R.string.map), + context.getString(com.android.internal.R.string.map_desc), + new Intent(Intent.ACTION_VIEW) + .setData(Uri.parse(String.format("geo:0,0?q=%s", encText))))); + } catch (UnsupportedEncodingException e) { + Log.e(LOG_TAG, "Could not encode address", e); + } + return actions; + } + + @NonNull + private static List<LabeledIntent> createForUrl(Context context, String text) { + final String httpPrefix = "http://"; + final String httpsPrefix = "https://"; + if (text.toLowerCase().startsWith(httpPrefix)) { + text = httpPrefix + text.substring(httpPrefix.length()); + } else if (text.toLowerCase().startsWith(httpsPrefix)) { + text = httpsPrefix + text.substring(httpsPrefix.length()); + } else { + text = httpPrefix + text; + } + return Arrays.asList(new LabeledIntent( + context.getString(com.android.internal.R.string.browse), + context.getString(com.android.internal.R.string.browse_desc), + new Intent(Intent.ACTION_VIEW, Uri.parse(text)) + .putExtra(Browser.EXTRA_APPLICATION_ID, context.getPackageName()))); + } + + @NonNull + private static List<LabeledIntent> createForDatetime( + Context context, String type, @Nullable Instant referenceTime, + Instant parsedTime) { + if (referenceTime == null) { + // If no reference time was given, use now. + referenceTime = Instant.now(); + } + List<LabeledIntent> actions = new ArrayList<>(); + actions.add(createCalendarViewIntent(context, parsedTime)); + final long millisUntilEvent = referenceTime.until(parsedTime, MILLIS); + if (millisUntilEvent > MIN_EVENT_FUTURE_MILLIS) { + actions.add(createCalendarCreateEventIntent(context, parsedTime, type)); } + return actions; + } + + @NonNull + private static List<LabeledIntent> createForFlight(Context context, String text) { + return Arrays.asList(new LabeledIntent( + context.getString(com.android.internal.R.string.view_flight), + context.getString(com.android.internal.R.string.view_flight_desc), + new Intent(Intent.ACTION_WEB_SEARCH) + .putExtra(SearchManager.QUERY, text))); + } + + @NonNull + private static LabeledIntent createCalendarViewIntent(Context context, Instant parsedTime) { + Uri.Builder builder = CalendarContract.CONTENT_URI.buildUpon(); + builder.appendPath("time"); + ContentUris.appendId(builder, parsedTime.toEpochMilli()); + return new LabeledIntent( + context.getString(com.android.internal.R.string.view_calendar), + context.getString(com.android.internal.R.string.view_calendar_desc), + new Intent(Intent.ACTION_VIEW).setData(builder.build())); + } + + @NonNull + private static LabeledIntent createCalendarCreateEventIntent( + Context context, Instant parsedTime, @EntityType String type) { + final boolean isAllDay = TextClassifier.TYPE_DATE.equals(type); + return new LabeledIntent( + context.getString(com.android.internal.R.string.add_calendar_event), + context.getString(com.android.internal.R.string.add_calendar_event_desc), + new Intent(Intent.ACTION_INSERT) + .setData(CalendarContract.Events.CONTENT_URI) + .putExtra(CalendarContract.EXTRA_EVENT_ALL_DAY, isAllDay) + .putExtra(CalendarContract.EXTRA_EVENT_BEGIN_TIME, + parsedTime.toEpochMilli()) + .putExtra(CalendarContract.EXTRA_EVENT_END_TIME, + parsedTime.toEpochMilli() + DEFAULT_EVENT_DURATION)); } } } diff --git a/android/view/textclassifier/TextClassifierImplNative.java b/android/view/textclassifier/TextClassifierImplNative.java new file mode 100644 index 00000000..3d4c8f28 --- /dev/null +++ b/android/view/textclassifier/TextClassifierImplNative.java @@ -0,0 +1,301 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.content.res.AssetFileDescriptor; + +/** + * Java wrapper for TextClassifier native library interface. This library is used for detecting + * entities in text. + */ +final class TextClassifierImplNative { + + static { + System.loadLibrary("textclassifier"); + } + + private final long mModelPtr; + + /** + * Creates a new instance of TextClassifierImplNative, using the provided model image, given as + * a file descriptor. + */ + TextClassifierImplNative(int fd) { + mModelPtr = nativeNew(fd); + if (mModelPtr == 0L) { + throw new IllegalArgumentException("Couldn't initialize TC from file descriptor."); + } + } + + /** + * Creates a new instance of TextClassifierImplNative, using the provided model image, given as + * a file path. + */ + TextClassifierImplNative(String path) { + mModelPtr = nativeNewFromPath(path); + if (mModelPtr == 0L) { + throw new IllegalArgumentException("Couldn't initialize TC from given file."); + } + } + + /** + * Creates a new instance of TextClassifierImplNative, using the provided model image, given as + * an AssetFileDescriptor. + */ + TextClassifierImplNative(AssetFileDescriptor afd) { + mModelPtr = nativeNewFromAssetFileDescriptor(afd, afd.getStartOffset(), afd.getLength()); + if (mModelPtr == 0L) { + throw new IllegalArgumentException( + "Couldn't initialize TC from given AssetFileDescriptor"); + } + } + + /** + * Given a string context and current selection, computes the SmartSelection suggestion. + * + * <p>The begin and end are character indices into the context UTF8 string. selectionBegin is + * the character index where the selection begins, and selectionEnd is the index of one + * character past the selection span. + * + * <p>The return value is an array of two ints: suggested selection beginning and end, with the + * same semantics as the input selectionBeginning and selectionEnd. + */ + public int[] suggestSelection( + String context, int selectionBegin, int selectionEnd, SelectionOptions options) { + return nativeSuggestSelection(mModelPtr, context, selectionBegin, selectionEnd, options); + } + + /** + * Given a string context and current selection, classifies the type of the selected text. + * + * <p>The begin and end params are character indices in the context string. + * + * <p>Returns an array of ClassificationResult objects with the probability scores for different + * collections. + */ + public ClassificationResult[] classifyText( + String context, int selectionBegin, int selectionEnd, ClassificationOptions options) { + return nativeClassifyText(mModelPtr, context, selectionBegin, selectionEnd, options); + } + + /** + * Annotates given input text. The annotations should cover the whole input context except for + * whitespaces, and are sorted by their position in the context string. + */ + public AnnotatedSpan[] annotate(String text, AnnotationOptions options) { + return nativeAnnotate(mModelPtr, text, options); + } + + /** Frees up the allocated memory. */ + public void close() { + nativeClose(mModelPtr); + } + + /** Returns a comma separated list of locales supported by the model as BCP 47 tags. */ + public static String getLocales(int fd) { + return nativeGetLocales(fd); + } + + /** Returns the version of the model. */ + public static int getVersion(int fd) { + return nativeGetVersion(fd); + } + + /** Represents a datetime parsing result from classifyText calls. */ + public static final class DatetimeResult { + static final int GRANULARITY_YEAR = 0; + static final int GRANULARITY_MONTH = 1; + static final int GRANULARITY_WEEK = 2; + static final int GRANULARITY_DAY = 3; + static final int GRANULARITY_HOUR = 4; + static final int GRANULARITY_MINUTE = 5; + static final int GRANULARITY_SECOND = 6; + + private final long mTimeMsUtc; + private final int mGranularity; + + DatetimeResult(long timeMsUtc, int granularity) { + mGranularity = granularity; + mTimeMsUtc = timeMsUtc; + } + + public long getTimeMsUtc() { + return mTimeMsUtc; + } + + public int getGranularity() { + return mGranularity; + } + } + + /** Represents a result of classifyText method call. */ + public static final class ClassificationResult { + private final String mCollection; + private final float mScore; + private final DatetimeResult mDatetimeResult; + + ClassificationResult( + String collection, float score, DatetimeResult datetimeResult) { + mCollection = collection; + mScore = score; + mDatetimeResult = datetimeResult; + } + + public String getCollection() { + if (mCollection.equals(TextClassifier.TYPE_DATE) && mDatetimeResult != null) { + switch (mDatetimeResult.getGranularity()) { + case DatetimeResult.GRANULARITY_HOUR: + // fall through + case DatetimeResult.GRANULARITY_MINUTE: + // fall through + case DatetimeResult.GRANULARITY_SECOND: + return TextClassifier.TYPE_DATE_TIME; + default: + return TextClassifier.TYPE_DATE; + } + } + return mCollection; + } + + public float getScore() { + return mScore; + } + + public DatetimeResult getDatetimeResult() { + return mDatetimeResult; + } + } + + /** Represents a result of Annotate call. */ + public static final class AnnotatedSpan { + private final int mStartIndex; + private final int mEndIndex; + private final ClassificationResult[] mClassification; + + AnnotatedSpan( + int startIndex, int endIndex, ClassificationResult[] classification) { + mStartIndex = startIndex; + mEndIndex = endIndex; + mClassification = classification; + } + + public int getStartIndex() { + return mStartIndex; + } + + public int getEndIndex() { + return mEndIndex; + } + + public ClassificationResult[] getClassification() { + return mClassification; + } + } + + /** Represents options for the suggestSelection call. */ + public static final class SelectionOptions { + private final String mLocales; + + SelectionOptions(String locales) { + mLocales = locales; + } + + public String getLocales() { + return mLocales; + } + } + + /** Represents options for the classifyText call. */ + public static final class ClassificationOptions { + private final long mReferenceTimeMsUtc; + private final String mReferenceTimezone; + private final String mLocales; + + ClassificationOptions(long referenceTimeMsUtc, String referenceTimezone, String locale) { + mReferenceTimeMsUtc = referenceTimeMsUtc; + mReferenceTimezone = referenceTimezone; + mLocales = locale; + } + + public long getReferenceTimeMsUtc() { + return mReferenceTimeMsUtc; + } + + public String getReferenceTimezone() { + return mReferenceTimezone; + } + + public String getLocale() { + return mLocales; + } + } + + /** Represents options for the Annotate call. */ + public static final class AnnotationOptions { + private final long mReferenceTimeMsUtc; + private final String mReferenceTimezone; + private final String mLocales; + + AnnotationOptions(long referenceTimeMsUtc, String referenceTimezone, String locale) { + mReferenceTimeMsUtc = referenceTimeMsUtc; + mReferenceTimezone = referenceTimezone; + mLocales = locale; + } + + public long getReferenceTimeMsUtc() { + return mReferenceTimeMsUtc; + } + + public String getReferenceTimezone() { + return mReferenceTimezone; + } + + public String getLocale() { + return mLocales; + } + } + + private static native long nativeNew(int fd); + + private static native long nativeNewFromPath(String path); + + private static native long nativeNewFromAssetFileDescriptor( + AssetFileDescriptor afd, long offset, long size); + + private static native int[] nativeSuggestSelection( + long context, + String text, + int selectionBegin, + int selectionEnd, + SelectionOptions options); + + private static native ClassificationResult[] nativeClassifyText( + long context, + String text, + int selectionBegin, + int selectionEnd, + ClassificationOptions options); + + private static native AnnotatedSpan[] nativeAnnotate( + long context, String text, AnnotationOptions options); + + private static native void nativeClose(long context); + + private static native String nativeGetLocales(int fd); + + private static native int nativeGetVersion(int fd); +} diff --git a/android/view/textclassifier/TextLinks.java b/android/view/textclassifier/TextLinks.java index ba854e04..17c7b13c 100644 --- a/android/view/textclassifier/TextLinks.java +++ b/android/view/textclassifier/TextLinks.java @@ -17,22 +17,32 @@ package android.view.textclassifier; import android.annotation.FloatRange; +import android.annotation.IntDef; import android.annotation.NonNull; import android.annotation.Nullable; +import android.content.Context; import android.os.LocaleList; import android.os.Parcel; import android.os.Parcelable; -import android.text.SpannableString; +import android.text.Spannable; +import android.text.method.MovementMethod; import android.text.style.ClickableSpan; +import android.text.style.URLSpan; import android.view.View; +import android.view.textclassifier.TextClassifier.EntityType; import android.widget.TextView; +import com.android.internal.annotations.VisibleForTesting; +import com.android.internal.annotations.VisibleForTesting.Visibility; import com.android.internal.util.Preconditions; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Function; @@ -41,48 +51,102 @@ import java.util.function.Function; * address, url, etc) they may be. */ public final class TextLinks implements Parcelable { + + /** + * Return status of an attempt to apply TextLinks to text. + * @hide + */ + @Retention(RetentionPolicy.SOURCE) + @IntDef({STATUS_LINKS_APPLIED, STATUS_NO_LINKS_FOUND, STATUS_NO_LINKS_APPLIED, + STATUS_DIFFERENT_TEXT}) + public @interface Status {} + + /** Links were successfully applied to the text. */ + public static final int STATUS_LINKS_APPLIED = 0; + + /** No links exist to apply to text. Links count is zero. */ + public static final int STATUS_NO_LINKS_FOUND = 1; + + /** No links applied to text. The links were filtered out. */ + public static final int STATUS_NO_LINKS_APPLIED = 2; + + /** The specified text does not match the text used to generate the links. */ + public static final int STATUS_DIFFERENT_TEXT = 3; + + /** @hide */ + @Retention(RetentionPolicy.SOURCE) + @IntDef({APPLY_STRATEGY_IGNORE, APPLY_STRATEGY_REPLACE}) + public @interface ApplyStrategy {} + + /** + * Do not replace {@link ClickableSpan}s that exist where the {@link TextLinkSpan} needs to + * be applied to. Do not apply the TextLinkSpan. + */ + public static final int APPLY_STRATEGY_IGNORE = 0; + + /** + * Replace any {@link ClickableSpan}s that exist where the {@link TextLinkSpan} needs to be + * applied to. + */ + public static final int APPLY_STRATEGY_REPLACE = 1; + private final String mFullText; private final List<TextLink> mLinks; - private TextLinks(String fullText, Collection<TextLink> links) { + private TextLinks(String fullText, ArrayList<TextLink> links) { mFullText = fullText; - mLinks = Collections.unmodifiableList(new ArrayList<>(links)); + mLinks = Collections.unmodifiableList(links); + } + + /** + * Returns the text that was used to generate these links. + * @hide + */ + @NonNull + public String getText() { + return mFullText; } /** * Returns an unmodifiable Collection of the links. */ + @NonNull public Collection<TextLink> getLinks() { return mLinks; } /** * Annotates the given text with the generated links. It will fail if the provided text doesn't - * match the original text used to crete the TextLinks. + * match the original text used to create the TextLinks. * - * @param text the text to apply the links to. Must match the original text. - * @param spanFactory a factory to generate spans from TextLinks. Will use a default if null. + * <p><strong>NOTE: </strong>It may be necessary to set a LinkMovementMethod on the TextView + * widget to properly handle links. See {@link TextView#setMovementMethod(MovementMethod)} * - * @return Success or failure. + * @param text the text to apply the links to. Must match the original text + * @param applyStrategy the apply strategy used to determine how to apply links to text. + * e.g {@link TextLinks#APPLY_STRATEGY_IGNORE} + * @param spanFactory a custom span factory for converting TextLinks to TextLinkSpans. + * Set to {@code null} to use the default span factory. + * + * @return a status code indicating whether or not the links were successfully applied + * e.g. {@link #STATUS_LINKS_APPLIED} */ - public boolean apply( - @NonNull SpannableString text, - @Nullable Function<TextLink, ClickableSpan> spanFactory) { + @Status + public int apply( + @NonNull Spannable text, + @ApplyStrategy int applyStrategy, + @Nullable Function<TextLink, TextLinkSpan> spanFactory) { Preconditions.checkNotNull(text); - if (!mFullText.equals(text.toString())) { - return false; - } + return new TextLinksParams.Builder() + .setApplyStrategy(applyStrategy) + .setSpanFactory(spanFactory) + .build() + .apply(text, this); + } - if (spanFactory == null) { - spanFactory = DEFAULT_SPAN_FACTORY; - } - for (TextLink link : mLinks) { - final ClickableSpan span = spanFactory.apply(link); - if (span != null) { - text.setSpan(span, link.getStart(), link.getEnd(), 0); - } - } - return true; + @Override + public String toString() { + return String.format(Locale.US, "TextLinks{fullText=%s, links=%s}", mFullText, mLinks); } @Override @@ -119,30 +183,35 @@ public final class TextLinks implements Parcelable { */ public static final class TextLink implements Parcelable { private final EntityConfidence mEntityScores; - private final String mOriginalText; private final int mStart; private final int mEnd; + @Nullable final URLSpan mUrlSpan; /** * Create a new TextLink. * - * @throws IllegalArgumentException if entityScores is null or empty. + * @param start The start index of the identified subsequence + * @param end The end index of the identified subsequence + * @param entityScores A mapping of entity type to confidence score + * @param urlSpan An optional URLSpan to delegate to. NOTE: Not parcelled + * + * @throws IllegalArgumentException if entityScores is null or empty */ - public TextLink(String originalText, int start, int end, Map<String, Float> entityScores) { - Preconditions.checkNotNull(originalText); + TextLink(int start, int end, Map<String, Float> entityScores, + @Nullable URLSpan urlSpan) { Preconditions.checkNotNull(entityScores); Preconditions.checkArgument(!entityScores.isEmpty()); Preconditions.checkArgument(start <= end); - mOriginalText = originalText; mStart = start; mEnd = end; mEntityScores = new EntityConfidence(entityScores); + mUrlSpan = urlSpan; } /** * Returns the start index of this link in the original text. * - * @return the start index. + * @return the start index */ public int getStart() { return mStart; @@ -151,7 +220,7 @@ public final class TextLinks implements Parcelable { /** * Returns the end index of this link in the original text. * - * @return the end index. + * @return the end index */ public int getEnd() { return mEnd; @@ -160,7 +229,7 @@ public final class TextLinks implements Parcelable { /** * Returns the number of entity types that have confidence scores. * - * @return the entity count. + * @return the entity count */ public int getEntityCount() { return mEntityScores.getEntities().size(); @@ -169,23 +238,30 @@ public final class TextLinks implements Parcelable { /** * Returns the entity type at a given index. Entity types are sorted by confidence. * - * @return the entity type at the provided index. + * @return the entity type at the provided index */ - @NonNull public @TextClassifier.EntityType String getEntity(int index) { + @NonNull public @EntityType String getEntity(int index) { return mEntityScores.getEntities().get(index); } /** * Returns the confidence score for a particular entity type. * - * @param entityType the entity type. + * @param entityType the entity type */ public @FloatRange(from = 0.0, to = 1.0) float getConfidenceScore( - @TextClassifier.EntityType String entityType) { + @EntityType String entityType) { return mEntityScores.getConfidenceScore(entityType); } @Override + public String toString() { + return String.format(Locale.US, + "TextLink{start=%s, end=%s, entityScores=%s, urlSpan=%s}", + mStart, mEnd, mEntityScores, mUrlSpan); + } + + @Override public int describeContents() { return 0; } @@ -193,7 +269,6 @@ public final class TextLinks implements Parcelable { @Override public void writeToParcel(Parcel dest, int flags) { mEntityScores.writeToParcel(dest, flags); - dest.writeString(mOriginalText); dest.writeInt(mStart); dest.writeInt(mEnd); } @@ -213,46 +288,47 @@ public final class TextLinks implements Parcelable { private TextLink(Parcel in) { mEntityScores = EntityConfidence.CREATOR.createFromParcel(in); - mOriginalText = in.readString(); mStart = in.readInt(); mEnd = in.readInt(); + mUrlSpan = null; } } /** - * Optional input parameters for generating TextLinks. + * A request object for generating TextLinks. */ - public static final class Options implements Parcelable { - - private LocaleList mDefaultLocales; - private TextClassifier.EntityConfig mEntityConfig; - - public Options() {} - - /** - * @param defaultLocales ordered list of locale preferences that may be used to - * disambiguate the provided text. If no locale preferences exist, - * set this to null or an empty locale list. - */ - public Options setDefaultLocales(@Nullable LocaleList defaultLocales) { + public static final class Request implements Parcelable { + + private final CharSequence mText; + @Nullable private final LocaleList mDefaultLocales; + @Nullable private final TextClassifier.EntityConfig mEntityConfig; + private final boolean mLegacyFallback; + private String mCallingPackageName; + + private Request( + CharSequence text, + LocaleList defaultLocales, + TextClassifier.EntityConfig entityConfig, + boolean legacyFallback, + String callingPackageName) { + mText = text; mDefaultLocales = defaultLocales; - return this; + mEntityConfig = entityConfig; + mLegacyFallback = legacyFallback; + mCallingPackageName = callingPackageName; } /** - * Sets the entity configuration to use. This determines what types of entities the - * TextClassifier will look for. - * - * @param entityConfig EntityConfig to use + * Returns the text to generate links for. */ - public Options setEntityConfig(@Nullable TextClassifier.EntityConfig entityConfig) { - mEntityConfig = entityConfig; - return this; + @NonNull + public CharSequence getText() { + return mText; } /** * @return ordered list of locale preferences that can be used to disambiguate - * the provided text. + * the provided text */ @Nullable public LocaleList getDefaultLocales() { @@ -260,7 +336,7 @@ public final class TextLinks implements Parcelable { } /** - * @return The config representing the set of entities to look for. + * @return The config representing the set of entities to look for * @see #setEntityConfig(TextClassifier.EntityConfig) */ @Nullable @@ -268,6 +344,114 @@ public final class TextLinks implements Parcelable { return mEntityConfig; } + /** + * Returns whether the TextClassifier can fallback to legacy links if smart linkify is + * disabled. + * <strong>Note: </strong>This is not parcelled. + * @hide + */ + public boolean isLegacyFallback() { + return mLegacyFallback; + } + + /** + * Sets the name of the package that requested the links to get generated. + */ + void setCallingPackageName(@Nullable String callingPackageName) { + mCallingPackageName = callingPackageName; + } + + /** + * A builder for building TextLinks requests. + */ + public static final class Builder { + + private final CharSequence mText; + + @Nullable private LocaleList mDefaultLocales; + @Nullable private TextClassifier.EntityConfig mEntityConfig; + private boolean mLegacyFallback = true; // Use legacy fall back by default. + private String mCallingPackageName; + + public Builder(@NonNull CharSequence text) { + mText = Preconditions.checkNotNull(text); + } + + /** + * @param defaultLocales ordered list of locale preferences that may be used to + * disambiguate the provided text. If no locale preferences exist, + * set this to null or an empty locale list. + * @return this builder + */ + @NonNull + public Builder setDefaultLocales(@Nullable LocaleList defaultLocales) { + mDefaultLocales = defaultLocales; + return this; + } + + /** + * Sets the entity configuration to use. This determines what types of entities the + * TextClassifier will look for. + * Set to {@code null} for the default entity config and teh TextClassifier will + * automatically determine what links to generate. + * + * @return this builder + */ + @NonNull + public Builder setEntityConfig(@Nullable TextClassifier.EntityConfig entityConfig) { + mEntityConfig = entityConfig; + return this; + } + + /** + * Sets whether the TextClassifier can fallback to legacy links if smart linkify is + * disabled. + * + * <p><strong>Note: </strong>This is not parcelled. + * + * @return this builder + * @hide + */ + @NonNull + public Builder setLegacyFallback(boolean legacyFallback) { + mLegacyFallback = legacyFallback; + return this; + } + + /** + * Sets the name of the package that requested the links to get generated. + * + * @return this builder + * @hide + */ + @NonNull + public Builder setCallingPackageName(@Nullable String callingPackageName) { + mCallingPackageName = callingPackageName; + return this; + } + + /** + * Builds and returns the request object. + */ + @NonNull + public Request build() { + return new Request( + mText, mDefaultLocales, mEntityConfig, + mLegacyFallback, mCallingPackageName); + } + + } + + /** + * @return the name of the package that requested the links to get generated. + * TODO: make available as system API + * @hide + */ + @Nullable + public String getCallingPackageName() { + return mCallingPackageName; + } + @Override public int describeContents() { return 0; @@ -275,6 +459,7 @@ public final class TextLinks implements Parcelable { @Override public void writeToParcel(Parcel dest, int flags) { + dest.writeString(mText.toString()); dest.writeInt(mDefaultLocales != null ? 1 : 0); if (mDefaultLocales != null) { mDefaultLocales.writeToParcel(dest, flags); @@ -283,61 +468,94 @@ public final class TextLinks implements Parcelable { if (mEntityConfig != null) { mEntityConfig.writeToParcel(dest, flags); } + dest.writeString(mCallingPackageName); } - public static final Parcelable.Creator<Options> CREATOR = - new Parcelable.Creator<Options>() { + public static final Parcelable.Creator<Request> CREATOR = + new Parcelable.Creator<Request>() { @Override - public Options createFromParcel(Parcel in) { - return new Options(in); + public Request createFromParcel(Parcel in) { + return new Request(in); } @Override - public Options[] newArray(int size) { - return new Options[size]; + public Request[] newArray(int size) { + return new Request[size]; } }; - private Options(Parcel in) { - if (in.readInt() > 0) { - mDefaultLocales = LocaleList.CREATOR.createFromParcel(in); - } - if (in.readInt() > 0) { - mEntityConfig = TextClassifier.EntityConfig.CREATOR.createFromParcel(in); - } + private Request(Parcel in) { + mText = in.readString(); + mDefaultLocales = in.readInt() == 0 ? null : LocaleList.CREATOR.createFromParcel(in); + mEntityConfig = in.readInt() == 0 + ? null : TextClassifier.EntityConfig.CREATOR.createFromParcel(in); + mLegacyFallback = true; + mCallingPackageName = in.readString(); } } /** - * A function to create spans from TextLinks. - * - * Applies only to TextViews. - * We can hide this until we are convinced we want it to be part of the public API. + * A ClickableSpan for a TextLink. * - * @hide + * <p>Applies only to TextViews. */ - public static final Function<TextLink, ClickableSpan> DEFAULT_SPAN_FACTORY = - textLink -> new ClickableSpan() { - @Override - public void onClick(View widget) { - if (widget instanceof TextView) { - final TextView textView = (TextView) widget; - textView.requestActionMode(textLink); + public static class TextLinkSpan extends ClickableSpan { + + private final TextLink mTextLink; + + public TextLinkSpan(@NonNull TextLink textLink) { + mTextLink = textLink; + } + + @Override + public void onClick(View widget) { + if (widget instanceof TextView) { + final TextView textView = (TextView) widget; + final Context context = textView.getContext(); + if (TextClassificationManager.getSettings(context).isSmartLinkifyEnabled()) { + if (textView.requestFocus()) { + textView.requestActionMode(this); + } else { + // If textView can not take focus, then simply handle the click as it will + // be difficult to get rid of the floating action mode. + textView.handleClick(this); + } + } else { + if (mTextLink.mUrlSpan != null) { + mTextLink.mUrlSpan.onClick(textView); + } else { + textView.handleClick(this); } } - }; + } + } + + public final TextLink getTextLink() { + return mTextLink; + } + + /** @hide */ + @VisibleForTesting(visibility = Visibility.PRIVATE) + @Nullable + public final String getUrl() { + if (mTextLink.mUrlSpan != null) { + return mTextLink.mUrlSpan.getURL(); + } + return null; + } + } /** * A builder to construct a TextLinks instance. */ public static final class Builder { private final String mFullText; - private final Collection<TextLink> mLinks; + private final ArrayList<TextLink> mLinks; /** * Create a new TextLinks.Builder. * - * @param fullText The full text that links will be added to. + * @param fullText The full text to annotate with links */ public Builder(@NonNull String fullText) { mFullText = Preconditions.checkNotNull(fullText); @@ -347,19 +565,44 @@ public final class TextLinks implements Parcelable { /** * Adds a TextLink. * - * @return this instance. + * @param start The start index of the identified subsequence + * @param end The end index of the identified subsequence + * @param entityScores A mapping of entity type to confidence score + * + * @throws IllegalArgumentException if entityScores is null or empty. + */ + @NonNull + public Builder addLink(int start, int end, Map<String, Float> entityScores) { + mLinks.add(new TextLink(start, end, entityScores, null)); + return this; + } + + /** + * @see #addLink(int, int, Map) + * @param urlSpan An optional URLSpan to delegate to. NOTE: Not parcelled. + */ + @NonNull + Builder addLink(int start, int end, Map<String, Float> entityScores, + @Nullable URLSpan urlSpan) { + mLinks.add(new TextLink(start, end, entityScores, urlSpan)); + return this; + } + + /** + * Removes all {@link TextLink}s. */ - public Builder addLink(TextLink link) { - Preconditions.checkNotNull(link); - mLinks.add(link); + @NonNull + public Builder clearTextLinks() { + mLinks.clear(); return this; } /** * Constructs a TextLinks instance. * - * @return the constructed TextLinks. + * @return the constructed TextLinks */ + @NonNull public TextLinks build() { return new TextLinks(mFullText, mLinks); } diff --git a/android/view/textclassifier/TextLinksParams.java b/android/view/textclassifier/TextLinksParams.java new file mode 100644 index 00000000..be4c3bcd --- /dev/null +++ b/android/view/textclassifier/TextLinksParams.java @@ -0,0 +1,206 @@ +/* + * Copyright 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.view.textclassifier; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.text.Spannable; +import android.text.style.ClickableSpan; +import android.text.util.Linkify; +import android.text.util.Linkify.LinkifyMask; +import android.view.textclassifier.TextLinks.TextLink; +import android.view.textclassifier.TextLinks.TextLinkSpan; + +import com.android.internal.util.Preconditions; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +/** + * Parameters for generating and applying links. + * @hide + */ +public final class TextLinksParams { + + /** + * A function to create spans from TextLinks. + */ + private static final Function<TextLink, TextLinkSpan> DEFAULT_SPAN_FACTORY = + textLink -> new TextLinkSpan(textLink); + + @TextLinks.ApplyStrategy + private final int mApplyStrategy; + private final Function<TextLink, TextLinkSpan> mSpanFactory; + private final TextClassifier.EntityConfig mEntityConfig; + + private TextLinksParams( + @TextLinks.ApplyStrategy int applyStrategy, + Function<TextLink, TextLinkSpan> spanFactory) { + mApplyStrategy = applyStrategy; + mSpanFactory = spanFactory; + mEntityConfig = TextClassifier.EntityConfig.createWithHints(null); + } + + /** + * Returns a new TextLinksParams object based on the specified link mask. + * + * @param mask the link mask + * e.g. {@link LinkifyMask#PHONE_NUMBERS} | {@link LinkifyMask#EMAIL_ADDRESSES} + * @hide + */ + @NonNull + public static TextLinksParams fromLinkMask(@LinkifyMask int mask) { + final List<String> entitiesToFind = new ArrayList<>(); + if ((mask & Linkify.WEB_URLS) != 0) { + entitiesToFind.add(TextClassifier.TYPE_URL); + } + if ((mask & Linkify.EMAIL_ADDRESSES) != 0) { + entitiesToFind.add(TextClassifier.TYPE_EMAIL); + } + if ((mask & Linkify.PHONE_NUMBERS) != 0) { + entitiesToFind.add(TextClassifier.TYPE_PHONE); + } + if ((mask & Linkify.MAP_ADDRESSES) != 0) { + entitiesToFind.add(TextClassifier.TYPE_ADDRESS); + } + return new TextLinksParams.Builder().setEntityConfig( + TextClassifier.EntityConfig.createWithExplicitEntityList(entitiesToFind)) + .build(); + } + + /** + * Returns the entity config used to determine what entity types to generate. + */ + @NonNull + public TextClassifier.EntityConfig getEntityConfig() { + return mEntityConfig; + } + + /** + * Annotates the given text with the generated links. It will fail if the provided text doesn't + * match the original text used to crete the TextLinks. + * + * @param text the text to apply the links to. Must match the original text + * @param textLinks the links to apply to the text + * + * @return a status code indicating whether or not the links were successfully applied + * @hide + */ + @TextLinks.Status + public int apply(@NonNull Spannable text, @NonNull TextLinks textLinks) { + Preconditions.checkNotNull(text); + Preconditions.checkNotNull(textLinks); + + final String textString = text.toString(); + if (!textString.startsWith(textLinks.getText())) { + return TextLinks.STATUS_DIFFERENT_TEXT; + } + if (textLinks.getLinks().isEmpty()) { + return TextLinks.STATUS_NO_LINKS_FOUND; + } + + int applyCount = 0; + for (TextLink link : textLinks.getLinks()) { + final TextLinkSpan span = mSpanFactory.apply(link); + if (span != null) { + final ClickableSpan[] existingSpans = text.getSpans( + link.getStart(), link.getEnd(), ClickableSpan.class); + if (existingSpans.length > 0) { + if (mApplyStrategy == TextLinks.APPLY_STRATEGY_REPLACE) { + for (ClickableSpan existingSpan : existingSpans) { + text.removeSpan(existingSpan); + } + text.setSpan(span, link.getStart(), link.getEnd(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + applyCount++; + } + } else { + text.setSpan(span, link.getStart(), link.getEnd(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + applyCount++; + } + } + } + if (applyCount == 0) { + return TextLinks.STATUS_NO_LINKS_APPLIED; + } + return TextLinks.STATUS_LINKS_APPLIED; + } + + /** + * A builder for building TextLinksParams. + */ + public static final class Builder { + + @TextLinks.ApplyStrategy + private int mApplyStrategy = TextLinks.APPLY_STRATEGY_IGNORE; + private Function<TextLink, TextLinkSpan> mSpanFactory = DEFAULT_SPAN_FACTORY; + + /** + * Sets the apply strategy used to determine how to apply links to text. + * e.g {@link TextLinks#APPLY_STRATEGY_IGNORE} + * + * @return this builder + */ + public Builder setApplyStrategy(@TextLinks.ApplyStrategy int applyStrategy) { + mApplyStrategy = checkApplyStrategy(applyStrategy); + return this; + } + + /** + * Sets a custom span factory for converting TextLinks to TextLinkSpans. + * Set to {@code null} to use the default span factory. + * + * @return this builder + */ + public Builder setSpanFactory(@Nullable Function<TextLink, TextLinkSpan> spanFactory) { + mSpanFactory = spanFactory == null ? DEFAULT_SPAN_FACTORY : spanFactory; + return this; + } + + /** + * Sets the entity configuration used to determine what entity types to generate. + * Set to {@code null} for the default entity config which will automatically determine + * what links to generate. + * + * @return this builder + */ + public Builder setEntityConfig(@Nullable TextClassifier.EntityConfig entityConfig) { + return this; + } + + /** + * Builds and returns a TextLinksParams object. + */ + public TextLinksParams build() { + return new TextLinksParams(mApplyStrategy, mSpanFactory); + } + } + + /** @throws IllegalArgumentException if the value is invalid */ + @TextLinks.ApplyStrategy + private static int checkApplyStrategy(int applyStrategy) { + if (applyStrategy != TextLinks.APPLY_STRATEGY_IGNORE + && applyStrategy != TextLinks.APPLY_STRATEGY_REPLACE) { + throw new IllegalArgumentException( + "Invalid apply strategy. See TextLinksParams.ApplyStrategy for options."); + } + return applyStrategy; + } +} + diff --git a/android/view/textclassifier/TextSelection.java b/android/view/textclassifier/TextSelection.java index 774d42db..939e7176 100644 --- a/android/view/textclassifier/TextSelection.java +++ b/android/view/textclassifier/TextSelection.java @@ -25,6 +25,7 @@ import android.os.Parcel; import android.os.Parcelable; import android.util.ArrayMap; import android.view.textclassifier.TextClassifier.EntityType; +import android.view.textclassifier.TextClassifier.Utils; import com.android.internal.util.Preconditions; @@ -34,20 +35,19 @@ import java.util.Map; /** * Information about where text selection should be. */ -public final class TextSelection { +public final class TextSelection implements Parcelable { private final int mStartIndex; private final int mEndIndex; - @NonNull private final EntityConfidence mEntityConfidence; - @NonNull private final String mSignature; + private final EntityConfidence mEntityConfidence; + @Nullable private final String mId; private TextSelection( - int startIndex, int endIndex, @NonNull Map<String, Float> entityConfidence, - @NonNull String signature) { + int startIndex, int endIndex, Map<String, Float> entityConfidence, String id) { mStartIndex = startIndex; mEndIndex = endIndex; mEntityConfidence = new EntityConfidence(entityConfidence); - mSignature = signature; + mId = id; } /** @@ -80,7 +80,8 @@ public final class TextSelection { * @see #getEntityCount() for the number of entities available. */ @NonNull - public @EntityType String getEntity(int index) { + @EntityType + public String getEntity(int index) { return mEntityConfidence.getEntities().get(index); } @@ -95,37 +96,19 @@ public final class TextSelection { } /** - * Returns the signature for this object. - * The TextClassifier that generates this object may use it as a way to internally identify - * this object. + * Returns the id, if one exists, for this object. */ - @NonNull - public String getSignature() { - return mSignature; + @Nullable + public String getId() { + return mId; } @Override public String toString() { return String.format( Locale.US, - "TextSelection {startIndex=%d, endIndex=%d, entities=%s, signature=%s}", - mStartIndex, mEndIndex, mEntityConfidence, mSignature); - } - - /** Helper for parceling via #ParcelableWrapper. */ - private void writeToParcel(Parcel dest, int flags) { - dest.writeInt(mStartIndex); - dest.writeInt(mEndIndex); - mEntityConfidence.writeToParcel(dest, flags); - dest.writeString(mSignature); - } - - /** Helper for unparceling via #ParcelableWrapper. */ - private TextSelection(Parcel in) { - mStartIndex = in.readInt(); - mEndIndex = in.readInt(); - mEntityConfidence = EntityConfidence.CREATOR.createFromParcel(in); - mSignature = in.readString(); + "TextSelection {id=%s, startIndex=%d, endIndex=%d, entities=%s}", + mId, mStartIndex, mEndIndex, mEntityConfidence); } /** @@ -135,8 +118,8 @@ public final class TextSelection { private final int mStartIndex; private final int mEndIndex; - @NonNull private final Map<String, Float> mEntityConfidence = new ArrayMap<>(); - @NonNull private String mSignature = ""; + private final Map<String, Float> mEntityConfidence = new ArrayMap<>(); + @Nullable private String mId; /** * Creates a builder used to build {@link TextSelection} objects. @@ -158,78 +141,86 @@ public final class TextSelection { * 0 implies the entity does not exist for the classified text. * Values greater than 1 are clamped to 1. */ + @NonNull public Builder setEntityType( @NonNull @EntityType String type, @FloatRange(from = 0.0, to = 1.0) float confidenceScore) { + Preconditions.checkNotNull(type); mEntityConfidence.put(type, confidenceScore); return this; } /** - * Sets a signature for the TextSelection object. - * - * The TextClassifier that generates the TextSelection object may use it as a way to - * internally identify the TextSelection object. + * Sets an id for the TextSelection object. */ - public Builder setSignature(@NonNull String signature) { - mSignature = Preconditions.checkNotNull(signature); + @NonNull + public Builder setId(@NonNull String id) { + mId = Preconditions.checkNotNull(id); return this; } /** * Builds and returns {@link TextSelection} object. */ + @NonNull public TextSelection build() { return new TextSelection( - mStartIndex, mEndIndex, mEntityConfidence, mSignature); + mStartIndex, mEndIndex, mEntityConfidence, mId); } } /** - * Optional input parameters for generating TextSelection. + * A request object for generating TextSelection. */ - public static final class Options implements Parcelable { + public static final class Request implements Parcelable { - private @Nullable LocaleList mDefaultLocales; - private boolean mDarkLaunchAllowed; - - public Options() {} + private final CharSequence mText; + private final int mStartIndex; + private final int mEndIndex; + @Nullable private final LocaleList mDefaultLocales; + private final boolean mDarkLaunchAllowed; + + private Request( + CharSequence text, + int startIndex, + int endIndex, + LocaleList defaultLocales, + boolean darkLaunchAllowed) { + mText = text; + mStartIndex = startIndex; + mEndIndex = endIndex; + mDefaultLocales = defaultLocales; + mDarkLaunchAllowed = darkLaunchAllowed; + } /** - * @param defaultLocales ordered list of locale preferences that may be used to disambiguate - * the provided text. If no locale preferences exist, set this to null or an empty - * locale list. + * Returns the text providing context for the selected text (which is specified by the + * sub sequence starting at startIndex and ending at endIndex). */ - public Options setDefaultLocales(@Nullable LocaleList defaultLocales) { - mDefaultLocales = defaultLocales; - return this; + @NonNull + public CharSequence getText() { + return mText; } /** - * @return ordered list of locale preferences that can be used to disambiguate - * the provided text. + * Returns start index of the selected part of text. */ - @Nullable - public LocaleList getDefaultLocales() { - return mDefaultLocales; + @IntRange(from = 0) + public int getStartIndex() { + return mStartIndex; } /** - * @param allowed whether or not the TextClassifier should return selection suggestions - * when "dark launched". When a TextClassifier is dark launched, it can suggest - * selection changes that should not be used to actually change the user's selection. - * Instead, the suggested selection is logged, compared with the user's selection - * interaction, and used to generate quality metrics for the TextClassifier. - * - * @hide + * Returns end index of the selected part of text. */ - public void setDarkLaunchAllowed(boolean allowed) { - mDarkLaunchAllowed = allowed; + @IntRange(from = 0) + public int getEndIndex() { + return mEndIndex; } /** - * Returns true if the TextClassifier should return selection suggestions when - * "dark launched". Otherwise, returns false. + * Returns true if the TextClassifier should return selection suggestions when "dark + * launched". Otherwise, returns false. * * @hide */ @@ -237,6 +228,83 @@ public final class TextSelection { return mDarkLaunchAllowed; } + /** + * @return ordered list of locale preferences that can be used to disambiguate the + * provided text. + */ + @Nullable + public LocaleList getDefaultLocales() { + return mDefaultLocales; + } + + /** + * A builder for building TextSelection requests. + */ + public static final class Builder { + + private final CharSequence mText; + private final int mStartIndex; + private final int mEndIndex; + + @Nullable private LocaleList mDefaultLocales; + private boolean mDarkLaunchAllowed; + + /** + * @param text text providing context for the selected text (which is specified by the + * sub sequence starting at selectionStartIndex and ending at selectionEndIndex) + * @param startIndex start index of the selected part of text + * @param endIndex end index of the selected part of text + */ + public Builder( + @NonNull CharSequence text, + @IntRange(from = 0) int startIndex, + @IntRange(from = 0) int endIndex) { + Utils.checkArgument(text, startIndex, endIndex); + mText = text; + mStartIndex = startIndex; + mEndIndex = endIndex; + } + + /** + * @param defaultLocales ordered list of locale preferences that may be used to + * disambiguate the provided text. If no locale preferences exist, set this to null + * or an empty locale list. + * + * @return this builder. + */ + @NonNull + public Builder setDefaultLocales(@Nullable LocaleList defaultLocales) { + mDefaultLocales = defaultLocales; + return this; + } + + /** + * @param allowed whether or not the TextClassifier should return selection suggestions + * when "dark launched". When a TextClassifier is dark launched, it can suggest + * selection changes that should not be used to actually change the user's + * selection. Instead, the suggested selection is logged, compared with the user's + * selection interaction, and used to generate quality metrics for the + * TextClassifier. Not parceled. + * + * @return this builder. + * @hide + */ + @NonNull + public Builder setDarkLaunchAllowed(boolean allowed) { + mDarkLaunchAllowed = allowed; + return this; + } + + /** + * Builds and returns the request object. + */ + @NonNull + public Request build() { + return new Request(mText, mStartIndex, mEndIndex, + mDefaultLocales, mDarkLaunchAllowed); + } + } + @Override public int describeContents() { return 0; @@ -244,74 +312,67 @@ public final class TextSelection { @Override public void writeToParcel(Parcel dest, int flags) { + dest.writeString(mText.toString()); + dest.writeInt(mStartIndex); + dest.writeInt(mEndIndex); dest.writeInt(mDefaultLocales != null ? 1 : 0); if (mDefaultLocales != null) { mDefaultLocales.writeToParcel(dest, flags); } - dest.writeInt(mDarkLaunchAllowed ? 1 : 0); } - public static final Parcelable.Creator<Options> CREATOR = - new Parcelable.Creator<Options>() { + public static final Parcelable.Creator<Request> CREATOR = + new Parcelable.Creator<Request>() { @Override - public Options createFromParcel(Parcel in) { - return new Options(in); + public Request createFromParcel(Parcel in) { + return new Request(in); } @Override - public Options[] newArray(int size) { - return new Options[size]; + public Request[] newArray(int size) { + return new Request[size]; } }; - private Options(Parcel in) { - if (in.readInt() > 0) { - mDefaultLocales = LocaleList.CREATOR.createFromParcel(in); - } - mDarkLaunchAllowed = in.readInt() != 0; + private Request(Parcel in) { + mText = in.readString(); + mStartIndex = in.readInt(); + mEndIndex = in.readInt(); + mDefaultLocales = in.readInt() == 0 ? null : LocaleList.CREATOR.createFromParcel(in); + mDarkLaunchAllowed = false; } } - /** - * Parcelable wrapper for TextSelection objects. - * @hide - */ - public static final class ParcelableWrapper implements Parcelable { - - @NonNull private TextSelection mTextSelection; - - public ParcelableWrapper(@NonNull TextSelection textSelection) { - Preconditions.checkNotNull(textSelection); - mTextSelection = textSelection; - } - - @NonNull - public TextSelection getTextSelection() { - return mTextSelection; - } - - @Override - public int describeContents() { - return 0; - } + @Override + public int describeContents() { + return 0; + } - @Override - public void writeToParcel(Parcel dest, int flags) { - mTextSelection.writeToParcel(dest, flags); - } + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeInt(mStartIndex); + dest.writeInt(mEndIndex); + mEntityConfidence.writeToParcel(dest, flags); + dest.writeString(mId); + } - public static final Parcelable.Creator<ParcelableWrapper> CREATOR = - new Parcelable.Creator<ParcelableWrapper>() { - @Override - public ParcelableWrapper createFromParcel(Parcel in) { - return new ParcelableWrapper(new TextSelection(in)); - } + public static final Parcelable.Creator<TextSelection> CREATOR = + new Parcelable.Creator<TextSelection>() { + @Override + public TextSelection createFromParcel(Parcel in) { + return new TextSelection(in); + } - @Override - public ParcelableWrapper[] newArray(int size) { - return new ParcelableWrapper[size]; - } - }; + @Override + public TextSelection[] newArray(int size) { + return new TextSelection[size]; + } + }; + private TextSelection(Parcel in) { + mStartIndex = in.readInt(); + mEndIndex = in.readInt(); + mEntityConfidence = EntityConfidence.CREATOR.createFromParcel(in); + mId = in.readString(); } } diff --git a/android/view/textclassifier/logging/SmartSelectionEventTracker.java b/android/view/textclassifier/logging/SmartSelectionEventTracker.java index 157b3d82..f7d75cd8 100644 --- a/android/view/textclassifier/logging/SmartSelectionEventTracker.java +++ b/android/view/textclassifier/logging/SmartSelectionEventTracker.java @@ -473,7 +473,7 @@ public final class SmartSelectionEventTracker { final String entityType = classification.getEntityCount() > 0 ? classification.getEntity(0) : TextClassifier.TYPE_UNKNOWN; - final String versionTag = getVersionInfo(classification.getSignature()); + final String versionTag = getVersionInfo(classification.getId()); return new SelectionEvent( start, end, EventType.SELECTION_MODIFIED, entityType, versionTag); } @@ -489,7 +489,7 @@ public final class SmartSelectionEventTracker { */ public static SelectionEvent selectionModified( int start, int end, @NonNull TextSelection selection) { - final boolean smartSelection = getSourceClassifier(selection.getSignature()) + final boolean smartSelection = getSourceClassifier(selection.getId()) .equals(TextClassifier.DEFAULT_LOG_TAG); final int eventType; if (smartSelection) { @@ -503,7 +503,7 @@ public final class SmartSelectionEventTracker { final String entityType = selection.getEntityCount() > 0 ? selection.getEntity(0) : TextClassifier.TYPE_UNKNOWN; - final String versionTag = getVersionInfo(selection.getSignature()); + final String versionTag = getVersionInfo(selection.getId()); return new SelectionEvent(start, end, eventType, entityType, versionTag); } @@ -538,7 +538,7 @@ public final class SmartSelectionEventTracker { final String entityType = classification.getEntityCount() > 0 ? classification.getEntity(0) : TextClassifier.TYPE_UNKNOWN; - final String versionTag = getVersionInfo(classification.getSignature()); + final String versionTag = getVersionInfo(classification.getId()); return new SelectionEvent(start, end, actionType, entityType, versionTag); } diff --git a/android/view/textservice/SpellCheckerSession.java b/android/view/textservice/SpellCheckerSession.java index 779eefb1..886f5c82 100644 --- a/android/view/textservice/SpellCheckerSession.java +++ b/android/view/textservice/SpellCheckerSession.java @@ -445,9 +445,15 @@ public class SpellCheckerSession { private void processOrEnqueueTask(SpellCheckerParams scp) { ISpellCheckerSession session; synchronized (this) { + if (scp.mWhat == TASK_CLOSE && (mState == STATE_CLOSED_AFTER_CONNECTION + || mState == STATE_CLOSED_BEFORE_CONNECTION)) { + // It is OK to call SpellCheckerSession#close() multiple times. + // Don't output confusing/misleading warning messages. + return; + } if (mState != STATE_WAIT_CONNECTION && mState != STATE_CONNECTED) { Log.e(TAG, "ignoring processOrEnqueueTask due to unexpected mState=" - + taskToString(scp.mWhat) + + stateToString(mState) + " scp.mWhat=" + taskToString(scp.mWhat)); return; } |