Merge "Add workaround to resolve adjusted crop size is incorrect caused by MediaCodecInfo provides incorrect supported widths/heights" into androidx-main

5 files changed, 374 insertions, 4 deletions

diff --git a/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java b/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java
index 94d174604a6..c2508920a64 100644
--- a/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java
+++ b/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java
@@ -110,6 +110,7 @@ import androidx.camera.video.internal.encoder.InvalidConfigException;
 import androidx.camera.video.internal.encoder.VideoEncoderConfig;
 import androidx.camera.video.internal.encoder.VideoEncoderInfo;
 import androidx.camera.video.internal.encoder.VideoEncoderInfoImpl;
+import androidx.camera.video.internal.workaround.VideoEncoderInfoWrapper;
 import androidx.concurrent.futures.CallbackToFutureAdapter;
 import androidx.core.util.Preconditions;
 import androidx.core.util.Supplier;
@@ -890,13 +891,21 @@ public final class VideoCapture<T extends VideoOutput> extends UseCase {
         if (mVideoEncoderInfo != null) {
             return mVideoEncoderInfo;
         }
+
+        VideoEncoderInfo videoEncoderInfo = resolveVideoEncoderInfo(videoEncoderInfoFinder,
+                videoCapabilities, timebase, mediaSpec, resolution, targetFps);
+        if (videoEncoderInfo == null) {
+            return null;
+        }
+
+        videoEncoderInfo = VideoEncoderInfoWrapper.from(videoEncoderInfo, resolution);
+
         // Cache the VideoEncoderInfo as it should be the same when recreating the pipeline.
         // This avoids recreating the MediaCodec instance to get encoder information.
         // Note: We should clear the cache if the MediaSpec changes at any time, especially when
         // the Encoder-related content in the VideoSpec changes. i.e. when we need to observe the
         // MediaSpec Observable.
-        return mVideoEncoderInfo = resolveVideoEncoderInfo(videoEncoderInfoFinder,
-                videoCapabilities, timebase, mediaSpec, resolution, targetFps);
+        return mVideoEncoderInfo = videoEncoderInfo;
     }
 
     @Nullable
diff --git a/camera/camera-video/src/main/java/androidx/camera/video/internal/workaround/VideoEncoderInfoWrapper.java b/camera/camera-video/src/main/java/androidx/camera/video/internal/workaround/VideoEncoderInfoWrapper.java
new file mode 100644
index 00000000000..e7cf9a2b194
--- /dev/null
+++ b/camera/camera-video/src/main/java/androidx/camera/video/internal/workaround/VideoEncoderInfoWrapper.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.video.internal.workaround;
+
+import android.media.MediaCodecInfo;
+import android.util.Range;
+import android.util.Size;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.RequiresApi;
+import androidx.camera.core.Logger;
+import androidx.camera.video.internal.compat.quirk.DeviceQuirks;
+import androidx.camera.video.internal.compat.quirk.MediaCodecInfoReportIncorrectInfoQuirk;
+import androidx.camera.video.internal.encoder.VideoEncoderInfo;
+import androidx.core.util.Preconditions;
+
+/**
+ * Workaround to wrap the VideoEncoderInfo in order to fix the wrong information provided by
+ * {@link MediaCodecInfo}.
+ *
+ * <p>One use case is VideoCapture resizing the crop to a size valid for the encoder.
+ *
+ * @see MediaCodecInfoReportIncorrectInfoQuirk
+ */
+@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
+public class VideoEncoderInfoWrapper implements VideoEncoderInfo {
+    private static final String TAG = "VideoEncoderInfoWrapper";
+
+    // The resolution of CamcorderProfile.QUALITY_4KDCI
+    private static final int WIDTH_4KDCI = 4096;
+    private static final int HEIGHT_4KDCI = 2160;
+
+    private final VideoEncoderInfo mVideoEncoderInfo;
+    private final Range<Integer> mSupportedWidths;
+    private final Range<Integer> mSupportedHeights;
+
+    /**
+     * Check and wrap an input VideoEncoderInfo
+     *
+     * <p>The input VideoEncoderInfo will be wrapped when
+     * <ul>
+     * <li>The device is a quirk device determined in
+     * {@link MediaCodecInfoReportIncorrectInfoQuirk}.</li>
+     * <li>The input {@code validSizeToCheck} is not supported by input VideoEncoderInfo.</li>
+     * </ul>
+     * Otherwise, the input VideoEncoderInfo will be returned.
+     *
+     * @param videoEncoderInfo the input VideoEncoderInfo.
+     * @param validSizeToCheck a valid size to check.
+     * @return a wrapped VideoEncoderInfo or the input VideoEncoderInfo.
+     */
+    @NonNull
+    public static VideoEncoderInfo from(@NonNull VideoEncoderInfo videoEncoderInfo,
+            @NonNull Size validSizeToCheck) {
+        boolean toWrap = false;
+        if (DeviceQuirks.get(MediaCodecInfoReportIncorrectInfoQuirk.class) != null) {
+            toWrap = true;
+        } else if (!isSizeSupported(videoEncoderInfo, validSizeToCheck)) {
+            // If the device does not support a size that should be valid, assume the device
+            // reports incorrect information. This is used to detect devices that we haven't
+            // discovered incorrect information yet.
+            Logger.w(TAG, String.format(
+                    "Detected that the device does not support a size %s that should be valid"
+                            + " in widths/heights = %s/%s", validSizeToCheck,
+                    videoEncoderInfo.getSupportedWidths(),
+                    videoEncoderInfo.getSupportedHeights()));
+            toWrap = true;
+        }
+        return toWrap ? new VideoEncoderInfoWrapper(videoEncoderInfo) : videoEncoderInfo;
+    }
+
+    VideoEncoderInfoWrapper(@NonNull VideoEncoderInfo videoEncoderInfo) {
+        mVideoEncoderInfo = videoEncoderInfo;
+
+        // Ideally we should find out supported widths/heights for each problematic device.
+        // As a workaround, simply return a big enough size for video encoding. i.e.
+        // CamcorderProfile.QUALITY_4KDCI. The size still need to follow the multiple of alignment.
+        int widthAlignment = videoEncoderInfo.getWidthAlignment();
+        int maxWidth = (int) Math.ceil((double) WIDTH_4KDCI / widthAlignment) * widthAlignment;
+        mSupportedWidths = Range.create(widthAlignment, maxWidth);
+        int heightAlignment = videoEncoderInfo.getHeightAlignment();
+        int maxHeight = (int) Math.ceil((double) HEIGHT_4KDCI / heightAlignment) * heightAlignment;
+        mSupportedHeights = Range.create(heightAlignment, maxHeight);
+    }
+
+    @NonNull
+    @Override
+    public String getName() {
+        return mVideoEncoderInfo.getName();
+    }
+
+    @NonNull
+    @Override
+    public Range<Integer> getSupportedWidths() {
+        return mSupportedWidths;
+    }
+
+    @NonNull
+    @Override
+    public Range<Integer> getSupportedHeights() {
+        return mSupportedHeights;
+    }
+
+    @NonNull
+    @Override
+    public Range<Integer> getSupportedWidthsFor(int height) {
+        Preconditions.checkArgument(mSupportedHeights.contains(height),
+                "Not supported height: " + height + " in " + mSupportedHeights);
+        return mSupportedWidths;
+    }
+
+    @NonNull
+    @Override
+    public Range<Integer> getSupportedHeightsFor(int width) {
+        Preconditions.checkArgument(mSupportedWidths.contains(width),
+                "Not supported width: " + width + " in " + mSupportedWidths);
+        return mSupportedHeights;
+    }
+
+    @Override
+    public int getWidthAlignment() {
+        return mVideoEncoderInfo.getWidthAlignment();
+    }
+
+    @Override
+    public int getHeightAlignment() {
+        return mVideoEncoderInfo.getHeightAlignment();
+    }
+
+    private static boolean isSizeSupported(@NonNull VideoEncoderInfo videoEncoderInfo,
+            @NonNull Size size) {
+        if (!videoEncoderInfo.getSupportedWidths().contains(size.getWidth())
+                || !videoEncoderInfo.getSupportedHeights().contains(size.getHeight())) {
+            return false;
+        }
+        try {
+            if (!videoEncoderInfo.getSupportedHeightsFor(size.getWidth()).contains(size.getHeight())
+                    || !videoEncoderInfo.getSupportedWidthsFor(size.getHeight()).contains(
+                    size.getWidth())) {
+                return false;
+            }
+        } catch (IllegalArgumentException e) {
+            Logger.w(TAG, "size is not supported", e);
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/camera/camera-video/src/test/java/androidx/camera/video/VideoCaptureTest.kt b/camera/camera-video/src/test/java/androidx/camera/video/VideoCaptureTest.kt
index af83bd431bd..df60502bfcd 100644
--- a/camera/camera-video/src/test/java/androidx/camera/video/VideoCaptureTest.kt
+++ b/camera/camera-video/src/test/java/androidx/camera/video/VideoCaptureTest.kt
@@ -659,8 +659,9 @@ class VideoCaptureTest {
             videoEncoderInfo = createVideoEncoderInfo(
                 widthAlignment = 8,
                 heightAlignment = 8,
-                supportedWidths = Range(80, 800),
-                supportedHeights = Range(100, 800),
+                // 1280x720 is a valid size
+                supportedWidths = Range(80, 1600),
+                supportedHeights = Range(100, 1600),
             ),
             cropRect = Rect(8, 8, 48, 48), // 40x40
             expectedCropRect = Rect(0, 0, 80, 100),
@@ -668,6 +669,21 @@ class VideoCaptureTest {
     }
 
     @Test
+    fun adjustCropRect_notValidSize_ignoreSupportedSizeAndClampByWorkaroundSize() {
+        testAdjustCropRectToValidSize(
+            videoEncoderInfo = createVideoEncoderInfo(
+                widthAlignment = 8,
+                heightAlignment = 8,
+                // 1280x720 is not a valid size, workaround size is [8-4096], [8-2160]
+                supportedWidths = Range(80, 80),
+                supportedHeights = Range(80, 80),
+            ),
+            cropRect = Rect(0, 0, 4, 4), // 4x4
+            expectedCropRect = Rect(0, 0, 8, 8), // 8x8
+        )
+    }
+
+    @Test
     fun adjustCropRect_toSmallestDimensionChange() {
         testAdjustCropRectToValidSize(
             videoEncoderInfo = createVideoEncoderInfo(widthAlignment = 8, heightAlignment = 8),
diff --git a/camera/camera-video/src/test/java/androidx/camera/video/internal/compat/quirk/DeviceQuirks.java b/camera/camera-video/src/test/java/androidx/camera/video/internal/compat/quirk/DeviceQuirks.java
new file mode 100644
index 00000000000..9f668eaaeae
--- /dev/null
+++ b/camera/camera-video/src/test/java/androidx/camera/video/internal/compat/quirk/DeviceQuirks.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.video.internal.compat.quirk;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.camera.core.impl.Quirk;
+
+import java.util.List;
+
+/**
+ * Tests version of main/.../DeviceQuirks.java, which provides device specific quirks, used for
+ * device specific workarounds.
+ * <p>
+ * In main/.../DeviceQuirks, Device quirks are loaded the first time a device workaround is
+ * encountered, and remain in memory until the process is killed. When running tests, this means
+ * that the same device quirks are used for all the tests. This causes an issue when tests modify
+ * device properties (using Robolectric for instance). Instead of force-reloading the device
+ * quirks in every test that uses a device workaround, this class internally reloads the quirks
+ * every time a device workaround is needed.
+ */
+public class DeviceQuirks {
+
+    private DeviceQuirks() {
+    }
+
+    /**
+     * Retrieves a specific device {@link Quirk} instance given its type.
+     *
+     * @param quirkClass The type of device quirk to retrieve.
+     * @return A device {@link Quirk} instance of the provided type, or {@code null} if it isn't
+     * found.
+     */
+    @SuppressWarnings("unchecked")
+    @Nullable
+    public static <T extends Quirk> T get(@NonNull final Class<T> quirkClass) {
+        final List<Quirk> quirks = DeviceQuirksLoader.loadQuirks();
+        for (final Quirk quirk : quirks) {
+            if (quirk.getClass() == quirkClass) {
+                return (T) quirk;
+            }
+        }
+        return null;
+    }
+}
diff --git a/camera/camera-video/src/test/java/androidx/camera/video/internal/workaround/VideoEncoderInfoWrapperTest.kt b/camera/camera-video/src/test/java/androidx/camera/video/internal/workaround/VideoEncoderInfoWrapperTest.kt
new file mode 100644
index 00000000000..8a04d492d59
--- /dev/null
+++ b/camera/camera-video/src/test/java/androidx/camera/video/internal/workaround/VideoEncoderInfoWrapperTest.kt
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.video.internal.workaround
+
+import android.os.Build
+import android.util.Range
+import android.util.Size
+import androidx.camera.video.internal.encoder.FakeVideoEncoderInfo
+import com.google.common.truth.Truth.assertThat
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.ParameterizedRobolectricTestRunner
+import org.robolectric.annotation.Config
+import org.robolectric.annotation.internal.DoNotInstrument
+import org.robolectric.util.ReflectionHelpers
+
+@RunWith(ParameterizedRobolectricTestRunner::class)
+@DoNotInstrument
+@Config(minSdk = Build.VERSION_CODES.LOLLIPOP)
+class VideoEncoderInfoWrapperTest(
+    private val brand: String,
+    private val model: String,
+    private val sizeToCheck: Size,
+    private val expectedSupportedWidths: Range<Int>,
+    private val expectedSupportedHeights: Range<Int>,
+) {
+
+    companion object {
+        private const val WIDTH_ALIGNMENT = 2
+        private const val HEIGHT_ALIGNMENT = 2
+        private val SUPPORTED_WIDTHS = Range.create(WIDTH_ALIGNMENT, 640)
+        private val SUPPORTED_HEIGHTS = Range.create(HEIGHT_ALIGNMENT, 480)
+        private val VALID_SIZE = Size(320, 240)
+        private val INVALID_SIZE = Size(1920, 1080)
+
+        private const val WIDTH_4KDCI = 4096
+        private const val HEIGHT_4KDCI = 2160
+        private val OVERRIDE_SUPPORTED_WIDTHS = Range.create(WIDTH_ALIGNMENT, WIDTH_4KDCI)
+        private val OVERRIDE_SUPPORTED_HEIGHTS = Range.create(HEIGHT_ALIGNMENT, HEIGHT_4KDCI)
+        private const val NONE_QUIRK_BRAND = "NoneQuirkBrand"
+        private const val NONE_QUIRK_MODEL = "NoneQuirkModel"
+
+        @JvmStatic
+        @ParameterizedRobolectricTestRunner.Parameters(
+            name = "brand={0}, model={1}, sizeToCheck={2}" +
+                ", expectedSupportedWidths={3}, expectedSupportedHeights={4}"
+        )
+        fun data() = mutableListOf<Array<Any?>>().apply {
+            add(
+                arrayOf(
+                    NONE_QUIRK_BRAND,
+                    NONE_QUIRK_MODEL,
+                    VALID_SIZE,
+                    SUPPORTED_WIDTHS,
+                    SUPPORTED_HEIGHTS,
+                )
+            )
+            add(
+                arrayOf(
+                    NONE_QUIRK_BRAND,
+                    NONE_QUIRK_MODEL,
+                    INVALID_SIZE,
+                    OVERRIDE_SUPPORTED_WIDTHS,
+                    OVERRIDE_SUPPORTED_HEIGHTS,
+                )
+            )
+            add(
+                arrayOf(
+                    "Nokia",
+                    "Nokia 1",
+                    VALID_SIZE,
+                    OVERRIDE_SUPPORTED_WIDTHS,
+                    OVERRIDE_SUPPORTED_HEIGHTS,
+                )
+            )
+            add(
+                arrayOf(
+                    "motorola",
+                    "moto c",
+                    VALID_SIZE,
+                    OVERRIDE_SUPPORTED_WIDTHS,
+                    OVERRIDE_SUPPORTED_HEIGHTS,
+                )
+            )
+            // No necessary to test all models.
+        }
+    }
+
+    private val baseVideoEncoderInfo = FakeVideoEncoderInfo(
+        _supportedWidths = SUPPORTED_WIDTHS,
+        _supportedHeights = SUPPORTED_HEIGHTS,
+        _widthAlignment = WIDTH_ALIGNMENT,
+        _heightAlignment = HEIGHT_ALIGNMENT,
+    )
+
+    @Before
+    fun setup() {
+        ReflectionHelpers.setStaticField(Build::class.java, "BRAND", brand)
+        ReflectionHelpers.setStaticField(Build::class.java, "MODEL", model)
+    }
+
+    @Test
+    fun from() {
+        val videoEncoderInfo = VideoEncoderInfoWrapper.from(baseVideoEncoderInfo, sizeToCheck)
+
+        assertThat(videoEncoderInfo.supportedWidths).isEqualTo(expectedSupportedWidths)
+        assertThat(videoEncoderInfo.supportedHeights).isEqualTo(expectedSupportedHeights)
+    }
+}