diff options
Diffstat (limited to 'talk/app/webrtc/java')
34 files changed, 3007 insertions, 1386 deletions
diff --git a/talk/app/webrtc/java/android/org/webrtc/Camera2Enumerator.java b/talk/app/webrtc/java/android/org/webrtc/Camera2Enumerator.java index 097d1cd906..3444529596 100644 --- a/talk/app/webrtc/java/android/org/webrtc/Camera2Enumerator.java +++ b/talk/app/webrtc/java/android/org/webrtc/Camera2Enumerator.java @@ -27,7 +27,9 @@ package org.webrtc; +import android.annotation.TargetApi; import android.content.Context; + import android.graphics.ImageFormat; import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; @@ -45,6 +47,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +@TargetApi(21) public class Camera2Enumerator implements CameraEnumerationAndroid.Enumerator { private final static String TAG = "Camera2Enumerator"; private final static double NANO_SECONDS_PER_SECOND = 1.0e9; diff --git a/talk/app/webrtc/java/android/org/webrtc/CameraEnumerationAndroid.java b/talk/app/webrtc/java/android/org/webrtc/CameraEnumerationAndroid.java index 3e37f6afdc..5f68c3759e 100644 --- a/talk/app/webrtc/java/android/org/webrtc/CameraEnumerationAndroid.java +++ b/talk/app/webrtc/java/android/org/webrtc/CameraEnumerationAndroid.java @@ -29,7 +29,6 @@ package org.webrtc; import static java.lang.Math.abs; import static java.lang.Math.ceil; -import android.hardware.Camera; import android.graphics.ImageFormat; import org.json.JSONArray; @@ -72,7 +71,7 @@ public class CameraEnumerationAndroid { // other image formats then this needs to be updated and // VideoCapturerAndroid.getSupportedFormats need to return CaptureFormats of // all imageFormats. - public final int imageFormat = ImageFormat.YV12; + public final int imageFormat = ImageFormat.NV21; public CaptureFormat(int width, int height, int minFramerate, int maxFramerate) { @@ -88,25 +87,15 @@ public class CameraEnumerationAndroid { } // Calculates the frame size of the specified image format. Currently only - // supporting ImageFormat.YV12. The YV12's stride is the closest rounded up - // multiple of 16 of the width and width and height are always even. - // Android guarantees this: - // http://developer.android.com/reference/android/hardware/Camera.Parameters.html#setPreviewFormat%28int%29 + // supporting ImageFormat.NV21. + // The size is width * height * number of bytes per pixel. + // http://developer.android.com/reference/android/hardware/Camera.html#addCallbackBuffer(byte[]) public static int frameSize(int width, int height, int imageFormat) { - if (imageFormat != ImageFormat.YV12) { + if (imageFormat != ImageFormat.NV21) { throw new UnsupportedOperationException("Don't know how to calculate " - + "the frame size of non-YV12 image formats."); + + "the frame size of non-NV21 image formats."); } - int yStride = roundUp(width, 16); - int uvStride = roundUp(yStride / 2, 16); - int ySize = yStride * height; - int uvSize = uvStride * height / 2; - return ySize + uvSize * 2; - } - - // Rounds up |x| to the closest value that is a multiple of |alignment|. - private static int roundUp(int x, int alignment) { - return (int)ceil(x / (double)alignment) * alignment; + return (width * height * ImageFormat.getBitsPerPixel(imageFormat)) / 8; } @Override @@ -114,21 +103,19 @@ public class CameraEnumerationAndroid { return width + "x" + height + "@[" + minFramerate + ":" + maxFramerate + "]"; } - @Override - public boolean equals(Object that) { - if (!(that instanceof CaptureFormat)) { + public boolean isSameFormat(final CaptureFormat that) { + if (that == null) { return false; } - final CaptureFormat c = (CaptureFormat) that; - return width == c.width && height == c.height && maxFramerate == c.maxFramerate - && minFramerate == c.minFramerate; + return width == that.width && height == that.height && maxFramerate == that.maxFramerate + && minFramerate == that.minFramerate; } } // Returns device names that can be used to create a new VideoCapturerAndroid. public static String[] getDeviceNames() { - String[] names = new String[Camera.getNumberOfCameras()]; - for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { + String[] names = new String[android.hardware.Camera.getNumberOfCameras()]; + for (int i = 0; i < android.hardware.Camera.getNumberOfCameras(); ++i) { names[i] = getDeviceName(i); } return names; @@ -136,22 +123,22 @@ public class CameraEnumerationAndroid { // Returns number of cameras on device. public static int getDeviceCount() { - return Camera.getNumberOfCameras(); + return android.hardware.Camera.getNumberOfCameras(); } // Returns the name of the camera with camera index. Returns null if the // camera can not be used. public static String getDeviceName(int index) { - Camera.CameraInfo info = new Camera.CameraInfo(); + android.hardware.Camera.CameraInfo info = new android.hardware.Camera.CameraInfo(); try { - Camera.getCameraInfo(index, info); + android.hardware.Camera.getCameraInfo(index, info); } catch (Exception e) { Logging.e(TAG, "getCameraInfo failed on index " + index,e); return null; } String facing = - (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) ? "front" : "back"; + (info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT) ? "front" : "back"; return "Camera " + index + ", Facing " + facing + ", Orientation " + info.orientation; } @@ -159,13 +146,13 @@ public class CameraEnumerationAndroid { // Returns the name of the front facing camera. Returns null if the // camera can not be used or does not exist. public static String getNameOfFrontFacingDevice() { - return getNameOfDevice(Camera.CameraInfo.CAMERA_FACING_FRONT); + return getNameOfDevice(android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT); } // Returns the name of the back facing camera. Returns null if the // camera can not be used or does not exist. public static String getNameOfBackFacingDevice() { - return getNameOfDevice(Camera.CameraInfo.CAMERA_FACING_BACK); + return getNameOfDevice(android.hardware.Camera.CameraInfo.CAMERA_FACING_BACK); } public static String getSupportedFormatsAsJson(int id) throws JSONException { @@ -194,7 +181,8 @@ public class CameraEnumerationAndroid { } } - public static int[] getFramerateRange(Camera.Parameters parameters, final int framerate) { + public static int[] getFramerateRange(android.hardware.Camera.Parameters parameters, + final int framerate) { List<int[]> listFpsRange = parameters.getSupportedPreviewFpsRange(); if (listFpsRange.isEmpty()) { Logging.w(TAG, "No supported preview fps range"); @@ -203,27 +191,30 @@ public class CameraEnumerationAndroid { return Collections.min(listFpsRange, new ClosestComparator<int[]>() { @Override int diff(int[] range) { - return abs(framerate - range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]) - + abs(framerate - range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); + final int maxFpsWeight = 10; + return range[android.hardware.Camera.Parameters.PREVIEW_FPS_MIN_INDEX] + + maxFpsWeight * abs(framerate + - range[android.hardware.Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); } }); } - public static Camera.Size getClosestSupportedSize( - List<Camera.Size> supportedSizes, final int requestedWidth, final int requestedHeight) { + public static android.hardware.Camera.Size getClosestSupportedSize( + List<android.hardware.Camera.Size> supportedSizes, final int requestedWidth, + final int requestedHeight) { return Collections.min(supportedSizes, - new ClosestComparator<Camera.Size>() { - @Override int diff(Camera.Size size) { + new ClosestComparator<android.hardware.Camera.Size>() { + @Override int diff(android.hardware.Camera.Size size) { return abs(requestedWidth - size.width) + abs(requestedHeight - size.height); } }); } private static String getNameOfDevice(int facing) { - final Camera.CameraInfo info = new Camera.CameraInfo(); - for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { + final android.hardware.Camera.CameraInfo info = new android.hardware.Camera.CameraInfo(); + for (int i = 0; i < android.hardware.Camera.getNumberOfCameras(); ++i) { try { - Camera.getCameraInfo(i, info); + android.hardware.Camera.getCameraInfo(i, info); if (info.facing == facing) { return getDeviceName(i); } diff --git a/talk/app/webrtc/java/android/org/webrtc/CameraEnumerator.java b/talk/app/webrtc/java/android/org/webrtc/CameraEnumerator.java index 2f35dc3493..54469cc341 100644 --- a/talk/app/webrtc/java/android/org/webrtc/CameraEnumerator.java +++ b/talk/app/webrtc/java/android/org/webrtc/CameraEnumerator.java @@ -27,7 +27,6 @@ package org.webrtc; -import android.hardware.Camera; import android.os.SystemClock; import org.webrtc.CameraEnumerationAndroid.CaptureFormat; @@ -60,11 +59,11 @@ public class CameraEnumerator implements CameraEnumerationAndroid.Enumerator { private List<CaptureFormat> enumerateFormats(int cameraId) { Logging.d(TAG, "Get supported formats for camera index " + cameraId + "."); final long startTimeMs = SystemClock.elapsedRealtime(); - final Camera.Parameters parameters; - Camera camera = null; + final android.hardware.Camera.Parameters parameters; + android.hardware.Camera camera = null; try { Logging.d(TAG, "Opening camera with index " + cameraId); - camera = Camera.open(cameraId); + camera = android.hardware.Camera.open(cameraId); parameters = camera.getParameters(); } catch (RuntimeException e) { Logging.e(TAG, "Open camera failed on camera index " + cameraId, e); @@ -84,10 +83,10 @@ public class CameraEnumerator implements CameraEnumerationAndroid.Enumerator { // getSupportedPreviewFpsRange() returns a sorted list. Take the fps range // corresponding to the highest fps. final int[] range = listFpsRange.get(listFpsRange.size() - 1); - minFps = range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; - maxFps = range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; + minFps = range[android.hardware.Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; + maxFps = range[android.hardware.Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; } - for (Camera.Size size : parameters.getSupportedPreviewSizes()) { + for (android.hardware.Camera.Size size : parameters.getSupportedPreviewSizes()) { formatList.add(new CaptureFormat(size.width, size.height, minFps, maxFps)); } } catch (Exception e) { diff --git a/talk/app/webrtc/java/android/org/webrtc/EglBase.java b/talk/app/webrtc/java/android/org/webrtc/EglBase.java index 2ee36882e8..035645bdd1 100644 --- a/talk/app/webrtc/java/android/org/webrtc/EglBase.java +++ b/talk/app/webrtc/java/android/org/webrtc/EglBase.java @@ -28,244 +28,108 @@ package org.webrtc; import android.graphics.SurfaceTexture; -import android.view.SurfaceHolder; - -import org.webrtc.Logging; +import android.view.Surface; import javax.microedition.khronos.egl.EGL10; -import javax.microedition.khronos.egl.EGLConfig; -import javax.microedition.khronos.egl.EGLContext; -import javax.microedition.khronos.egl.EGLDisplay; -import javax.microedition.khronos.egl.EGLSurface; + /** - * Holds EGL state and utility methods for handling an EGLContext, an EGLDisplay, and an EGLSurface. + * Holds EGL state and utility methods for handling an egl 1.0 EGLContext, an EGLDisplay, + * and an EGLSurface. */ -public final class EglBase { - private static final String TAG = "EglBase"; +public abstract class EglBase { + // EGL wrapper for an actual EGLContext. + public static class Context { + } + // These constants are taken from EGL14.EGL_OPENGL_ES2_BIT and EGL14.EGL_CONTEXT_CLIENT_VERSION. // https://android.googlesource.com/platform/frameworks/base/+/master/opengl/java/android/opengl/EGL14.java // This is similar to how GlSurfaceView does: // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.android/android/5.1.1_r1/android/opengl/GLSurfaceView.java#760 private static final int EGL_OPENGL_ES2_BIT = 4; - private static final int EGL_CONTEXT_CLIENT_VERSION = 0x3098; // Android-specific extension. private static final int EGL_RECORDABLE_ANDROID = 0x3142; - private final EGL10 egl; - private EGLContext eglContext; - private ConfigType configType; - private EGLConfig eglConfig; - private EGLDisplay eglDisplay; - private EGLSurface eglSurface = EGL10.EGL_NO_SURFACE; - - // EGLConfig constructor type. Influences eglChooseConfig arguments. - public static enum ConfigType { - // No special parameters. - PLAIN, - // Configures with EGL_SURFACE_TYPE = EGL_PBUFFER_BIT. - PIXEL_BUFFER, - // Configures with EGL_RECORDABLE_ANDROID = 1. - // Discourages EGL from using pixel formats that cannot efficiently be - // converted to something usable by the video encoder. - RECORDABLE - } - - // Create root context without any EGLSurface or parent EGLContext. This can be used for branching - // new contexts that share data. - public EglBase() { - this(EGL10.EGL_NO_CONTEXT, ConfigType.PLAIN); - } - - // Create a new context with the specified config type, sharing data with sharedContext. - public EglBase(EGLContext sharedContext, ConfigType configType) { - this.egl = (EGL10) EGLContext.getEGL(); - this.configType = configType; - eglDisplay = getEglDisplay(); - eglConfig = getEglConfig(eglDisplay, configType); - eglContext = createEglContext(sharedContext, eglDisplay, eglConfig); - } - - // Create EGLSurface from the Android SurfaceHolder. - public void createSurface(SurfaceHolder surfaceHolder) { - createSurfaceInternal(surfaceHolder); - } + public static final int[] CONFIG_PLAIN = { + EGL10.EGL_RED_SIZE, 8, + EGL10.EGL_GREEN_SIZE, 8, + EGL10.EGL_BLUE_SIZE, 8, + EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL10.EGL_NONE + }; + public static final int[] CONFIG_RGBA = { + EGL10.EGL_RED_SIZE, 8, + EGL10.EGL_GREEN_SIZE, 8, + EGL10.EGL_BLUE_SIZE, 8, + EGL10.EGL_ALPHA_SIZE, 8, + EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL10.EGL_NONE + }; + public static final int[] CONFIG_PIXEL_BUFFER = { + EGL10.EGL_RED_SIZE, 8, + EGL10.EGL_GREEN_SIZE, 8, + EGL10.EGL_BLUE_SIZE, 8, + EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL10.EGL_SURFACE_TYPE, EGL10.EGL_PBUFFER_BIT, + EGL10.EGL_NONE + }; + public static final int[] CONFIG_PIXEL_RGBA_BUFFER = { + EGL10.EGL_RED_SIZE, 8, + EGL10.EGL_GREEN_SIZE, 8, + EGL10.EGL_BLUE_SIZE, 8, + EGL10.EGL_ALPHA_SIZE, 8, + EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL10.EGL_SURFACE_TYPE, EGL10.EGL_PBUFFER_BIT, + EGL10.EGL_NONE + }; + public static final int[] CONFIG_RECORDABLE = { + EGL10.EGL_RED_SIZE, 8, + EGL10.EGL_GREEN_SIZE, 8, + EGL10.EGL_BLUE_SIZE, 8, + EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RECORDABLE_ANDROID, 1, + EGL10.EGL_NONE + }; + + // Create a new context with the specified config attributes, sharing data with sharedContext. + // |sharedContext| can be null. + public static EglBase create(Context sharedContext, int[] configAttributes) { + return (EglBase14.isEGL14Supported() + && (sharedContext == null || sharedContext instanceof EglBase14.Context)) + ? new EglBase14((EglBase14.Context) sharedContext, configAttributes) + : new EglBase10((EglBase10.Context) sharedContext, configAttributes); + } + + public static EglBase create() { + return create(null, CONFIG_PLAIN); + } + + public abstract void createSurface(Surface surface); // Create EGLSurface from the Android SurfaceTexture. - public void createSurface(SurfaceTexture surfaceTexture) { - createSurfaceInternal(surfaceTexture); - } - - // Create EGLSurface from either a SurfaceHolder or a SurfaceTexture. - private void createSurfaceInternal(Object nativeWindow) { - if (!(nativeWindow instanceof SurfaceHolder) && !(nativeWindow instanceof SurfaceTexture)) { - throw new IllegalStateException("Input must be either a SurfaceHolder or SurfaceTexture"); - } - checkIsNotReleased(); - if (configType == ConfigType.PIXEL_BUFFER) { - Logging.w(TAG, "This EGL context is configured for PIXEL_BUFFER, but uses regular Surface"); - } - if (eglSurface != EGL10.EGL_NO_SURFACE) { - throw new RuntimeException("Already has an EGLSurface"); - } - int[] surfaceAttribs = {EGL10.EGL_NONE}; - eglSurface = egl.eglCreateWindowSurface(eglDisplay, eglConfig, nativeWindow, surfaceAttribs); - if (eglSurface == EGL10.EGL_NO_SURFACE) { - throw new RuntimeException("Failed to create window surface"); - } - } + public abstract void createSurface(SurfaceTexture surfaceTexture); // Create dummy 1x1 pixel buffer surface so the context can be made current. - public void createDummyPbufferSurface() { - createPbufferSurface(1, 1); - } - - public void createPbufferSurface(int width, int height) { - checkIsNotReleased(); - if (configType != ConfigType.PIXEL_BUFFER) { - throw new RuntimeException( - "This EGL context is not configured to use a pixel buffer: " + configType); - } - if (eglSurface != EGL10.EGL_NO_SURFACE) { - throw new RuntimeException("Already has an EGLSurface"); - } - int[] surfaceAttribs = {EGL10.EGL_WIDTH, width, EGL10.EGL_HEIGHT, height, EGL10.EGL_NONE}; - eglSurface = egl.eglCreatePbufferSurface(eglDisplay, eglConfig, surfaceAttribs); - if (eglSurface == EGL10.EGL_NO_SURFACE) { - throw new RuntimeException("Failed to create pixel buffer surface"); - } - } + public abstract void createDummyPbufferSurface(); - public EGLContext getContext() { - return eglContext; - } + public abstract void createPbufferSurface(int width, int height); - public boolean hasSurface() { - return eglSurface != EGL10.EGL_NO_SURFACE; - } + public abstract Context getEglBaseContext(); - public int surfaceWidth() { - final int widthArray[] = new int[1]; - egl.eglQuerySurface(eglDisplay, eglSurface, EGL10.EGL_WIDTH, widthArray); - return widthArray[0]; - } + public abstract boolean hasSurface(); - public int surfaceHeight() { - final int heightArray[] = new int[1]; - egl.eglQuerySurface(eglDisplay, eglSurface, EGL10.EGL_HEIGHT, heightArray); - return heightArray[0]; - } + public abstract int surfaceWidth(); - public void releaseSurface() { - if (eglSurface != EGL10.EGL_NO_SURFACE) { - egl.eglDestroySurface(eglDisplay, eglSurface); - eglSurface = EGL10.EGL_NO_SURFACE; - } - } + public abstract int surfaceHeight(); - private void checkIsNotReleased() { - if (eglDisplay == EGL10.EGL_NO_DISPLAY || eglContext == EGL10.EGL_NO_CONTEXT - || eglConfig == null) { - throw new RuntimeException("This object has been released"); - } - } + public abstract void releaseSurface(); - public void release() { - checkIsNotReleased(); - releaseSurface(); - detachCurrent(); - egl.eglDestroyContext(eglDisplay, eglContext); - egl.eglTerminate(eglDisplay); - eglContext = EGL10.EGL_NO_CONTEXT; - eglDisplay = EGL10.EGL_NO_DISPLAY; - eglConfig = null; - } + public abstract void release(); - public void makeCurrent() { - checkIsNotReleased(); - if (eglSurface == EGL10.EGL_NO_SURFACE) { - throw new RuntimeException("No EGLSurface - can't make current"); - } - if (!egl.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { - throw new RuntimeException("eglMakeCurrent failed"); - } - } + public abstract void makeCurrent(); // Detach the current EGL context, so that it can be made current on another thread. - public void detachCurrent() { - if (!egl.eglMakeCurrent( - eglDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_CONTEXT)) { - throw new RuntimeException("eglMakeCurrent failed"); - } - } + public abstract void detachCurrent(); - public void swapBuffers() { - checkIsNotReleased(); - if (eglSurface == EGL10.EGL_NO_SURFACE) { - throw new RuntimeException("No EGLSurface - can't swap buffers"); - } - egl.eglSwapBuffers(eglDisplay, eglSurface); - } - - // Return an EGLDisplay, or die trying. - private EGLDisplay getEglDisplay() { - EGLDisplay eglDisplay = egl.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY); - if (eglDisplay == EGL10.EGL_NO_DISPLAY) { - throw new RuntimeException("Unable to get EGL10 display"); - } - int[] version = new int[2]; - if (!egl.eglInitialize(eglDisplay, version)) { - throw new RuntimeException("Unable to initialize EGL10"); - } - return eglDisplay; - } - - // Return an EGLConfig, or die trying. - private EGLConfig getEglConfig(EGLDisplay eglDisplay, ConfigType configType) { - // Always RGB888, GLES2. - int[] configAttributes = { - EGL10.EGL_RED_SIZE, 8, - EGL10.EGL_GREEN_SIZE, 8, - EGL10.EGL_BLUE_SIZE, 8, - EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, - EGL10.EGL_NONE, 0, // Allocate dummy fields for specific options. - EGL10.EGL_NONE - }; - - // Fill in dummy fields based on configType. - switch (configType) { - case PLAIN: - break; - case PIXEL_BUFFER: - configAttributes[configAttributes.length - 3] = EGL10.EGL_SURFACE_TYPE; - configAttributes[configAttributes.length - 2] = EGL10.EGL_PBUFFER_BIT; - break; - case RECORDABLE: - configAttributes[configAttributes.length - 3] = EGL_RECORDABLE_ANDROID; - configAttributes[configAttributes.length - 2] = 1; - break; - default: - throw new IllegalArgumentException(); - } - - EGLConfig[] configs = new EGLConfig[1]; - int[] numConfigs = new int[1]; - if (!egl.eglChooseConfig( - eglDisplay, configAttributes, configs, configs.length, numConfigs)) { - throw new RuntimeException("Unable to find RGB888 " + configType + " EGL config"); - } - return configs[0]; - } - - // Return an EGLConfig, or die trying. - private EGLContext createEglContext( - EGLContext sharedContext, EGLDisplay eglDisplay, EGLConfig eglConfig) { - int[] contextAttributes = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL10.EGL_NONE}; - EGLContext eglContext = - egl.eglCreateContext(eglDisplay, eglConfig, sharedContext, contextAttributes); - if (eglContext == EGL10.EGL_NO_CONTEXT) { - throw new RuntimeException("Failed to create EGL context"); - } - return eglContext; - } + public abstract void swapBuffers(); } diff --git a/talk/app/webrtc/java/android/org/webrtc/EglBase10.java b/talk/app/webrtc/java/android/org/webrtc/EglBase10.java new file mode 100644 index 0000000000..f2aa9857fa --- /dev/null +++ b/talk/app/webrtc/java/android/org/webrtc/EglBase10.java @@ -0,0 +1,299 @@ +/* + * libjingle + * Copyright 2015 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.webrtc; + +import android.graphics.Canvas; +import android.graphics.SurfaceTexture; +import android.graphics.Rect; +import android.view.Surface; +import android.view.SurfaceHolder; + +import javax.microedition.khronos.egl.EGL10; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLContext; +import javax.microedition.khronos.egl.EGLDisplay; +import javax.microedition.khronos.egl.EGLSurface; + +/** + * Holds EGL state and utility methods for handling an egl 1.0 EGLContext, an EGLDisplay, + * and an EGLSurface. + */ +final class EglBase10 extends EglBase { + // This constant is taken from EGL14.EGL_CONTEXT_CLIENT_VERSION. + private static final int EGL_CONTEXT_CLIENT_VERSION = 0x3098; + + private final EGL10 egl; + private EGLContext eglContext; + private EGLConfig eglConfig; + private EGLDisplay eglDisplay; + private EGLSurface eglSurface = EGL10.EGL_NO_SURFACE; + + // EGL wrapper for an actual EGLContext. + public static class Context extends EglBase.Context { + private final EGLContext eglContext; + + public Context(EGLContext eglContext) { + this.eglContext = eglContext; + } + } + + // Create a new context with the specified config type, sharing data with sharedContext. + EglBase10(Context sharedContext, int[] configAttributes) { + this.egl = (EGL10) EGLContext.getEGL(); + eglDisplay = getEglDisplay(); + eglConfig = getEglConfig(eglDisplay, configAttributes); + eglContext = createEglContext(sharedContext, eglDisplay, eglConfig); + } + + @Override + public void createSurface(Surface surface) { + /** + * We have to wrap Surface in a SurfaceHolder because for some reason eglCreateWindowSurface + * couldn't actually take a Surface object until API 17. Older versions fortunately just call + * SurfaceHolder.getSurface(), so we'll do that. No other methods are relevant. + */ + class FakeSurfaceHolder implements SurfaceHolder { + private final Surface surface; + + FakeSurfaceHolder(Surface surface) { + this.surface = surface; + } + + @Override + public void addCallback(Callback callback) {} + + @Override + public void removeCallback(Callback callback) {} + + @Override + public boolean isCreating() { + return false; + } + + @Deprecated + @Override + public void setType(int i) {} + + @Override + public void setFixedSize(int i, int i2) {} + + @Override + public void setSizeFromLayout() {} + + @Override + public void setFormat(int i) {} + + @Override + public void setKeepScreenOn(boolean b) {} + + @Override + public Canvas lockCanvas() { + return null; + } + + @Override + public Canvas lockCanvas(Rect rect) { + return null; + } + + @Override + public void unlockCanvasAndPost(Canvas canvas) {} + + @Override + public Rect getSurfaceFrame() { + return null; + } + + @Override + public Surface getSurface() { + return surface; + } + } + + createSurfaceInternal(new FakeSurfaceHolder(surface)); + } + + // Create EGLSurface from the Android SurfaceTexture. + @Override + public void createSurface(SurfaceTexture surfaceTexture) { + createSurfaceInternal(surfaceTexture); + } + + // Create EGLSurface from either a SurfaceHolder or a SurfaceTexture. + private void createSurfaceInternal(Object nativeWindow) { + if (!(nativeWindow instanceof SurfaceHolder) && !(nativeWindow instanceof SurfaceTexture)) { + throw new IllegalStateException("Input must be either a SurfaceHolder or SurfaceTexture"); + } + checkIsNotReleased(); + if (eglSurface != EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + int[] surfaceAttribs = {EGL10.EGL_NONE}; + eglSurface = egl.eglCreateWindowSurface(eglDisplay, eglConfig, nativeWindow, surfaceAttribs); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("Failed to create window surface"); + } + } + + // Create dummy 1x1 pixel buffer surface so the context can be made current. + @Override + public void createDummyPbufferSurface() { + createPbufferSurface(1, 1); + } + + @Override + public void createPbufferSurface(int width, int height) { + checkIsNotReleased(); + if (eglSurface != EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + int[] surfaceAttribs = {EGL10.EGL_WIDTH, width, EGL10.EGL_HEIGHT, height, EGL10.EGL_NONE}; + eglSurface = egl.eglCreatePbufferSurface(eglDisplay, eglConfig, surfaceAttribs); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("Failed to create pixel buffer surface"); + } + } + + @Override + public org.webrtc.EglBase.Context getEglBaseContext() { + return new EglBase10.Context(eglContext); + } + + @Override + public boolean hasSurface() { + return eglSurface != EGL10.EGL_NO_SURFACE; + } + + @Override + public int surfaceWidth() { + final int widthArray[] = new int[1]; + egl.eglQuerySurface(eglDisplay, eglSurface, EGL10.EGL_WIDTH, widthArray); + return widthArray[0]; + } + + @Override + public int surfaceHeight() { + final int heightArray[] = new int[1]; + egl.eglQuerySurface(eglDisplay, eglSurface, EGL10.EGL_HEIGHT, heightArray); + return heightArray[0]; + } + + @Override + public void releaseSurface() { + if (eglSurface != EGL10.EGL_NO_SURFACE) { + egl.eglDestroySurface(eglDisplay, eglSurface); + eglSurface = EGL10.EGL_NO_SURFACE; + } + } + + private void checkIsNotReleased() { + if (eglDisplay == EGL10.EGL_NO_DISPLAY || eglContext == EGL10.EGL_NO_CONTEXT + || eglConfig == null) { + throw new RuntimeException("This object has been released"); + } + } + + @Override + public void release() { + checkIsNotReleased(); + releaseSurface(); + detachCurrent(); + egl.eglDestroyContext(eglDisplay, eglContext); + egl.eglTerminate(eglDisplay); + eglContext = EGL10.EGL_NO_CONTEXT; + eglDisplay = EGL10.EGL_NO_DISPLAY; + eglConfig = null; + } + + @Override + public void makeCurrent() { + checkIsNotReleased(); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't make current"); + } + if (!egl.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { + throw new RuntimeException("eglMakeCurrent failed"); + } + } + + // Detach the current EGL context, so that it can be made current on another thread. + @Override + public void detachCurrent() { + if (!egl.eglMakeCurrent( + eglDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_CONTEXT)) { + throw new RuntimeException("eglMakeCurrent failed"); + } + } + + @Override + public void swapBuffers() { + checkIsNotReleased(); + if (eglSurface == EGL10.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't swap buffers"); + } + egl.eglSwapBuffers(eglDisplay, eglSurface); + } + + // Return an EGLDisplay, or die trying. + private EGLDisplay getEglDisplay() { + EGLDisplay eglDisplay = egl.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY); + if (eglDisplay == EGL10.EGL_NO_DISPLAY) { + throw new RuntimeException("Unable to get EGL10 display"); + } + int[] version = new int[2]; + if (!egl.eglInitialize(eglDisplay, version)) { + throw new RuntimeException("Unable to initialize EGL10"); + } + return eglDisplay; + } + + // Return an EGLConfig, or die trying. + private EGLConfig getEglConfig(EGLDisplay eglDisplay, int[] configAttributes) { + EGLConfig[] configs = new EGLConfig[1]; + int[] numConfigs = new int[1]; + if (!egl.eglChooseConfig( + eglDisplay, configAttributes, configs, configs.length, numConfigs)) { + throw new RuntimeException("Unable to find any matching EGL config"); + } + return configs[0]; + } + + // Return an EGLConfig, or die trying. + private EGLContext createEglContext( + Context sharedContext, EGLDisplay eglDisplay, EGLConfig eglConfig) { + int[] contextAttributes = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL10.EGL_NONE}; + EGLContext rootContext = + sharedContext == null ? EGL10.EGL_NO_CONTEXT : sharedContext.eglContext; + EGLContext eglContext = + egl.eglCreateContext(eglDisplay, eglConfig, rootContext, contextAttributes); + if (eglContext == EGL10.EGL_NO_CONTEXT) { + throw new RuntimeException("Failed to create EGL context"); + } + return eglContext; + } +} diff --git a/talk/app/webrtc/java/android/org/webrtc/EglBase14.java b/talk/app/webrtc/java/android/org/webrtc/EglBase14.java new file mode 100644 index 0000000000..c6f98c3b31 --- /dev/null +++ b/talk/app/webrtc/java/android/org/webrtc/EglBase14.java @@ -0,0 +1,254 @@ +/* + * libjingle + * Copyright 2015 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.webrtc; + +import android.annotation.TargetApi; +import android.graphics.SurfaceTexture; +import android.opengl.EGL14; +import android.opengl.EGLConfig; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLExt; +import android.opengl.EGLSurface; +import android.view.Surface; + +import org.webrtc.Logging; + +/** + * Holds EGL state and utility methods for handling an EGL14 EGLContext, an EGLDisplay, + * and an EGLSurface. + */ +@TargetApi(18) +final class EglBase14 extends EglBase { + private static final String TAG = "EglBase14"; + private static final int EGLExt_SDK_VERSION = android.os.Build.VERSION_CODES.JELLY_BEAN_MR2; + private static final int CURRENT_SDK_VERSION = android.os.Build.VERSION.SDK_INT; + private EGLContext eglContext; + private EGLConfig eglConfig; + private EGLDisplay eglDisplay; + private EGLSurface eglSurface = EGL14.EGL_NO_SURFACE; + + // EGL 1.4 is supported from API 17. But EGLExt that is used for setting presentation + // time stamp on a surface is supported from 18 so we require 18. + public static boolean isEGL14Supported() { + Logging.d(TAG, "SDK version: " + CURRENT_SDK_VERSION + + ". isEGL14Supported: " + (CURRENT_SDK_VERSION >= EGLExt_SDK_VERSION)); + return (CURRENT_SDK_VERSION >= EGLExt_SDK_VERSION); + } + + public static class Context extends EglBase.Context { + private final android.opengl.EGLContext egl14Context; + + Context(android.opengl.EGLContext eglContext) { + this.egl14Context = eglContext; + } + } + + // Create a new context with the specified config type, sharing data with sharedContext. + // |sharedContext| may be null. + EglBase14(EglBase14.Context sharedContext, int[] configAttributes) { + eglDisplay = getEglDisplay(); + eglConfig = getEglConfig(eglDisplay, configAttributes); + eglContext = createEglContext(sharedContext, eglDisplay, eglConfig); + } + + // Create EGLSurface from the Android Surface. + @Override + public void createSurface(Surface surface) { + createSurfaceInternal(surface); + } + + // Create EGLSurface from the Android SurfaceTexture. + @Override + public void createSurface(SurfaceTexture surfaceTexture) { + createSurfaceInternal(surfaceTexture); + } + + // Create EGLSurface from either Surface or SurfaceTexture. + private void createSurfaceInternal(Object surface) { + if (!(surface instanceof Surface) && !(surface instanceof SurfaceTexture)) { + throw new IllegalStateException("Input must be either a Surface or SurfaceTexture"); + } + checkIsNotReleased(); + if (eglSurface != EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + int[] surfaceAttribs = {EGL14.EGL_NONE}; + eglSurface = EGL14.eglCreateWindowSurface(eglDisplay, eglConfig, surface, surfaceAttribs, 0); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("Failed to create window surface"); + } + } + + @Override + public void createDummyPbufferSurface() { + createPbufferSurface(1, 1); + } + + @Override + public void createPbufferSurface(int width, int height) { + checkIsNotReleased(); + if (eglSurface != EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("Already has an EGLSurface"); + } + int[] surfaceAttribs = {EGL14.EGL_WIDTH, width, EGL14.EGL_HEIGHT, height, EGL14.EGL_NONE}; + eglSurface = EGL14.eglCreatePbufferSurface(eglDisplay, eglConfig, surfaceAttribs, 0); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("Failed to create pixel buffer surface"); + } + } + + @Override + public Context getEglBaseContext() { + return new EglBase14.Context(eglContext); + } + + @Override + public boolean hasSurface() { + return eglSurface != EGL14.EGL_NO_SURFACE; + } + + @Override + public int surfaceWidth() { + final int widthArray[] = new int[1]; + EGL14.eglQuerySurface(eglDisplay, eglSurface, EGL14.EGL_WIDTH, widthArray, 0); + return widthArray[0]; + } + + @Override + public int surfaceHeight() { + final int heightArray[] = new int[1]; + EGL14.eglQuerySurface(eglDisplay, eglSurface, EGL14.EGL_HEIGHT, heightArray, 0); + return heightArray[0]; + } + + @Override + public void releaseSurface() { + if (eglSurface != EGL14.EGL_NO_SURFACE) { + EGL14.eglDestroySurface(eglDisplay, eglSurface); + eglSurface = EGL14.EGL_NO_SURFACE; + } + } + + private void checkIsNotReleased() { + if (eglDisplay == EGL14.EGL_NO_DISPLAY || eglContext == EGL14.EGL_NO_CONTEXT + || eglConfig == null) { + throw new RuntimeException("This object has been released"); + } + } + + @Override + public void release() { + checkIsNotReleased(); + releaseSurface(); + detachCurrent(); + EGL14.eglDestroyContext(eglDisplay, eglContext); + EGL14.eglReleaseThread(); + EGL14.eglTerminate(eglDisplay); + eglContext = EGL14.EGL_NO_CONTEXT; + eglDisplay = EGL14.EGL_NO_DISPLAY; + eglConfig = null; + } + + @Override + public void makeCurrent() { + checkIsNotReleased(); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't make current"); + } + if (!EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)) { + throw new RuntimeException("eglMakeCurrent failed"); + } + } + + // Detach the current EGL context, so that it can be made current on another thread. + @Override + public void detachCurrent() { + if (!EGL14.eglMakeCurrent( + eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT)) { + throw new RuntimeException("eglMakeCurrent failed"); + } + } + + @Override + public void swapBuffers() { + checkIsNotReleased(); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't swap buffers"); + } + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + } + + public void swapBuffers(long timeStampNs) { + checkIsNotReleased(); + if (eglSurface == EGL14.EGL_NO_SURFACE) { + throw new RuntimeException("No EGLSurface - can't swap buffers"); + } + // See https://android.googlesource.com/platform/frameworks/native/+/tools_r22.2/opengl/specs/EGL_ANDROID_presentation_time.txt + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, timeStampNs); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + } + + // Return an EGLDisplay, or die trying. + private static EGLDisplay getEglDisplay() { + EGLDisplay eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY); + if (eglDisplay == EGL14.EGL_NO_DISPLAY) { + throw new RuntimeException("Unable to get EGL14 display"); + } + int[] version = new int[2]; + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + throw new RuntimeException("Unable to initialize EGL14"); + } + return eglDisplay; + } + + // Return an EGLConfig, or die trying. + private static EGLConfig getEglConfig(EGLDisplay eglDisplay, int[] configAttributes) { + EGLConfig[] configs = new EGLConfig[1]; + int[] numConfigs = new int[1]; + if (!EGL14.eglChooseConfig( + eglDisplay, configAttributes, 0, configs, 0, configs.length, numConfigs, 0)) { + throw new RuntimeException("Unable to find any matching EGL config"); + } + return configs[0]; + } + + // Return an EGLConfig, or die trying. + private static EGLContext createEglContext( + EglBase14.Context sharedContext, EGLDisplay eglDisplay, EGLConfig eglConfig) { + int[] contextAttributes = {EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE}; + EGLContext rootContext = + sharedContext == null ? EGL14.EGL_NO_CONTEXT : sharedContext.egl14Context; + EGLContext eglContext = + EGL14.eglCreateContext(eglDisplay, eglConfig, rootContext, contextAttributes, 0); + if (eglContext == EGL14.EGL_NO_CONTEXT) { + throw new RuntimeException("Failed to create EGL context"); + } + return eglContext; + } +} diff --git a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java index 2cb8af754d..6d3d5d2563 100644 --- a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java +++ b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java @@ -40,13 +40,13 @@ import java.util.IdentityHashMap; import java.util.Map; /** - * Helper class to draw a quad that covers the entire viewport. Rotation, mirror, and cropping is - * specified using a 4x4 texture coordinate transform matrix. The frame input can either be an OES - * texture or YUV textures in I420 format. The GL state must be preserved between draw calls, this - * is intentional to maximize performance. The function release() must be called manually to free - * the resources held by this object. + * Helper class to draw an opaque quad on the target viewport location. Rotation, mirror, and + * cropping is specified using a 4x4 texture coordinate transform matrix. The frame input can either + * be an OES texture or YUV textures in I420 format. The GL state must be preserved between draw + * calls, this is intentional to maximize performance. The function release() must be called + * manually to free the resources held by this object. */ -public class GlRectDrawer { +public class GlRectDrawer implements RendererCommon.GlDrawer { // Simple vertex shader, used for both YUV and OES. private static final String VERTEX_SHADER_STRING = "varying vec2 interp_tc;\n" @@ -118,67 +118,31 @@ public class GlRectDrawer { 1.0f, 1.0f // Top right. }); - // The keys are one of the fragments shaders above. - private final Map<String, GlShader> shaders = new IdentityHashMap<String, GlShader>(); - private GlShader currentShader; - private float[] currentTexMatrix; - private int texMatrixLocation; - // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width. - // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader that - // handles stride and compare performance with intermediate copy. - private ByteBuffer copyBuffer; + private static class Shader { + public final GlShader glShader; + public final int texMatrixLocation; - /** - * Upload |planes| into |outputYuvTextures|, taking stride into consideration. |outputYuvTextures| - * must have been generated in advance. - */ - public void uploadYuvData( - int[] outputYuvTextures, int width, int height, int[] strides, ByteBuffer[] planes) { - // Make a first pass to see if we need a temporary copy buffer. - int copyCapacityNeeded = 0; - for (int i = 0; i < 3; ++i) { - final int planeWidth = (i == 0) ? width : width / 2; - final int planeHeight = (i == 0) ? height : height / 2; - if (strides[i] > planeWidth) { - copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidth * planeHeight); - } - } - // Allocate copy buffer if necessary. - if (copyCapacityNeeded > 0 - && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { - copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); - } - // Upload each plane. - for (int i = 0; i < 3; ++i) { - GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputYuvTextures[i]); - final int planeWidth = (i == 0) ? width : width / 2; - final int planeHeight = (i == 0) ? height : height / 2; - // GLES only accepts packed data, i.e. stride == planeWidth. - final ByteBuffer packedByteBuffer; - if (strides[i] == planeWidth) { - // Input is packed already. - packedByteBuffer = planes[i]; - } else { - VideoRenderer.nativeCopyPlane( - planes[i], planeWidth, planeHeight, strides[i], copyBuffer, planeWidth); - packedByteBuffer = copyBuffer; - } - GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidth, planeHeight, 0, - GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); + public Shader(String fragmentShader) { + this.glShader = new GlShader(VERTEX_SHADER_STRING, fragmentShader); + this.texMatrixLocation = glShader.getUniformLocation("texMatrix"); } } + // The keys are one of the fragments shaders above. + private final Map<String, Shader> shaders = new IdentityHashMap<String, Shader>(); + /** * Draw an OES texture frame with specified texture transformation matrix. Required resources are * allocated at the first call to this function. */ - public void drawOes(int oesTextureId, float[] texMatrix) { - prepareShader(OES_FRAGMENT_SHADER_STRING); + @Override + public void drawOes(int oesTextureId, float[] texMatrix, int x, int y, int width, int height) { + prepareShader(OES_FRAGMENT_SHADER_STRING, texMatrix); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); // updateTexImage() may be called from another thread in another EGL context, so we need to // bind/unbind the texture in each draw call so that GLES understads it's a new texture. GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId); - drawRectangle(texMatrix); + drawRectangle(x, y, width, height); GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); } @@ -186,10 +150,12 @@ public class GlRectDrawer { * Draw a RGB(A) texture frame with specified texture transformation matrix. Required resources * are allocated at the first call to this function. */ - public void drawRgb(int textureId, float[] texMatrix) { - prepareShader(RGB_FRAGMENT_SHADER_STRING); + @Override + public void drawRgb(int textureId, float[] texMatrix, int x, int y, int width, int height) { + prepareShader(RGB_FRAGMENT_SHADER_STRING, texMatrix); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); - drawRectangle(texMatrix); + drawRectangle(x, y, width, height); // Unbind the texture as a precaution. GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); } @@ -198,14 +164,15 @@ public class GlRectDrawer { * Draw a YUV frame with specified texture transformation matrix. Required resources are * allocated at the first call to this function. */ - public void drawYuv(int[] yuvTextures, float[] texMatrix) { - prepareShader(YUV_FRAGMENT_SHADER_STRING); + @Override + public void drawYuv(int[] yuvTextures, float[] texMatrix, int x, int y, int width, int height) { + prepareShader(YUV_FRAGMENT_SHADER_STRING, texMatrix); // Bind the textures. for (int i = 0; i < 3; ++i) { GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); } - drawRectangle(texMatrix); + drawRectangle(x, y, width, height); // Unbind the textures as a precaution.. for (int i = 0; i < 3; ++i) { GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); @@ -213,60 +180,51 @@ public class GlRectDrawer { } } - private void drawRectangle(float[] texMatrix) { - // Try avoid uploading the texture if possible. - if (!Arrays.equals(currentTexMatrix, texMatrix)) { - currentTexMatrix = texMatrix.clone(); - // Copy the texture transformation matrix over. - GLES20.glUniformMatrix4fv(texMatrixLocation, 1, false, texMatrix, 0); - } + private void drawRectangle(int x, int y, int width, int height) { // Draw quad. + GLES20.glViewport(x, y, width, height); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); } - private void prepareShader(String fragmentShader) { - // Lazy allocation. - if (!shaders.containsKey(fragmentShader)) { - final GlShader shader = new GlShader(VERTEX_SHADER_STRING, fragmentShader); + private void prepareShader(String fragmentShader, float[] texMatrix) { + final Shader shader; + if (shaders.containsKey(fragmentShader)) { + shader = shaders.get(fragmentShader); + } else { + // Lazy allocation. + shader = new Shader(fragmentShader); shaders.put(fragmentShader, shader); - shader.useProgram(); + shader.glShader.useProgram(); // Initialize fragment shader uniform values. if (fragmentShader == YUV_FRAGMENT_SHADER_STRING) { - GLES20.glUniform1i(shader.getUniformLocation("y_tex"), 0); - GLES20.glUniform1i(shader.getUniformLocation("u_tex"), 1); - GLES20.glUniform1i(shader.getUniformLocation("v_tex"), 2); + GLES20.glUniform1i(shader.glShader.getUniformLocation("y_tex"), 0); + GLES20.glUniform1i(shader.glShader.getUniformLocation("u_tex"), 1); + GLES20.glUniform1i(shader.glShader.getUniformLocation("v_tex"), 2); } else if (fragmentShader == RGB_FRAGMENT_SHADER_STRING) { - GLES20.glUniform1i(shader.getUniformLocation("rgb_tex"), 0); + GLES20.glUniform1i(shader.glShader.getUniformLocation("rgb_tex"), 0); } else if (fragmentShader == OES_FRAGMENT_SHADER_STRING) { - GLES20.glUniform1i(shader.getUniformLocation("oes_tex"), 0); + GLES20.glUniform1i(shader.glShader.getUniformLocation("oes_tex"), 0); } else { throw new IllegalStateException("Unknown fragment shader: " + fragmentShader); } GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); // Initialize vertex shader attributes. - shader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF); - shader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF); - } - - // Update GLES state if shader is not already current. - final GlShader shader = shaders.get(fragmentShader); - if (currentShader != shader) { - currentShader = shader; - shader.useProgram(); - GLES20.glActiveTexture(GLES20.GL_TEXTURE0); - currentTexMatrix = null; - texMatrixLocation = shader.getUniformLocation("texMatrix"); + shader.glShader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF); + shader.glShader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF); } + shader.glShader.useProgram(); + // Copy the texture transformation matrix over. + GLES20.glUniformMatrix4fv(shader.texMatrixLocation, 1, false, texMatrix, 0); } /** * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked. */ + @Override public void release() { - for (GlShader shader : shaders.values()) { - shader.release(); + for (Shader shader : shaders.values()) { + shader.glShader.release(); } shaders.clear(); - copyBuffer = null; } } diff --git a/talk/app/webrtc/java/android/org/webrtc/NetworkMonitorAutoDetect.java b/talk/app/webrtc/java/android/org/webrtc/NetworkMonitorAutoDetect.java index e3a7850db4..950dcdfa44 100644 --- a/talk/app/webrtc/java/android/org/webrtc/NetworkMonitorAutoDetect.java +++ b/talk/app/webrtc/java/android/org/webrtc/NetworkMonitorAutoDetect.java @@ -55,7 +55,7 @@ import android.util.Log; * ACCESS_NETWORK_STATE permission. */ public class NetworkMonitorAutoDetect extends BroadcastReceiver { - static enum ConnectionType { + public static enum ConnectionType { CONNECTION_UNKNOWN, CONNECTION_ETHERNET, CONNECTION_WIFI, @@ -96,6 +96,10 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { /** Queries the ConnectivityManager for information about the current connection. */ static class ConnectivityManagerDelegate { + /** + * Note: In some rare Android systems connectivityManager is null. We handle that + * gracefully below. + */ private final ConnectivityManager connectivityManager; ConnectivityManagerDelegate(Context context) { @@ -114,6 +118,9 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { * default network. */ NetworkState getNetworkState() { + if (connectivityManager == null) { + return new NetworkState(false, -1, -1); + } return getNetworkState(connectivityManager.getActiveNetworkInfo()); } @@ -123,6 +130,9 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { */ @SuppressLint("NewApi") NetworkState getNetworkState(Network network) { + if (connectivityManager == null) { + return new NetworkState(false, -1, -1); + } return getNetworkState(connectivityManager.getNetworkInfo(network)); } @@ -142,6 +152,9 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { */ @SuppressLint("NewApi") Network[] getAllNetworks() { + if (connectivityManager == null) { + return new Network[0]; + } return connectivityManager.getAllNetworks(); } @@ -152,6 +165,9 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { */ @SuppressLint("NewApi") int getDefaultNetId() { + if (connectivityManager == null) { + return INVALID_NET_ID; + } // Android Lollipop had no API to get the default network; only an // API to return the NetworkInfo for the default network. To // determine the default network one can find the network with @@ -188,6 +204,9 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { */ @SuppressLint("NewApi") boolean hasInternetCapability(Network network) { + if (connectivityManager == null) { + return false; + } final NetworkCapabilities capabilities = connectivityManager.getNetworkCapabilities(network); return capabilities != null && capabilities.hasCapability(NET_CAPABILITY_INTERNET); @@ -240,7 +259,6 @@ public class NetworkMonitorAutoDetect extends BroadcastReceiver { static final int INVALID_NET_ID = -1; private static final String TAG = "NetworkMonitorAutoDetect"; - private static final int UNKNOWN_LINK_SPEED = -1; private final IntentFilter intentFilter; // Observer for the connection type change. diff --git a/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java b/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java index 94d180da5a..5ada4cc416 100644 --- a/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java +++ b/talk/app/webrtc/java/android/org/webrtc/RendererCommon.java @@ -28,8 +28,11 @@ package org.webrtc; import android.graphics.Point; +import android.opengl.GLES20; import android.opengl.Matrix; +import java.nio.ByteBuffer; + /** * Static helper functions for renderer implementations. */ @@ -47,6 +50,73 @@ public class RendererCommon { public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation); } + /** Interface for rendering frames on an EGLSurface. */ + public static interface GlDrawer { + /** + * Functions for drawing frames with different sources. The rendering surface target is + * implied by the current EGL context of the calling thread and requires no explicit argument. + * The coordinates specify the viewport location on the surface target. + */ + void drawOes(int oesTextureId, float[] texMatrix, int x, int y, int width, int height); + void drawRgb(int textureId, float[] texMatrix, int x, int y, int width, int height); + void drawYuv(int[] yuvTextures, float[] texMatrix, int x, int y, int width, int height); + + /** + * Release all GL resources. This needs to be done manually, otherwise resources may leak. + */ + void release(); + } + + /** + * Helper class for uploading YUV bytebuffer frames to textures that handles stride > width. This + * class keeps an internal ByteBuffer to avoid unnecessary allocations for intermediate copies. + */ + public static class YuvUploader { + // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width. + // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader + // that handles stride and compare performance with intermediate copy. + private ByteBuffer copyBuffer; + + /** + * Upload |planes| into |outputYuvTextures|, taking stride into consideration. + * |outputYuvTextures| must have been generated in advance. + */ + public void uploadYuvData( + int[] outputYuvTextures, int width, int height, int[] strides, ByteBuffer[] planes) { + final int[] planeWidths = new int[] {width, width / 2, width / 2}; + final int[] planeHeights = new int[] {height, height / 2, height / 2}; + // Make a first pass to see if we need a temporary copy buffer. + int copyCapacityNeeded = 0; + for (int i = 0; i < 3; ++i) { + if (strides[i] > planeWidths[i]) { + copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidths[i] * planeHeights[i]); + } + } + // Allocate copy buffer if necessary. + if (copyCapacityNeeded > 0 + && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { + copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); + } + // Upload each plane. + for (int i = 0; i < 3; ++i) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputYuvTextures[i]); + // GLES only accepts packed data, i.e. stride == planeWidth. + final ByteBuffer packedByteBuffer; + if (strides[i] == planeWidths[i]) { + // Input is packed already. + packedByteBuffer = planes[i]; + } else { + VideoRenderer.nativeCopyPlane( + planes[i], planeWidths[i], planeHeights[i], strides[i], copyBuffer, planeWidths[i]); + packedByteBuffer = copyBuffer; + } + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i], + planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); + } + } + } + // Types of video scaling: // SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by // maintaining the aspect ratio (black borders may be displayed). @@ -182,9 +252,9 @@ public class RendererCommon { } // Each dimension is constrained on max display size and how much we are allowed to crop. final int width = Math.min(maxDisplayWidth, - (int) (maxDisplayHeight / minVisibleFraction * videoAspectRatio)); + Math.round(maxDisplayHeight / minVisibleFraction * videoAspectRatio)); final int height = Math.min(maxDisplayHeight, - (int) (maxDisplayWidth / minVisibleFraction / videoAspectRatio)); + Math.round(maxDisplayWidth / minVisibleFraction / videoAspectRatio)); return new Point(width, height); } } diff --git a/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java b/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java index b9c158f848..b001d2a101 100644 --- a/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java +++ b/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java @@ -35,12 +35,12 @@ import android.os.Handler; import android.os.HandlerThread; import android.os.SystemClock; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import javax.microedition.khronos.egl.EGLContext; - /** * Helper class to create and synchronize access to a SurfaceTexture. The caller will get notified * of new frames in onTextureFrameAvailable(), and should call returnTextureFrame() when done with @@ -51,7 +51,7 @@ import javax.microedition.khronos.egl.EGLContext; * wrapping texture frames into webrtc::VideoFrames and also handles calling returnTextureFrame() * when the webrtc::VideoFrame is no longer used. */ -final class SurfaceTextureHelper { +class SurfaceTextureHelper { private static final String TAG = "SurfaceTextureHelper"; /** * Callback interface for being notified that a new texture frame is available. The calls will be @@ -65,7 +65,7 @@ final class SurfaceTextureHelper { int oesTextureId, float[] transformMatrix, long timestampNs); } - public static SurfaceTextureHelper create(EGLContext sharedContext) { + public static SurfaceTextureHelper create(EglBase.Context sharedContext) { return create(sharedContext, null); } @@ -74,7 +74,8 @@ final class SurfaceTextureHelper { * |handler| is non-null, the callback will be executed on that handler's thread. If |handler| is * null, a dedicated private thread is created for the callbacks. */ - public static SurfaceTextureHelper create(final EGLContext sharedContext, final Handler handler) { + public static SurfaceTextureHelper create(final EglBase.Context sharedContext, + final Handler handler) { final Handler finalHandler; if (handler != null) { finalHandler = handler; @@ -94,25 +95,240 @@ final class SurfaceTextureHelper { }); } + // State for YUV conversion, instantiated on demand. + static private class YuvConverter { + private final EglBase eglBase; + private final GlShader shader; + private boolean released = false; + + // Vertex coordinates in Normalized Device Coordinates, i.e. + // (-1, -1) is bottom-left and (1, 1) is top-right. + private static final FloatBuffer DEVICE_RECTANGLE = + GlUtil.createFloatBuffer(new float[] { + -1.0f, -1.0f, // Bottom left. + 1.0f, -1.0f, // Bottom right. + -1.0f, 1.0f, // Top left. + 1.0f, 1.0f, // Top right. + }); + + // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. + private static final FloatBuffer TEXTURE_RECTANGLE = + GlUtil.createFloatBuffer(new float[] { + 0.0f, 0.0f, // Bottom left. + 1.0f, 0.0f, // Bottom right. + 0.0f, 1.0f, // Top left. + 1.0f, 1.0f // Top right. + }); + + private static final String VERTEX_SHADER = + "varying vec2 interp_tc;\n" + + "attribute vec4 in_pos;\n" + + "attribute vec4 in_tc;\n" + + "\n" + + "uniform mat4 texMatrix;\n" + + "\n" + + "void main() {\n" + + " gl_Position = in_pos;\n" + + " interp_tc = (texMatrix * in_tc).xy;\n" + + "}\n"; + + private static final String FRAGMENT_SHADER = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "varying vec2 interp_tc;\n" + + "\n" + + "uniform samplerExternalOES oesTex;\n" + // Difference in texture coordinate corresponding to one + // sub-pixel in the x direction. + + "uniform vec2 xUnit;\n" + // Color conversion coefficients, including constant term + + "uniform vec4 coeffs;\n" + + "\n" + + "void main() {\n" + // Since the alpha read from the texture is always 1, this could + // be written as a mat4 x vec4 multiply. However, that seems to + // give a worse framerate, possibly because the additional + // multiplies by 1.0 consume resources. TODO(nisse): Could also + // try to do it as a vec3 x mat3x4, followed by an add in of a + // constant vector. + + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" + + " texture2D(oesTex, interp_tc - 1.5 * xUnit).rgb);\n" + + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" + + " texture2D(oesTex, interp_tc - 0.5 * xUnit).rgb);\n" + + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" + + " texture2D(oesTex, interp_tc + 0.5 * xUnit).rgb);\n" + + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" + + " texture2D(oesTex, interp_tc + 1.5 * xUnit).rgb);\n" + + "}\n"; + + private int texMatrixLoc; + private int xUnitLoc; + private int coeffsLoc;; + + YuvConverter (EglBase.Context sharedContext) { + eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_RGBA_BUFFER); + eglBase.createDummyPbufferSurface(); + eglBase.makeCurrent(); + + shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER); + shader.useProgram(); + texMatrixLoc = shader.getUniformLocation("texMatrix"); + xUnitLoc = shader.getUniformLocation("xUnit"); + coeffsLoc = shader.getUniformLocation("coeffs"); + GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0); + GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); + // Initialize vertex shader attributes. + shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE); + // If the width is not a multiple of 4 pixels, the texture + // will be scaled up slightly and clipped at the right border. + shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE); + eglBase.detachCurrent(); + } + + synchronized void convert(ByteBuffer buf, + int width, int height, int stride, int textureId, float [] transformMatrix) { + if (released) { + throw new IllegalStateException( + "YuvConverter.convert called on released object"); + } + + // We draw into a buffer laid out like + // + // +---------+ + // | | + // | Y | + // | | + // | | + // +----+----+ + // | U | V | + // | | | + // +----+----+ + // + // In memory, we use the same stride for all of Y, U and V. The + // U data starts at offset |height| * |stride| from the Y data, + // and the V data starts at at offset |stride/2| from the U + // data, with rows of U and V data alternating. + // + // Now, it would have made sense to allocate a pixel buffer with + // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, + // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be + // unsupported by devices. So do the following hack: Allocate an + // RGBA buffer, of width |stride|/4. To render each of these + // large pixels, sample the texture at 4 different x coordinates + // and store the results in the four components. + // + // Since the V data needs to start on a boundary of such a + // larger pixel, it is not sufficient that |stride| is even, it + // has to be a multiple of 8 pixels. + + if (stride % 8 != 0) { + throw new IllegalArgumentException( + "Invalid stride, must be a multiple of 8"); + } + if (stride < width){ + throw new IllegalArgumentException( + "Invalid stride, must >= width"); + } + + int y_width = (width+3) / 4; + int uv_width = (width+7) / 8; + int uv_height = (height+1)/2; + int total_height = height + uv_height; + int size = stride * total_height; + + if (buf.capacity() < size) { + throw new IllegalArgumentException("YuvConverter.convert called with too small buffer"); + } + // Produce a frame buffer starting at top-left corner, not + // bottom-left. + transformMatrix = + RendererCommon.multiplyMatrices(transformMatrix, + RendererCommon.verticalFlipMatrix()); + + // Create new pBuffferSurface with the correct size if needed. + if (eglBase.hasSurface()) { + if (eglBase.surfaceWidth() != stride/4 || + eglBase.surfaceHeight() != total_height){ + eglBase.releaseSurface(); + eglBase.createPbufferSurface(stride/4, total_height); + } + } else { + eglBase.createPbufferSurface(stride/4, total_height); + } + + eglBase.makeCurrent(); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); + GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0); + + // Draw Y + GLES20.glViewport(0, 0, y_width, height); + // Matrix * (1;0;0;0) / width. Note that opengl uses column major order. + GLES20.glUniform2f(xUnitLoc, + transformMatrix[0] / width, + transformMatrix[1] / width); + // Y'UV444 to RGB888, see + // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion. + // We use the ITU-R coefficients for U and V */ + GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + + // Draw U + GLES20.glViewport(0, height, uv_width, uv_height); + // Matrix * (1;0;0;0) / (2*width). Note that opengl uses column major order. + GLES20.glUniform2f(xUnitLoc, + transformMatrix[0] / (2.0f*width), + transformMatrix[1] / (2.0f*width)); + GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + + // Draw V + GLES20.glViewport(stride/8, height, uv_width, uv_height); + GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + + GLES20.glReadPixels(0, 0, stride/4, total_height, GLES20.GL_RGBA, + GLES20.GL_UNSIGNED_BYTE, buf); + + GlUtil.checkNoGLES2Error("YuvConverter.convert"); + + // Unbind texture. Reportedly needed on some devices to get + // the texture updated from the camera. + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); + eglBase.detachCurrent(); + } + + synchronized void release() { + released = true; + eglBase.makeCurrent(); + shader.release(); + eglBase.release(); + } + } + private final Handler handler; - private final boolean isOwningThread; + private boolean isOwningThread; private final EglBase eglBase; private final SurfaceTexture surfaceTexture; private final int oesTextureId; + private YuvConverter yuvConverter; + private OnTextureFrameAvailableListener listener; // The possible states of this class. private boolean hasPendingTexture = false; - private boolean isTextureInUse = false; + private volatile boolean isTextureInUse = false; private boolean isQuitting = false; - private SurfaceTextureHelper(EGLContext sharedContext, Handler handler, boolean isOwningThread) { + private SurfaceTextureHelper(EglBase.Context sharedContext, + Handler handler, boolean isOwningThread) { if (handler.getLooper().getThread() != Thread.currentThread()) { throw new IllegalStateException("SurfaceTextureHelper must be created on the handler thread"); } this.handler = handler; this.isOwningThread = isOwningThread; - eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER); + eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); eglBase.createDummyPbufferSurface(); eglBase.makeCurrent(); @@ -120,6 +336,18 @@ final class SurfaceTextureHelper { surfaceTexture = new SurfaceTexture(oesTextureId); } + private YuvConverter getYuvConverter() { + // yuvConverter is assigned once + if (yuvConverter != null) + return yuvConverter; + + synchronized(this) { + if (yuvConverter == null) + yuvConverter = new YuvConverter(eglBase.getEglBaseContext()); + return yuvConverter; + } + } + /** * Start to stream textures to the given |listener|. * A Listener can only be set once. @@ -164,12 +392,19 @@ final class SurfaceTextureHelper { }); } + public boolean isTextureInUse() { + return isTextureInUse; + } + /** * Call disconnect() to stop receiving frames. Resources are released when the texture frame has * been returned by a call to returnTextureFrame(). You are guaranteed to not receive any more * onTextureFrameAvailable() after this function returns. */ public void disconnect() { + if (!isOwningThread) { + throw new IllegalStateException("Must call disconnect(handler)."); + } if (handler.getLooper().getThread() == Thread.currentThread()) { isQuitting = true; if (!isTextureInUse) { @@ -190,6 +425,28 @@ final class SurfaceTextureHelper { ThreadUtils.awaitUninterruptibly(barrier); } + /** + * Call disconnect() to stop receiving frames and quit the looper used by |handler|. + * Resources are released when the texture frame has been returned by a call to + * returnTextureFrame(). You are guaranteed to not receive any more + * onTextureFrameAvailable() after this function returns. + */ + public void disconnect(Handler handler) { + if (this.handler != handler) { + throw new IllegalStateException("Wrong handler."); + } + isOwningThread = true; + disconnect(); + } + + public void textureToYUV(ByteBuffer buf, + int width, int height, int stride, int textureId, float [] transformMatrix) { + if (textureId != oesTextureId) + throw new IllegalStateException("textureToByteBuffer called with unexpected textureId"); + + getYuvConverter().convert(buf, width, height, stride, textureId, transformMatrix); + } + private void tryDeliverTextureFrame() { if (handler.getLooper().getThread() != Thread.currentThread()) { throw new IllegalStateException("Wrong thread."); @@ -218,12 +475,14 @@ final class SurfaceTextureHelper { if (isTextureInUse || !isQuitting) { throw new IllegalStateException("Unexpected release."); } + synchronized (this) { + if (yuvConverter != null) + yuvConverter.release(); + } eglBase.makeCurrent(); GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); surfaceTexture.release(); eglBase.release(); - if (isOwningThread) { - handler.getLooper().quit(); - } + handler.getLooper().quit(); } } diff --git a/talk/app/webrtc/java/android/org/webrtc/SurfaceViewRenderer.java b/talk/app/webrtc/java/android/org/webrtc/SurfaceViewRenderer.java index d7c9e2af0a..fa199b33c8 100644 --- a/talk/app/webrtc/java/android/org/webrtc/SurfaceViewRenderer.java +++ b/talk/app/webrtc/java/android/org/webrtc/SurfaceViewRenderer.java @@ -28,10 +28,9 @@ package org.webrtc; import android.content.Context; +import android.content.res.Resources.NotFoundException; import android.graphics.Point; -import android.graphics.SurfaceTexture; import android.opengl.GLES20; -import android.opengl.Matrix; import android.os.Handler; import android.os.HandlerThread; import android.util.AttributeSet; @@ -67,7 +66,8 @@ public class SurfaceViewRenderer extends SurfaceView // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed // from the render thread. private EglBase eglBase; - private GlRectDrawer drawer; + private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader(); + private RendererCommon.GlDrawer drawer; // Texture ids for YUV frames. Allocated on first arrival of a YUV frame. private int[] yuvTextures = null; @@ -77,23 +77,22 @@ public class SurfaceViewRenderer extends SurfaceView // These variables are synchronized on |layoutLock|. private final Object layoutLock = new Object(); - // These three different dimension values are used to keep track of the state in these functions: - // requestLayout() -> onMeasure() -> onLayout() -> surfaceChanged(). - // requestLayout() is triggered internally by frame size changes, but can also be triggered - // externally by layout update requests. - // Most recent measurement specification from onMeasure(). - private int widthSpec; - private int heightSpec; - // Current size on screen in pixels. Updated in onLayout(), and should be consistent with - // |widthSpec|/|heightSpec| after that. - private int layoutWidth; - private int layoutHeight; - // Current surface size of the underlying Surface. Updated in surfaceChanged(), and should be - // consistent with |layoutWidth|/|layoutHeight| after that. + // These dimension values are used to keep track of the state in these functions: onMeasure(), + // onLayout(), and surfaceChanged(). A new layout is triggered with requestLayout(). This happens + // internally when the incoming frame size changes. requestLayout() can also be triggered + // externally. The layout change is a two pass process: first onMeasure() is called in a top-down + // traversal of the View tree, followed by an onLayout() pass that is also top-down. During the + // onLayout() pass, each parent is responsible for positioning its children using the sizes + // computed in the measure pass. + // |desiredLayoutsize| is the layout size we have requested in onMeasure() and are waiting for to + // take effect. + private Point desiredLayoutSize = new Point(); + // |layoutSize|/|surfaceSize| is the actual current layout/surface size. They are updated in + // onLayout() and surfaceChanged() respectively. + private final Point layoutSize = new Point(); // TODO(magjed): Enable hardware scaler with SurfaceHolder.setFixedSize(). This will decouple // layout and surface size. - private int surfaceWidth; - private int surfaceHeight; + private final Point surfaceSize = new Point(); // |isSurfaceCreated| keeps track of the current status in surfaceCreated()/surfaceDestroyed(). private boolean isSurfaceCreated; // Last rendered frame dimensions, or 0 if no frame has been rendered yet. @@ -121,12 +120,18 @@ public class SurfaceViewRenderer extends SurfaceView // Time in ns spent in renderFrameOnRenderThread() function. private long renderTimeNs; - // Runnable for posting frames to render thread.. + // Runnable for posting frames to render thread. private final Runnable renderFrameRunnable = new Runnable() { @Override public void run() { renderFrameOnRenderThread(); } }; + // Runnable for clearing Surface to black. + private final Runnable makeBlackRunnable = new Runnable() { + @Override public void run() { + makeBlack(); + } + }; /** * Standard View constructor. In order to render something, you must first call init(). @@ -149,17 +154,28 @@ public class SurfaceViewRenderer extends SurfaceView * reinitialize the renderer after a previous init()/release() cycle. */ public void init( - EGLContext sharedContext, RendererCommon.RendererEvents rendererEvents) { + EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents) { + init(sharedContext, rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawer()); + } + + /** + * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used + * for drawing frames on the EGLSurface. This class is responsible for calling release() on + * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous + * init()/release() cycle. + */ + public void init(EglBase.Context sharedContext, RendererCommon.RendererEvents rendererEvents, + int[] configAttributes, RendererCommon.GlDrawer drawer) { synchronized (handlerLock) { if (renderThreadHandler != null) { - throw new IllegalStateException("Already initialized"); + throw new IllegalStateException(getResourceName() + "Already initialized"); } - Logging.d(TAG, "Initializing"); + Logging.d(TAG, getResourceName() + "Initializing."); this.rendererEvents = rendererEvents; + this.drawer = drawer; renderThread = new HandlerThread(TAG); renderThread.start(); - drawer = new GlRectDrawer(); - eglBase = new EglBase(sharedContext, EglBase.ConfigType.PLAIN); + eglBase = EglBase.create(sharedContext, configAttributes); renderThreadHandler = new Handler(renderThread.getLooper()); } tryCreateEglSurface(); @@ -174,8 +190,8 @@ public class SurfaceViewRenderer extends SurfaceView runOnRenderThread(new Runnable() { @Override public void run() { synchronized (layoutLock) { - if (isSurfaceCreated) { - eglBase.createSurface(getHolder()); + if (isSurfaceCreated && !eglBase.hasSurface()) { + eglBase.createSurface(getHolder().getSurface()); eglBase.makeCurrent(); // Necessary for YUV frames with odd width. GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); @@ -195,7 +211,7 @@ public class SurfaceViewRenderer extends SurfaceView final CountDownLatch eglCleanupBarrier = new CountDownLatch(1); synchronized (handlerLock) { if (renderThreadHandler == null) { - Logging.d(TAG, "Already released"); + Logging.d(TAG, getResourceName() + "Already released"); return; } // Release EGL and GL resources on render thread. @@ -210,11 +226,8 @@ public class SurfaceViewRenderer extends SurfaceView GLES20.glDeleteTextures(3, yuvTextures, 0); yuvTextures = null; } - if (eglBase.hasSurface()) { - // Clear last rendered image to black. - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - eglBase.swapBuffers(); - } + // Clear last rendered image to black. + makeBlack(); eglBase.release(); eglBase = null; eglCleanupBarrier.countDown(); @@ -242,6 +255,14 @@ public class SurfaceViewRenderer extends SurfaceView frameRotation = 0; rendererEvents = null; } + resetStatistics(); + } + + /** + * Reset statistics. This will reset the logged statistics in logStatistics(), and + * RendererEvents.onFirstFrameRendered() will be called for the next frame. + */ + public void resetStatistics() { synchronized (statisticsLock) { framesReceived = 0; framesDropped = 0; @@ -277,27 +298,28 @@ public class SurfaceViewRenderer extends SurfaceView } synchronized (handlerLock) { if (renderThreadHandler == null) { - Logging.d(TAG, "Dropping frame - SurfaceViewRenderer not initialized or already released."); - } else { - synchronized (frameLock) { - if (pendingFrame == null) { - updateFrameDimensionsAndReportEvents(frame); - pendingFrame = frame; - renderThreadHandler.post(renderFrameRunnable); - return; + Logging.d(TAG, getResourceName() + + "Dropping frame - Not initialized or already released."); + VideoRenderer.renderFrameDone(frame); + return; + } + synchronized (frameLock) { + if (pendingFrame != null) { + // Drop old frame. + synchronized (statisticsLock) { + ++framesDropped; } + VideoRenderer.renderFrameDone(pendingFrame); } + pendingFrame = frame; + updateFrameDimensionsAndReportEvents(frame); + renderThreadHandler.post(renderFrameRunnable); } } - // Drop frame. - synchronized (statisticsLock) { - ++framesDropped; - } - VideoRenderer.renderFrameDone(frame); } // Returns desired layout size given current measure specification and video aspect ratio. - private Point getDesiredLayoutSize() { + private Point getDesiredLayoutSize(int widthSpec, int heightSpec) { synchronized (layoutLock) { final int maxWidth = getDefaultSize(Integer.MAX_VALUE, widthSpec); final int maxHeight = getDefaultSize(Integer.MAX_VALUE, heightSpec); @@ -317,18 +339,30 @@ public class SurfaceViewRenderer extends SurfaceView @Override protected void onMeasure(int widthSpec, int heightSpec) { synchronized (layoutLock) { - this.widthSpec = widthSpec; - this.heightSpec = heightSpec; - final Point size = getDesiredLayoutSize(); - setMeasuredDimension(size.x, size.y); + if (frameWidth == 0 || frameHeight == 0) { + super.onMeasure(widthSpec, heightSpec); + return; + } + desiredLayoutSize = getDesiredLayoutSize(widthSpec, heightSpec); + if (desiredLayoutSize.x != getMeasuredWidth() || desiredLayoutSize.y != getMeasuredHeight()) { + // Clear the surface asap before the layout change to avoid stretched video and other + // render artifacs. Don't wait for it to finish because the IO thread should never be + // blocked, so it's a best-effort attempt. + synchronized (handlerLock) { + if (renderThreadHandler != null) { + renderThreadHandler.postAtFrontOfQueue(makeBlackRunnable); + } + } + } + setMeasuredDimension(desiredLayoutSize.x, desiredLayoutSize.y); } } @Override protected void onLayout(boolean changed, int left, int top, int right, int bottom) { synchronized (layoutLock) { - layoutWidth = right - left; - layoutHeight = bottom - top; + layoutSize.x = right - left; + layoutSize.y = bottom - top; } // Might have a pending frame waiting for a layout of correct size. runOnRenderThread(renderFrameRunnable); @@ -337,7 +371,7 @@ public class SurfaceViewRenderer extends SurfaceView // SurfaceHolder.Callback interface. @Override public void surfaceCreated(final SurfaceHolder holder) { - Logging.d(TAG, "Surface created"); + Logging.d(TAG, getResourceName() + "Surface created."); synchronized (layoutLock) { isSurfaceCreated = true; } @@ -346,11 +380,11 @@ public class SurfaceViewRenderer extends SurfaceView @Override public void surfaceDestroyed(SurfaceHolder holder) { - Logging.d(TAG, "Surface destroyed"); + Logging.d(TAG, getResourceName() + "Surface destroyed."); synchronized (layoutLock) { isSurfaceCreated = false; - surfaceWidth = 0; - surfaceHeight = 0; + surfaceSize.x = 0; + surfaceSize.y = 0; } runOnRenderThread(new Runnable() { @Override public void run() { @@ -361,10 +395,10 @@ public class SurfaceViewRenderer extends SurfaceView @Override public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { - Logging.d(TAG, "Surface changed: " + width + "x" + height); + Logging.d(TAG, getResourceName() + "Surface changed: " + width + "x" + height); synchronized (layoutLock) { - surfaceWidth = width; - surfaceHeight = height; + surfaceSize.x = width; + surfaceSize.y = height; } // Might have a pending frame waiting for a surface of correct size. runOnRenderThread(renderFrameRunnable); @@ -381,26 +415,35 @@ public class SurfaceViewRenderer extends SurfaceView } } + private String getResourceName() { + try { + return getResources().getResourceEntryName(getId()) + ": "; + } catch (NotFoundException e) { + return ""; + } + } + + private void makeBlack() { + if (Thread.currentThread() != renderThread) { + throw new IllegalStateException(getResourceName() + "Wrong thread."); + } + if (eglBase != null && eglBase.hasSurface()) { + GLES20.glClearColor(0, 0, 0, 0); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + eglBase.swapBuffers(); + } + } + /** * Requests new layout if necessary. Returns true if layout and surface size are consistent. */ private boolean checkConsistentLayout() { + if (Thread.currentThread() != renderThread) { + throw new IllegalStateException(getResourceName() + "Wrong thread."); + } synchronized (layoutLock) { - final Point desiredLayoutSize = getDesiredLayoutSize(); - if (desiredLayoutSize.x != layoutWidth || desiredLayoutSize.y != layoutHeight) { - Logging.d(TAG, "Requesting new layout with size: " - + desiredLayoutSize.x + "x" + desiredLayoutSize.y); - // Request layout update on UI thread. - post(new Runnable() { - @Override public void run() { - requestLayout(); - } - }); - return false; - } - // Wait for requestLayout() to propagate through this sequence before returning true: - // requestLayout() -> onMeasure() -> onLayout() -> surfaceChanged(). - return surfaceWidth == layoutWidth && surfaceHeight == layoutHeight; + // Return false while we are in the middle of a layout change. + return layoutSize.equals(desiredLayoutSize) && surfaceSize.equals(layoutSize); } } @@ -408,61 +451,51 @@ public class SurfaceViewRenderer extends SurfaceView * Renders and releases |pendingFrame|. */ private void renderFrameOnRenderThread() { + if (Thread.currentThread() != renderThread) { + throw new IllegalStateException(getResourceName() + "Wrong thread."); + } + // Fetch and render |pendingFrame|. + final VideoRenderer.I420Frame frame; + synchronized (frameLock) { + if (pendingFrame == null) { + return; + } + frame = pendingFrame; + pendingFrame = null; + } if (eglBase == null || !eglBase.hasSurface()) { - Logging.d(TAG, "No surface to draw on"); + Logging.d(TAG, getResourceName() + "No surface to draw on"); + VideoRenderer.renderFrameDone(frame); return; } if (!checkConsistentLayout()) { // Output intermediate black frames while the layout is updated. - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - eglBase.swapBuffers(); + makeBlack(); + VideoRenderer.renderFrameDone(frame); return; } // After a surface size change, the EGLSurface might still have a buffer of the old size in the // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet // changed. Such a buffer will be rendered incorrectly, so flush it with a black frame. synchronized (layoutLock) { - if (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != surfaceHeight) { - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - eglBase.swapBuffers(); + if (eglBase.surfaceWidth() != surfaceSize.x || eglBase.surfaceHeight() != surfaceSize.y) { + makeBlack(); } } - // Fetch and render |pendingFrame|. - final VideoRenderer.I420Frame frame; - synchronized (frameLock) { - if (pendingFrame == null) { - return; - } - frame = pendingFrame; - pendingFrame = null; - } final long startTimeNs = System.nanoTime(); - final float[] samplingMatrix; - if (frame.yuvFrame) { - // The convention in WebRTC is that the first element in a ByteBuffer corresponds to the - // top-left corner of the image, but in glTexImage2D() the first element corresponds to the - // bottom-left corner. We correct this discrepancy by setting a vertical flip as sampling - // matrix. - samplingMatrix = RendererCommon.verticalFlipMatrix(); - } else { - // TODO(magjed): Move updateTexImage() to the video source instead. - SurfaceTexture surfaceTexture = (SurfaceTexture) frame.textureObject; - surfaceTexture.updateTexImage(); - samplingMatrix = new float[16]; - surfaceTexture.getTransformMatrix(samplingMatrix); - } - final float[] texMatrix; synchronized (layoutLock) { final float[] rotatedSamplingMatrix = - RendererCommon.rotateTextureMatrix(samplingMatrix, frame.rotationDegree); + RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree); final float[] layoutMatrix = RendererCommon.getLayoutMatrix( - mirror, frameAspectRatio(), (float) layoutWidth / layoutHeight); + mirror, frameAspectRatio(), (float) layoutSize.x / layoutSize.y); texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix); } - GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight); + // TODO(magjed): glClear() shouldn't be necessary since every pixel is covered anyway, but it's + // a workaround for bug 5147. Performance will be slightly worse. + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); if (frame.yuvFrame) { // Make sure YUV textures are allocated. if (yuvTextures == null) { @@ -471,11 +504,11 @@ public class SurfaceViewRenderer extends SurfaceView yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D); } } - drawer.uploadYuvData( + yuvUploader.uploadYuvData( yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes); - drawer.drawYuv(yuvTextures, texMatrix); + drawer.drawYuv(yuvTextures, texMatrix, 0, 0, surfaceSize.x, surfaceSize.y); } else { - drawer.drawOes(frame.textureId, texMatrix); + drawer.drawOes(frame.textureId, texMatrix, 0, 0, surfaceSize.x, surfaceSize.y); } eglBase.swapBuffers(); @@ -483,6 +516,12 @@ public class SurfaceViewRenderer extends SurfaceView synchronized (statisticsLock) { if (framesRendered == 0) { firstFrameTimeNs = startTimeNs; + synchronized (layoutLock) { + Logging.d(TAG, getResourceName() + "Reporting first rendered frame."); + if (rendererEvents != null) { + rendererEvents.onFirstFrameRendered(); + } + } } ++framesRendered; renderTimeNs += (System.nanoTime() - startTimeNs); @@ -508,32 +547,32 @@ public class SurfaceViewRenderer extends SurfaceView synchronized (layoutLock) { if (frameWidth != frame.width || frameHeight != frame.height || frameRotation != frame.rotationDegree) { + Logging.d(TAG, getResourceName() + "Reporting frame resolution changed to " + + frame.width + "x" + frame.height + " with rotation " + frame.rotationDegree); if (rendererEvents != null) { - final String id = getResources().getResourceEntryName(getId()); - if (frameWidth == 0 || frameHeight == 0) { - Logging.d(TAG, "ID: " + id + ". Reporting first rendered frame."); - rendererEvents.onFirstFrameRendered(); - } - Logging.d(TAG, "ID: " + id + ". Reporting frame resolution changed to " - + frame.width + "x" + frame.height + " with rotation " + frame.rotationDegree); rendererEvents.onFrameResolutionChanged(frame.width, frame.height, frame.rotationDegree); } frameWidth = frame.width; frameHeight = frame.height; frameRotation = frame.rotationDegree; + post(new Runnable() { + @Override public void run() { + requestLayout(); + } + }); } } } private void logStatistics() { synchronized (statisticsLock) { - Logging.d(TAG, "ID: " + getResources().getResourceEntryName(getId()) + ". Frames received: " + Logging.d(TAG, getResourceName() + "Frames received: " + framesReceived + ". Dropped: " + framesDropped + ". Rendered: " + framesRendered); if (framesReceived > 0 && framesRendered > 0) { final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs; - Logging.d(TAG, "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + - " ms. FPS: " + (float) framesRendered * 1e9 / timeSinceFirstFrameNs); - Logging.d(TAG, "Average render time: " + Logging.d(TAG, getResourceName() + "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + + " ms. FPS: " + framesRendered * 1e9 / timeSinceFirstFrameNs); + Logging.d(TAG, getResourceName() + "Average render time: " + (int) (renderTimeNs / (1000 * framesRendered)) + " us."); } } diff --git a/talk/app/webrtc/java/android/org/webrtc/ThreadUtils.java b/talk/app/webrtc/java/android/org/webrtc/ThreadUtils.java index 0d8968aba9..e60ead9f00 100644 --- a/talk/app/webrtc/java/android/org/webrtc/ThreadUtils.java +++ b/talk/app/webrtc/java/android/org/webrtc/ThreadUtils.java @@ -28,11 +28,13 @@ package org.webrtc; import android.os.Handler; +import android.os.SystemClock; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; -final class ThreadUtils { +public class ThreadUtils { /** * Utility class to be used for checking that a method is called on the correct thread. */ @@ -86,6 +88,29 @@ final class ThreadUtils { } } + public static boolean joinUninterruptibly(final Thread thread, long timeoutMs) { + final long startTimeMs = SystemClock.elapsedRealtime(); + long timeRemainingMs = timeoutMs; + boolean wasInterrupted = false; + while (timeRemainingMs > 0) { + try { + thread.join(timeRemainingMs); + break; + } catch (InterruptedException e) { + // Someone is asking us to return early at our convenience. We can't cancel this operation, + // but we should preserve the information and pass it along. + wasInterrupted = true; + final long elapsedTimeMs = SystemClock.elapsedRealtime() - startTimeMs; + timeRemainingMs = timeoutMs - elapsedTimeMs; + } + } + // Pass interruption information along. + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + return !thread.isAlive(); + } + public static void joinUninterruptibly(final Thread thread) { executeUninterruptibly(new BlockingOperation() { @Override @@ -104,6 +129,30 @@ final class ThreadUtils { }); } + public static boolean awaitUninterruptibly(CountDownLatch barrier, long timeoutMs) { + final long startTimeMs = SystemClock.elapsedRealtime(); + long timeRemainingMs = timeoutMs; + boolean wasInterrupted = false; + boolean result = false; + do { + try { + result = barrier.await(timeRemainingMs, TimeUnit.MILLISECONDS); + break; + } catch (InterruptedException e) { + // Someone is asking us to return early at our convenience. We can't cancel this operation, + // but we should preserve the information and pass it along. + wasInterrupted = true; + final long elapsedTimeMs = SystemClock.elapsedRealtime() - startTimeMs; + timeRemainingMs = timeoutMs - elapsedTimeMs; + } + } while (timeRemainingMs > 0); + // Pass interruption information along. + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + return result; + } + /** * Post |callable| to |handler| and wait for the result. */ diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java b/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java index 4caefc513d..36f60edd5c 100644 --- a/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java +++ b/talk/app/webrtc/java/android/org/webrtc/VideoCapturerAndroid.java @@ -28,9 +28,6 @@ package org.webrtc; import android.content.Context; -import android.graphics.SurfaceTexture; -import android.hardware.Camera; -import android.hardware.Camera.PreviewCallback; import android.os.Handler; import android.os.HandlerThread; import android.os.SystemClock; @@ -53,9 +50,6 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import javax.microedition.khronos.egl.EGLContext; -import javax.microedition.khronos.egl.EGL10; - // Android specific implementation of VideoCapturer. // An instance of this class can be created by an application using // VideoCapturerAndroid.create(); @@ -68,21 +62,22 @@ import javax.microedition.khronos.egl.EGL10; // camera thread. The internal *OnCameraThread() methods must check |camera| for null to check if // the camera has been stopped. @SuppressWarnings("deprecation") -public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallback, +public class VideoCapturerAndroid extends VideoCapturer implements + android.hardware.Camera.PreviewCallback, SurfaceTextureHelper.OnTextureFrameAvailableListener { private final static String TAG = "VideoCapturerAndroid"; private final static int CAMERA_OBSERVER_PERIOD_MS = 2000; + private final static int CAMERA_FREEZE_REPORT_TIMOUT_MS = 6000; - private Camera camera; // Only non-null while capturing. + private android.hardware.Camera camera; // Only non-null while capturing. private HandlerThread cameraThread; private final Handler cameraThreadHandler; private Context applicationContext; // Synchronization lock for |id|. private final Object cameraIdLock = new Object(); private int id; - private Camera.CameraInfo info; - private final FramePool videoBuffers; - private final CameraStatistics cameraStatistics = new CameraStatistics(); + private android.hardware.Camera.CameraInfo info; + private final CameraStatistics cameraStatistics; // Remember the requested format in case we want to switch cameras. private int requestedWidth; private int requestedHeight; @@ -94,17 +89,28 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private CapturerObserver frameObserver = null; private final CameraEventsHandler eventsHandler; private boolean firstFrameReported; + // Arbitrary queue depth. Higher number means more memory allocated & held, + // lower number means more sensitivity to processing time in the client (and + // potentially stalling the capturer if it runs out of buffers to write to). + private static final int NUMBER_OF_CAPTURE_BUFFERS = 3; + private final Set<byte[]> queuedBuffers = new HashSet<byte[]>(); private final boolean isCapturingToTexture; - private final SurfaceTextureHelper surfaceHelper; + final SurfaceTextureHelper surfaceHelper; // Package visible for testing purposes. // The camera API can output one old frame after the camera has been switched or the resolution // has been changed. This flag is used for dropping the first frame after camera restart. private boolean dropNextFrame = false; + // |openCameraOnCodecThreadRunner| is used for retrying to open the camera if it is in use by + // another application when startCaptureOnCameraThread is called. + private Runnable openCameraOnCodecThreadRunner; + private final static int MAX_OPEN_CAMERA_ATTEMPTS = 3; + private final static int OPEN_CAMERA_DELAY_MS = 500; + private int openCameraAttempts; // Camera error callback. - private final Camera.ErrorCallback cameraErrorCallback = - new Camera.ErrorCallback() { + private final android.hardware.Camera.ErrorCallback cameraErrorCallback = + new android.hardware.Camera.ErrorCallback() { @Override - public void onError(int error, Camera camera) { + public void onError(int error, android.hardware.Camera camera) { String errorMessage; if (error == android.hardware.Camera.CAMERA_ERROR_SERVER_DIED) { errorMessage = "Camera server died!"; @@ -120,47 +126,45 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Camera observer - monitors camera framerate. Observer is executed on camera thread. private final Runnable cameraObserver = new Runnable() { + private int freezePeriodCount; @Override public void run() { int cameraFramesCount = cameraStatistics.getAndResetFrameCount(); int cameraFps = (cameraFramesCount * 1000 + CAMERA_OBSERVER_PERIOD_MS / 2) / CAMERA_OBSERVER_PERIOD_MS; - Logging.d(TAG, "Camera fps: " + cameraFps + - ". Pending buffers: " + cameraStatistics.pendingFramesTimeStamps()); + Logging.d(TAG, "Camera fps: " + cameraFps +"."); if (cameraFramesCount == 0) { - Logging.e(TAG, "Camera freezed."); - if (eventsHandler != null) { - eventsHandler.onCameraError("Camera failure."); + ++freezePeriodCount; + if (CAMERA_OBSERVER_PERIOD_MS * freezePeriodCount > CAMERA_FREEZE_REPORT_TIMOUT_MS + && eventsHandler != null) { + Logging.e(TAG, "Camera freezed."); + if (surfaceHelper.isTextureInUse()) { + // This can only happen if we are capturing to textures. + eventsHandler.onCameraFreezed("Camera failure. Client must return video buffers."); + } else { + eventsHandler.onCameraFreezed("Camera failure."); + } + return; } } else { - cameraThreadHandler.postDelayed(this, CAMERA_OBSERVER_PERIOD_MS); + freezePeriodCount = 0; } + cameraThreadHandler.postDelayed(this, CAMERA_OBSERVER_PERIOD_MS); } }; private static class CameraStatistics { private int frameCount = 0; private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); - private final Set<Long> timeStampsNs = new HashSet<Long>(); CameraStatistics() { threadChecker.detachThread(); } - public void addPendingFrame(long timestamp) { + public void addFrame() { threadChecker.checkIsOnValidThread(); ++frameCount; - timeStampsNs.add(timestamp); - } - - public void frameReturned(long timestamp) { - threadChecker.checkIsOnValidThread(); - if (!timeStampsNs.contains(timestamp)) { - throw new IllegalStateException( - "CameraStatistics.frameReturned called with unknown timestamp " + timestamp); - } - timeStampsNs.remove(timestamp); } public int getAndResetFrameCount() { @@ -169,28 +173,16 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba frameCount = 0; return count; } - - // Return number of pending frames that have not been returned. - public int pendingFramesCount() { - threadChecker.checkIsOnValidThread(); - return timeStampsNs.size(); - } - - public String pendingFramesTimeStamps() { - threadChecker.checkIsOnValidThread(); - List<Long> timeStampsMs = new ArrayList<Long>(); - for (long ts : timeStampsNs) { - timeStampsMs.add(TimeUnit.NANOSECONDS.toMillis(ts)); - } - return timeStampsMs.toString(); - } } public static interface CameraEventsHandler { - // Camera error handler - invoked when camera stops receiving frames + // Camera error handler - invoked when camera can not be opened // or any camera exception happens on camera thread. void onCameraError(String errorDescription); + // Invoked when camera stops receiving frames + void onCameraFreezed(String errorDescription); + // Callback invoked when camera is opening. void onCameraOpening(int cameraId); @@ -216,7 +208,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } public static VideoCapturerAndroid create(String name, - CameraEventsHandler eventsHandler, EGLContext sharedEglContext) { + CameraEventsHandler eventsHandler, EglBase.Context sharedEglContext) { final int cameraId = lookupDeviceName(name); if (cameraId == -1) { return null; @@ -224,7 +216,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba final VideoCapturerAndroid capturer = new VideoCapturerAndroid(cameraId, eventsHandler, sharedEglContext); - capturer.setNativeCapturer(nativeCreateVideoCapturer(capturer)); + capturer.setNativeCapturer( + nativeCreateVideoCapturer(capturer, capturer.surfaceHelper)); return capturer; } @@ -243,7 +236,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Switch camera to the next valid camera id. This can only be called while // the camera is running. public void switchCamera(final CameraSwitchHandler handler) { - if (Camera.getNumberOfCameras() < 2) { + if (android.hardware.Camera.getNumberOfCameras() < 2) { if (handler != null) { handler.onCameraSwitchError("No camera to switch to."); } @@ -274,7 +267,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba pendingCameraSwitch = false; } if (handler != null) { - handler.onCameraSwitchDone(info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT); + handler.onCameraSwitchDone( + info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT); } } }); @@ -282,6 +276,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Requests a new output format from the video capturer. Captured frames // by the camera will be scaled/or dropped by the video capturer. + // It does not matter if width and height are flipped. I.E, |width| = 640, |height| = 480 produce + // the same result as |width| = 480, |height| = 640. // TODO(magjed/perkj): Document what this function does. Change name? public void onOutputFormatRequest(final int width, final int height, final int framerate) { cameraThreadHandler.post(new Runnable() { @@ -303,7 +299,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Helper function to retrieve the current camera id synchronously. Note that the camera id might // change at any point by switchCamera() calls. - private int getCurrentCameraId() { + int getCurrentCameraId() { synchronized (cameraIdLock) { return id; } @@ -329,20 +325,19 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } private VideoCapturerAndroid(int cameraId, CameraEventsHandler eventsHandler, - EGLContext sharedContext) { - Logging.d(TAG, "VideoCapturerAndroid"); + EglBase.Context sharedContext) { this.id = cameraId; this.eventsHandler = eventsHandler; cameraThread = new HandlerThread(TAG); cameraThread.start(); cameraThreadHandler = new Handler(cameraThread.getLooper()); - videoBuffers = new FramePool(cameraThread); isCapturingToTexture = (sharedContext != null); - surfaceHelper = SurfaceTextureHelper.create( - isCapturingToTexture ? sharedContext : EGL10.EGL_NO_CONTEXT, cameraThreadHandler); + cameraStatistics = new CameraStatistics(); + surfaceHelper = SurfaceTextureHelper.create(sharedContext, cameraThreadHandler); if (isCapturingToTexture) { surfaceHelper.setListener(this); } + Logging.d(TAG, "VideoCapturerAndroid isCapturingToTexture : " + isCapturingToTexture); } private void checkIsOnCameraThread() { @@ -355,13 +350,13 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // found. If |deviceName| is empty, the first available device is used. private static int lookupDeviceName(String deviceName) { Logging.d(TAG, "lookupDeviceName: " + deviceName); - if (deviceName == null || Camera.getNumberOfCameras() == 0) { + if (deviceName == null || android.hardware.Camera.getNumberOfCameras() == 0) { return -1; } if (deviceName.isEmpty()) { return 0; } - for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { + for (int i = 0; i < android.hardware.Camera.getNumberOfCameras(); ++i) { if (deviceName.equals(CameraEnumerationAndroid.getDeviceName(i))) { return i; } @@ -382,14 +377,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba if (camera != null) { throw new IllegalStateException("Release called while camera is running"); } - if (cameraStatistics.pendingFramesCount() != 0) { - throw new IllegalStateException("Release called with pending frames left"); - } } }); - surfaceHelper.disconnect(); - cameraThread.quit(); - ThreadUtils.joinUninterruptibly(cameraThread); + surfaceHelper.disconnect(cameraThreadHandler); cameraThread = null; } @@ -413,6 +403,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba if (frameObserver == null) { throw new RuntimeException("frameObserver not set."); } + cameraThreadHandler.post(new Runnable() { @Override public void run() { startCaptureOnCameraThread(width, height, framerate, frameObserver, @@ -422,8 +413,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } private void startCaptureOnCameraThread( - int width, int height, int framerate, CapturerObserver frameObserver, - Context applicationContext) { + final int width, final int height, final int framerate, final CapturerObserver frameObserver, + final Context applicationContext) { Throwable error = null; checkIsOnCameraThread(); if (camera != null) { @@ -431,17 +422,36 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } this.applicationContext = applicationContext; this.frameObserver = frameObserver; + this.firstFrameReported = false; + try { - synchronized (cameraIdLock) { - Logging.d(TAG, "Opening camera " + id); - firstFrameReported = false; - if (eventsHandler != null) { - eventsHandler.onCameraOpening(id); + try { + synchronized (cameraIdLock) { + Logging.d(TAG, "Opening camera " + id); + if (eventsHandler != null) { + eventsHandler.onCameraOpening(id); + } + camera = android.hardware.Camera.open(id); + info = new android.hardware.Camera.CameraInfo(); + android.hardware.Camera.getCameraInfo(id, info); + } + } catch (RuntimeException e) { + openCameraAttempts++; + if (openCameraAttempts < MAX_OPEN_CAMERA_ATTEMPTS) { + Logging.e(TAG, "Camera.open failed, retrying", e); + openCameraOnCodecThreadRunner = new Runnable() { + @Override public void run() { + startCaptureOnCameraThread(width, height, framerate, frameObserver, + applicationContext); + } + }; + cameraThreadHandler.postDelayed(openCameraOnCodecThreadRunner, OPEN_CAMERA_DELAY_MS); + return; } - camera = Camera.open(id); - info = new Camera.CameraInfo(); - Camera.getCameraInfo(id, info); + openCameraAttempts = 0; + throw e; } + try { camera.setPreviewTexture(surfaceHelper.getSurfaceTexture()); } catch (IOException e) { @@ -485,17 +495,18 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba requestedFramerate = framerate; // Find closest supported format for |width| x |height| @ |framerate|. - final Camera.Parameters parameters = camera.getParameters(); + final android.hardware.Camera.Parameters parameters = camera.getParameters(); final int[] range = CameraEnumerationAndroid.getFramerateRange(parameters, framerate * 1000); - final Camera.Size previewSize = CameraEnumerationAndroid.getClosestSupportedSize( - parameters.getSupportedPreviewSizes(), width, height); + final android.hardware.Camera.Size previewSize = + CameraEnumerationAndroid.getClosestSupportedSize( + parameters.getSupportedPreviewSizes(), width, height); final CaptureFormat captureFormat = new CaptureFormat( previewSize.width, previewSize.height, - range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], - range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); + range[android.hardware.Camera.Parameters.PREVIEW_FPS_MIN_INDEX], + range[android.hardware.Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); // Check if we are already using this capture format, then we don't need to do anything. - if (captureFormat.equals(this.captureFormat)) { + if (captureFormat.isSameFormat(this.captureFormat)) { return; } @@ -511,11 +522,15 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba parameters.setPreviewFpsRange(captureFormat.minFramerate, captureFormat.maxFramerate); } parameters.setPreviewSize(captureFormat.width, captureFormat.height); - parameters.setPreviewFormat(captureFormat.imageFormat); + + if (!isCapturingToTexture) { + parameters.setPreviewFormat(captureFormat.imageFormat); + } // Picture size is for taking pictures and not for preview/video, but we need to set it anyway // as a workaround for an aspect ratio problem on Nexus 7. - final Camera.Size pictureSize = CameraEnumerationAndroid.getClosestSupportedSize( - parameters.getSupportedPictureSizes(), width, height); + final android.hardware.Camera.Size pictureSize = + CameraEnumerationAndroid.getClosestSupportedSize( + parameters.getSupportedPictureSizes(), width, height); parameters.setPictureSize(pictureSize.width, pictureSize.height); // Temporarily stop preview if it's already running. @@ -532,13 +547,19 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba this.captureFormat = captureFormat; List<String> focusModes = parameters.getSupportedFocusModes(); - if (focusModes.contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { - parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); + if (focusModes.contains(android.hardware.Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { + parameters.setFocusMode(android.hardware.Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); } camera.setParameters(parameters); if (!isCapturingToTexture) { - videoBuffers.queueCameraBuffers(captureFormat.frameSize(), camera); + queuedBuffers.clear(); + final int frameSize = captureFormat.frameSize(); + for (int i = 0; i < NUMBER_OF_CAPTURE_BUFFERS; ++i) { + final ByteBuffer buffer = ByteBuffer.allocateDirect(frameSize); + queuedBuffers.add(buffer.array()); + camera.addCallbackBuffer(buffer.array()); + } camera.setPreviewCallbackWithBuffer(this); } camera.startPreview(); @@ -561,6 +582,10 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private void stopCaptureOnCameraThread() { checkIsOnCameraThread(); Logging.d(TAG, "stopCaptureOnCameraThread"); + if (openCameraOnCodecThreadRunner != null) { + cameraThreadHandler.removeCallbacks(openCameraOnCodecThreadRunner); + } + openCameraAttempts = 0; if (camera == null) { Logging.e(TAG, "Calling stopCapture() for already stopped camera."); return; @@ -571,13 +596,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba Logging.d(TAG, "Stop preview."); camera.stopPreview(); camera.setPreviewCallbackWithBuffer(null); - if (!isCapturingToTexture()) { - videoBuffers.stopReturnBuffersToCamera(); - Logging.d(TAG, "stopReturnBuffersToCamera called." - + (cameraStatistics.pendingFramesCount() == 0? - " All buffers have been returned." - : " Pending buffers: " + cameraStatistics.pendingFramesTimeStamps() + ".")); - } + queuedBuffers.clear(); captureFormat = null; Logging.d(TAG, "Release camera."); @@ -593,7 +612,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba Logging.d(TAG, "switchCameraOnCameraThread"); stopCaptureOnCameraThread(); synchronized (cameraIdLock) { - id = (id + 1) % Camera.getNumberOfCameras(); + id = (id + 1) % android.hardware.Camera.getNumberOfCameras(); } dropNextFrame = true; startCaptureOnCameraThread(requestedWidth, requestedHeight, requestedFramerate, frameObserver, @@ -612,17 +631,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba frameObserver.onOutputFormatRequest(width, height, framerate); } - public void returnBuffer(final long timeStamp) { - cameraThreadHandler.post(new Runnable() { - @Override public void run() { - cameraStatistics.frameReturned(timeStamp); - if (isCapturingToTexture) { - surfaceHelper.returnTextureFrame(); - } else { - videoBuffers.returnBuffer(timeStamp); - } - } - }); + // Exposed for testing purposes only. + Handler getCameraThreadHandler() { + return cameraThreadHandler; } private int getDeviceOrientation() { @@ -650,7 +661,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private int getFrameOrientation() { int rotation = getDeviceOrientation(); - if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { + if (info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_BACK) { rotation = 360 - rotation; } return (info.orientation + rotation) % 360; @@ -658,9 +669,10 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Called on cameraThread so must not "synchronized". @Override - public void onPreviewFrame(byte[] data, Camera callbackCamera) { + public void onPreviewFrame(byte[] data, android.hardware.Camera callbackCamera) { checkIsOnCameraThread(); - if (camera == null) { + if (camera == null || !queuedBuffers.contains(data)) { + // The camera has been stopped or |data| is an old invalid buffer. return; } if (camera != callbackCamera) { @@ -675,16 +687,10 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba firstFrameReported = true; } - // Mark the frame owning |data| as used. - // Note that since data is directBuffer, - // data.length >= videoBuffers.frameSize. - if (videoBuffers.reserveByteBuffer(data, captureTimeNs)) { - cameraStatistics.addPendingFrame(captureTimeNs); - frameObserver.onByteBufferFrameCaptured(data, videoBuffers.frameSize, captureFormat.width, - captureFormat.height, getFrameOrientation(), captureTimeNs); - } else { - Logging.w(TAG, "reserveByteBuffer failed - dropping frame."); - } + cameraStatistics.addFrame(); + frameObserver.onByteBufferFrameCaptured(data, captureFormat.width, captureFormat.height, + getFrameOrientation(), captureTimeNs); + camera.addCallbackBuffer(data); } @Override @@ -696,135 +702,22 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba surfaceHelper.returnTextureFrame(); return; } - if (!dropNextFrame) { + if (dropNextFrame) { surfaceHelper.returnTextureFrame(); - dropNextFrame = true; + dropNextFrame = false; return; } int rotation = getFrameOrientation(); - if (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { + if (info.facing == android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT) { // Undo the mirror that the OS "helps" us with. // http://developer.android.com/reference/android/hardware/Camera.html#setDisplayOrientation(int) transformMatrix = RendererCommon.multiplyMatrices(transformMatrix, RendererCommon.horizontalFlipMatrix()); } - transformMatrix = RendererCommon.rotateTextureMatrix(transformMatrix, rotation); - - final int rotatedWidth = (rotation % 180 == 0) ? captureFormat.width : captureFormat.height; - final int rotatedHeight = (rotation % 180 == 0) ? captureFormat.height : captureFormat.width; - cameraStatistics.addPendingFrame(timestampNs); - frameObserver.onTextureFrameCaptured(rotatedWidth, rotatedHeight, oesTextureId, - transformMatrix, timestampNs); - } - - // Class used for allocating and bookkeeping video frames. All buffers are - // direct allocated so that they can be directly used from native code. This class is - // not thread-safe, and enforces single thread use. - private static class FramePool { - // Thread that all calls should be made on. - private final Thread thread; - // Arbitrary queue depth. Higher number means more memory allocated & held, - // lower number means more sensitivity to processing time in the client (and - // potentially stalling the capturer if it runs out of buffers to write to). - private static final int numCaptureBuffers = 3; - // This container tracks the buffers added as camera callback buffers. It is needed for finding - // the corresponding ByteBuffer given a byte[]. - private final Map<byte[], ByteBuffer> queuedBuffers = new IdentityHashMap<byte[], ByteBuffer>(); - // This container tracks the frames that have been sent but not returned. It is needed for - // keeping the buffers alive and for finding the corresponding ByteBuffer given a timestamp. - private final Map<Long, ByteBuffer> pendingBuffers = new HashMap<Long, ByteBuffer>(); - private int frameSize = 0; - private Camera camera; - - public FramePool(Thread thread) { - this.thread = thread; - } - - private void checkIsOnValidThread() { - if (Thread.currentThread() != thread) { - throw new IllegalStateException("Wrong thread"); - } - } - - // Discards previous queued buffers and adds new callback buffers to camera. - public void queueCameraBuffers(int frameSize, Camera camera) { - checkIsOnValidThread(); - this.camera = camera; - this.frameSize = frameSize; - - queuedBuffers.clear(); - for (int i = 0; i < numCaptureBuffers; ++i) { - final ByteBuffer buffer = ByteBuffer.allocateDirect(frameSize); - camera.addCallbackBuffer(buffer.array()); - queuedBuffers.put(buffer.array(), buffer); - } - Logging.d(TAG, "queueCameraBuffers enqueued " + numCaptureBuffers - + " buffers of size " + frameSize + "."); - } - - public void stopReturnBuffersToCamera() { - checkIsOnValidThread(); - this.camera = null; - queuedBuffers.clear(); - // Frames in |pendingBuffers| need to be kept alive until they are returned. - } - - public boolean reserveByteBuffer(byte[] data, long timeStamp) { - checkIsOnValidThread(); - final ByteBuffer buffer = queuedBuffers.remove(data); - if (buffer == null) { - // Frames might be posted to |onPreviewFrame| with the previous format while changing - // capture format in |startPreviewOnCameraThread|. Drop these old frames. - Logging.w(TAG, "Received callback buffer from previous configuration with length: " - + (data == null ? "null" : data.length)); - return false; - } - if (buffer.capacity() != frameSize) { - throw new IllegalStateException("Callback buffer has unexpected frame size"); - } - if (pendingBuffers.containsKey(timeStamp)) { - Logging.e(TAG, "Timestamp already present in pending buffers - they need to be unique"); - return false; - } - pendingBuffers.put(timeStamp, buffer); - if (queuedBuffers.isEmpty()) { - Logging.d(TAG, "Camera is running out of capture buffers."); - } - return true; - } - - public void returnBuffer(long timeStamp) { - checkIsOnValidThread(); - final ByteBuffer returnedFrame = pendingBuffers.remove(timeStamp); - if (returnedFrame == null) { - throw new RuntimeException("unknown data buffer with time stamp " - + timeStamp + "returned?!?"); - } - - if (camera != null && returnedFrame.capacity() == frameSize) { - camera.addCallbackBuffer(returnedFrame.array()); - if (queuedBuffers.isEmpty()) { - Logging.d(TAG, "Frame returned when camera is running out of capture" - + " buffers for TS " + TimeUnit.NANOSECONDS.toMillis(timeStamp)); - } - queuedBuffers.put(returnedFrame.array(), returnedFrame); - return; - } - - if (returnedFrame.capacity() != frameSize) { - Logging.d(TAG, "returnBuffer with time stamp " - + TimeUnit.NANOSECONDS.toMillis(timeStamp) - + " called with old frame size, " + returnedFrame.capacity() + "."); - // Since this frame has the wrong size, don't requeue it. Frames with the correct size are - // created in queueCameraBuffers so this must be an old buffer. - return; - } - - Logging.d(TAG, "returnBuffer with time stamp " - + TimeUnit.NANOSECONDS.toMillis(timeStamp) - + " called after camera has been stopped."); - } + cameraStatistics.addFrame(); + frameObserver.onTextureFrameCaptured(captureFormat.width, captureFormat.height, oesTextureId, + transformMatrix, rotation, timestampNs); } // Interface used for providing callbacks to an observer. @@ -835,13 +728,14 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba // Delivers a captured frame. Called on a Java thread owned by // VideoCapturerAndroid. - abstract void onByteBufferFrameCaptured(byte[] data, int length, int width, int height, - int rotation, long timeStamp); + abstract void onByteBufferFrameCaptured(byte[] data, int width, int height, int rotation, + long timeStamp); // Delivers a captured frame in a texture with id |oesTextureId|. Called on a Java thread // owned by VideoCapturerAndroid. abstract void onTextureFrameCaptured( - int width, int height, int oesTextureId, float[] transformMatrix, long timestamp); + int width, int height, int oesTextureId, float[] transformMatrix, int rotation, + long timestamp); // Requests an output format from the video capturer. Captured frames // by the camera will be scaled/or dropped by the video capturer. @@ -864,17 +758,18 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba } @Override - public void onByteBufferFrameCaptured(byte[] data, int length, int width, int height, + public void onByteBufferFrameCaptured(byte[] data, int width, int height, int rotation, long timeStamp) { - nativeOnByteBufferFrameCaptured(nativeCapturer, data, length, width, height, rotation, + nativeOnByteBufferFrameCaptured(nativeCapturer, data, data.length, width, height, rotation, timeStamp); } @Override public void onTextureFrameCaptured( - int width, int height, int oesTextureId, float[] transformMatrix, long timestamp) { + int width, int height, int oesTextureId, float[] transformMatrix, int rotation, + long timestamp) { nativeOnTextureFrameCaptured(nativeCapturer, width, height, oesTextureId, transformMatrix, - timestamp); + rotation, timestamp); } @Override @@ -887,10 +782,12 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba private native void nativeOnByteBufferFrameCaptured(long nativeCapturer, byte[] data, int length, int width, int height, int rotation, long timeStamp); private native void nativeOnTextureFrameCaptured(long nativeCapturer, int width, int height, - int oesTextureId, float[] transformMatrix, long timestamp); + int oesTextureId, float[] transformMatrix, int rotation, long timestamp); private native void nativeOnOutputFormatRequest(long nativeCapturer, int width, int height, int framerate); } - private static native long nativeCreateVideoCapturer(VideoCapturerAndroid videoCapturer); + private static native long nativeCreateVideoCapturer( + VideoCapturerAndroid videoCapturer, + SurfaceTextureHelper surfaceHelper); } diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java index bacd0cf11f..bb6f01cea2 100644 --- a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java +++ b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java @@ -38,7 +38,7 @@ import javax.microedition.khronos.opengles.GL10; import android.annotation.SuppressLint; import android.graphics.Point; import android.graphics.Rect; -import android.graphics.SurfaceTexture; +import android.opengl.EGL14; import android.opengl.GLES20; import android.opengl.GLSurfaceView; @@ -59,7 +59,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private static Runnable eglContextReady = null; private static final String TAG = "VideoRendererGui"; private GLSurfaceView surface; - private static EGLContext eglContext = null; + private static EglBase.Context eglContext = null; // Indicates if SurfaceView.Renderer.onSurfaceCreated was called. // If true then for every newly created yuv image renderer createTexture() // should be called. The variable is accessed on multiple threads and @@ -69,8 +69,6 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private int screenHeight; // List of yuv renderers. private final ArrayList<YuvImageRenderer> yuvImageRenderers; - // |drawer| is synchronized on |yuvImageRenderers|. - private GlRectDrawer drawer; // Render and draw threads. private static Thread renderFrameThread; private static Thread drawThread; @@ -99,6 +97,8 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { // currently leaking resources to avoid a rare crash in release() where the EGLContext has // become invalid beforehand. private int[] yuvTextures = { 0, 0, 0 }; + private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader(); + private final RendererCommon.GlDrawer drawer; // Resources for making a deep copy of incoming OES texture frame. private GlTextureFrameBuffer textureCopy; @@ -157,12 +157,13 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private YuvImageRenderer( GLSurfaceView surface, int id, int x, int y, int width, int height, - RendererCommon.ScalingType scalingType, boolean mirror) { + RendererCommon.ScalingType scalingType, boolean mirror, RendererCommon.GlDrawer drawer) { Logging.d(TAG, "YuvImageRenderer.Create id: " + id); this.surface = surface; this.id = id; this.scalingType = scalingType; this.mirror = mirror; + this.drawer = drawer; layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height)); updateLayoutProperties = false; rotationDegree = 0; @@ -174,6 +175,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private synchronized void release() { surface = null; + drawer.release(); synchronized (pendingFrameLock) { if (pendingFrame != null) { VideoRenderer.renderFrameDone(pendingFrame); @@ -226,7 +228,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { } } - private void draw(GlRectDrawer drawer) { + private void draw() { if (!seenFrame) { // No frame received yet - nothing to render. return; @@ -241,29 +243,15 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { } if (isNewFrame) { + rotatedSamplingMatrix = RendererCommon.rotateTextureMatrix( + pendingFrame.samplingMatrix, pendingFrame.rotationDegree); if (pendingFrame.yuvFrame) { rendererType = RendererType.RENDERER_YUV; - drawer.uploadYuvData(yuvTextures, pendingFrame.width, pendingFrame.height, + yuvUploader.uploadYuvData(yuvTextures, pendingFrame.width, pendingFrame.height, pendingFrame.yuvStrides, pendingFrame.yuvPlanes); - // The convention in WebRTC is that the first element in a ByteBuffer corresponds to the - // top-left corner of the image, but in glTexImage2D() the first element corresponds to - // the bottom-left corner. We correct this discrepancy by setting a vertical flip as - // sampling matrix. - final float[] samplingMatrix = RendererCommon.verticalFlipMatrix(); - rotatedSamplingMatrix = - RendererCommon.rotateTextureMatrix(samplingMatrix, pendingFrame.rotationDegree); } else { rendererType = RendererType.RENDERER_TEXTURE; - // External texture rendering. Update texture image to latest and make a deep copy of - // the external texture. - // TODO(magjed): Move updateTexImage() to the video source instead. - final SurfaceTexture surfaceTexture = (SurfaceTexture) pendingFrame.textureObject; - surfaceTexture.updateTexImage(); - final float[] samplingMatrix = new float[16]; - surfaceTexture.getTransformMatrix(samplingMatrix); - rotatedSamplingMatrix = - RendererCommon.rotateTextureMatrix(samplingMatrix, pendingFrame.rotationDegree); - + // External texture rendering. Make a deep copy of the external texture. // Reallocate offscreen texture if necessary. textureCopy.setSize(pendingFrame.rotatedWidth(), pendingFrame.rotatedHeight()); @@ -272,12 +260,13 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { GlUtil.checkNoGLES2Error("glBindFramebuffer"); // Copy the OES texture content. This will also normalize the sampling matrix. - GLES20.glViewport(0, 0, textureCopy.getWidth(), textureCopy.getHeight()); - drawer.drawOes(pendingFrame.textureId, rotatedSamplingMatrix); + drawer.drawOes(pendingFrame.textureId, rotatedSamplingMatrix, + 0, 0, textureCopy.getWidth(), textureCopy.getHeight()); rotatedSamplingMatrix = RendererCommon.identityMatrix(); // Restore normal framebuffer. GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + GLES20.glFinish(); } copyTimeNs += (System.nanoTime() - now); VideoRenderer.renderFrameDone(pendingFrame); @@ -285,17 +274,17 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { } } - // OpenGL defaults to lower left origin - flip vertically. - GLES20.glViewport(displayLayout.left, screenHeight - displayLayout.bottom, - displayLayout.width(), displayLayout.height()); - updateLayoutMatrix(); final float[] texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix); + // OpenGL defaults to lower left origin - flip viewport position vertically. + final int viewportY = screenHeight - displayLayout.bottom; if (rendererType == RendererType.RENDERER_YUV) { - drawer.drawYuv(yuvTextures, texMatrix); + drawer.drawYuv(yuvTextures, texMatrix, + displayLayout.left, viewportY, displayLayout.width(), displayLayout.height()); } else { - drawer.drawRgb(textureCopy.getTextureId(), texMatrix); + drawer.drawRgb(textureCopy.getTextureId(), texMatrix, + displayLayout.left, viewportY, displayLayout.width(), displayLayout.height()); } if (isNewFrame) { @@ -314,7 +303,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { ". Dropped: " + framesDropped + ". Rendered: " + framesRendered); if (framesReceived > 0 && framesRendered > 0) { Logging.d(TAG, "Duration: " + (int)(timeSinceFirstFrameNs / 1e6) + - " ms. FPS: " + (float)framesRendered * 1e9 / timeSinceFirstFrameNs); + " ms. FPS: " + framesRendered * 1e9 / timeSinceFirstFrameNs); Logging.d(TAG, "Draw time: " + (int) (drawTimeNs / (1000 * framesRendered)) + " us. Copy time: " + (int) (copyTimeNs / (1000 * framesReceived)) + " us"); @@ -429,7 +418,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { eglContextReady = eglContextReadyCallback; } - public static synchronized EGLContext getEGLContext() { + public static synchronized EglBase.Context getEglBaseContext() { return eglContext; } @@ -477,6 +466,16 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { */ public static synchronized YuvImageRenderer create(int x, int y, int width, int height, RendererCommon.ScalingType scalingType, boolean mirror) { + return create(x, y, width, height, scalingType, mirror, new GlRectDrawer()); + } + + /** + * Creates VideoRenderer.Callbacks with top left corner at (x, y) and resolution (width, height). + * All parameters are in percentage of screen resolution. The custom |drawer| will be used for + * drawing frames on the EGLSurface. This class is responsible for calling release() on |drawer|. + */ + public static synchronized YuvImageRenderer create(int x, int y, int width, int height, + RendererCommon.ScalingType scalingType, boolean mirror, RendererCommon.GlDrawer drawer) { // Check display region parameters. if (x < 0 || x > 100 || y < 0 || y > 100 || width < 0 || width > 100 || height < 0 || height > 100 || @@ -490,7 +489,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { } final YuvImageRenderer yuvImageRenderer = new YuvImageRenderer( instance.surface, instance.yuvImageRenderers.size(), - x, y, width, height, scalingType, mirror); + x, y, width, height, scalingType, mirror, drawer); synchronized (instance.yuvImageRenderers) { if (instance.onSurfaceCreatedCalled) { // onSurfaceCreated has already been called for VideoRendererGui - @@ -498,6 +497,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { // rendering list. final CountDownLatch countDownLatch = new CountDownLatch(1); instance.surface.queueEvent(new Runnable() { + @Override public void run() { yuvImageRenderer.createTextures(); yuvImageRenderer.setScreenSize( @@ -608,13 +608,16 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { Logging.d(TAG, "VideoRendererGui.onSurfaceCreated"); // Store render EGL context. synchronized (VideoRendererGui.class) { - eglContext = ((EGL10) EGLContext.getEGL()).eglGetCurrentContext(); + if (EglBase14.isEGL14Supported()) { + eglContext = new EglBase14.Context(EGL14.eglGetCurrentContext()); + } else { + eglContext = new EglBase10.Context(((EGL10) EGLContext.getEGL()).eglGetCurrentContext()); + } + Logging.d(TAG, "VideoRendererGui EGL Context: " + eglContext); } synchronized (yuvImageRenderers) { - // Create drawer for YUV/OES frames. - drawer = new GlRectDrawer(); // Create textures for all images. for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) { yuvImageRenderer.createTextures(); @@ -655,7 +658,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); synchronized (yuvImageRenderers) { for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) { - yuvImageRenderer.draw(drawer); + yuvImageRenderer.draw(); } } } diff --git a/talk/app/webrtc/java/jni/androidmediacodeccommon.h b/talk/app/webrtc/java/jni/androidmediacodeccommon.h index 348a716496..92ea135f12 100644 --- a/talk/app/webrtc/java/jni/androidmediacodeccommon.h +++ b/talk/app/webrtc/java/jni/androidmediacodeccommon.h @@ -72,6 +72,8 @@ enum { kMediaCodecTimeoutMs = 1000 }; enum { kMediaCodecStatisticsIntervalMs = 3000 }; // Maximum amount of pending frames for VP8 decoder. enum { kMaxPendingFramesVp8 = 1 }; +// Maximum amount of pending frames for VP9 decoder. +enum { kMaxPendingFramesVp9 = 1 }; // Maximum amount of pending frames for H.264 decoder. enum { kMaxPendingFramesH264 = 30 }; // Maximum amount of decoded frames for which per-frame logging is enabled. diff --git a/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc b/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc index b664f16e2e..c3d287ce0d 100644 --- a/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc +++ b/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc @@ -33,14 +33,15 @@ #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" #include "talk/app/webrtc/java/jni/classreferenceholder.h" #include "talk/app/webrtc/java/jni/native_handle_impl.h" +#include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/scoped_ref_ptr.h" #include "webrtc/base/thread.h" #include "webrtc/base/timeutils.h" -#include "webrtc/common_video/interface/i420_buffer_pool.h" -#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" +#include "webrtc/common_video/include/i420_buffer_pool.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/system_wrappers/include/logcat_trace_context.h" #include "webrtc/system_wrappers/include/tick_util.h" #include "third_party/libyuv/include/libyuv/convert.h" @@ -62,6 +63,7 @@ using webrtc::VideoCodec; using webrtc::VideoCodecType; using webrtc::kVideoCodecH264; using webrtc::kVideoCodecVP8; +using webrtc::kVideoCodecVP9; namespace webrtc_jni { @@ -87,9 +89,14 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, int32_t Release() override; int32_t Reset() override; + + bool PrefersLateDecoding() const override { return true; } + // rtc::MessageHandler implementation. void OnMessage(rtc::Message* msg) override; + const char* ImplementationName() const override; + private: // CHECK-fail if not running on |codec_thread_|. void CheckOnCodecThread(); @@ -105,13 +112,17 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, // Type of video codec. VideoCodecType codecType_; + // Render EGL context - owned by factory, should not be allocated/destroyed + // by VideoDecoder. + jobject render_egl_context_; + bool key_frame_required_; bool inited_; bool sw_fallback_required_; bool use_surface_; VideoCodec codec_; webrtc::I420BufferPool decoded_frame_pool_; - NativeHandleImpl native_handle_; + rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_; DecodedImageCallback* callback_; int frames_received_; // Number of frames received by decoder. int frames_decoded_; // Number of frames decoded by decoder. @@ -120,10 +131,6 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, int current_bytes_; // Encoded bytes in the current statistics interval. int current_decoding_time_ms_; // Overall decoding time in the current second uint32_t max_pending_frames_; // Maximum number of pending input frames - std::vector<int32_t> timestamps_; - std::vector<int64_t> ntp_times_ms_; - std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to - // decoder input. // State that is constant for the lifetime of this object once the ctor // returns. @@ -134,7 +141,8 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, jmethodID j_release_method_; jmethodID j_dequeue_input_buffer_method_; jmethodID j_queue_input_buffer_method_; - jmethodID j_dequeue_output_buffer_method_; + jmethodID j_dequeue_byte_buffer_method_; + jmethodID j_dequeue_texture_buffer_method_; jmethodID j_return_decoded_byte_buffer_method_; // MediaCodecVideoDecoder fields. jfieldID j_input_buffers_field_; @@ -144,24 +152,23 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder, jfieldID j_height_field_; jfieldID j_stride_field_; jfieldID j_slice_height_field_; - jfieldID j_surface_texture_field_; // MediaCodecVideoDecoder.DecodedTextureBuffer fields. - jfieldID j_textureID_field_; - jfieldID j_texture_presentation_timestamp_us_field_; - // MediaCodecVideoDecoder.DecodedByteBuffer fields. + jfieldID j_texture_id_field_; + jfieldID j_transform_matrix_field_; + jfieldID j_texture_timestamp_ms_field_; + jfieldID j_texture_ntp_timestamp_ms_field_; + jfieldID j_texture_decode_time_ms_field_; + jfieldID j_texture_frame_delay_ms_field_; + // MediaCodecVideoDecoder.DecodedOutputBuffer fields. jfieldID j_info_index_field_; jfieldID j_info_offset_field_; jfieldID j_info_size_field_; - jfieldID j_info_presentation_timestamp_us_field_; + jfieldID j_info_timestamp_ms_field_; + jfieldID j_info_ntp_timestamp_ms_field_; + jfieldID j_byte_buffer_decode_time_ms_field_; // Global references; must be deleted in Release(). std::vector<jobject> input_buffers_; - jobject surface_texture_; - jobject previous_surface_texture_; - - // Render EGL context - owned by factory, should not be allocated/destroyed - // by VideoDecoder. - jobject render_egl_context_; }; MediaCodecVideoDecoder::MediaCodecVideoDecoder( @@ -171,8 +178,6 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( key_frame_required_(true), inited_(false), sw_fallback_required_(false), - surface_texture_(NULL), - previous_surface_texture_(NULL), codec_thread_(new Thread()), j_media_codec_video_decoder_class_( jni, @@ -191,19 +196,22 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( j_init_decode_method_ = GetMethodID( jni, *j_media_codec_video_decoder_class_, "initDecode", "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;" - "IILjavax/microedition/khronos/egl/EGLContext;)Z"); + "IILorg/webrtc/SurfaceTextureHelper;)Z"); j_release_method_ = GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V"); j_dequeue_input_buffer_method_ = GetMethodID( jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I"); j_queue_input_buffer_method_ = GetMethodID( - jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJ)Z"); - j_dequeue_output_buffer_method_ = GetMethodID( + jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJJJ)Z"); + j_dequeue_byte_buffer_method_ = GetMethodID( jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer", - "(I)Ljava/lang/Object;"); + "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer;"); + j_dequeue_texture_buffer_method_ = GetMethodID( + jni, *j_media_codec_video_decoder_class_, "dequeueTextureBuffer", + "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer;"); j_return_decoded_byte_buffer_method_ = GetMethodID(jni, *j_media_codec_video_decoder_class_, - "returnDecodedByteBuffer", "(I)V"); + "returnDecodedOutputBuffer", "(I)V"); j_input_buffers_field_ = GetFieldID( jni, *j_media_codec_video_decoder_class_, @@ -221,28 +229,36 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( jni, *j_media_codec_video_decoder_class_, "stride", "I"); j_slice_height_field_ = GetFieldID( jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I"); - j_surface_texture_field_ = GetFieldID( - jni, *j_media_codec_video_decoder_class_, "surfaceTexture", - "Landroid/graphics/SurfaceTexture;"); - jclass j_decoder_decoded_texture_buffer_class = FindClass(jni, + jclass j_decoded_texture_buffer_class = FindClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); - j_textureID_field_ = GetFieldID( - jni, j_decoder_decoded_texture_buffer_class, "textureID", "I"); - j_texture_presentation_timestamp_us_field_ = - GetFieldID(jni, j_decoder_decoded_texture_buffer_class, - "presentationTimestampUs", "J"); - - jclass j_decoder_decoded_byte_buffer_class = FindClass(jni, - "org/webrtc/MediaCodecVideoDecoder$DecodedByteBuffer"); + j_texture_id_field_ = GetFieldID( + jni, j_decoded_texture_buffer_class, "textureID", "I"); + j_transform_matrix_field_ = GetFieldID( + jni, j_decoded_texture_buffer_class, "transformMatrix", "[F"); + j_texture_timestamp_ms_field_ = GetFieldID( + jni, j_decoded_texture_buffer_class, "timeStampMs", "J"); + j_texture_ntp_timestamp_ms_field_ = GetFieldID( + jni, j_decoded_texture_buffer_class, "ntpTimeStampMs", "J"); + j_texture_decode_time_ms_field_ = GetFieldID( + jni, j_decoded_texture_buffer_class, "decodeTimeMs", "J"); + j_texture_frame_delay_ms_field_ = GetFieldID( + jni, j_decoded_texture_buffer_class, "frameDelayMs", "J"); + + jclass j_decoded_output_buffer_class = FindClass(jni, + "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer"); j_info_index_field_ = GetFieldID( - jni, j_decoder_decoded_byte_buffer_class, "index", "I"); + jni, j_decoded_output_buffer_class, "index", "I"); j_info_offset_field_ = GetFieldID( - jni, j_decoder_decoded_byte_buffer_class, "offset", "I"); + jni, j_decoded_output_buffer_class, "offset", "I"); j_info_size_field_ = GetFieldID( - jni, j_decoder_decoded_byte_buffer_class, "size", "I"); - j_info_presentation_timestamp_us_field_ = GetFieldID( - jni, j_decoder_decoded_byte_buffer_class, "presentationTimestampUs", "J"); + jni, j_decoded_output_buffer_class, "size", "I"); + j_info_timestamp_ms_field_ = GetFieldID( + jni, j_decoded_output_buffer_class, "timeStampMs", "J"); + j_info_ntp_timestamp_ms_field_ = GetFieldID( + jni, j_decoded_output_buffer_class, "ntpTimeStampMs", "J"); + j_byte_buffer_decode_time_ms_field_ = GetFieldID( + jni, j_decoded_output_buffer_class, "decodeTimeMs", "J"); CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed"; use_surface_ = (render_egl_context_ != NULL); @@ -254,14 +270,6 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( MediaCodecVideoDecoder::~MediaCodecVideoDecoder() { // Call Release() to ensure no more callbacks to us after we are deleted. Release(); - // Delete global references. - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - if (previous_surface_texture_ != NULL) { - jni->DeleteGlobalRef(previous_surface_texture_); - } - if (surface_texture_ != NULL) { - jni->DeleteGlobalRef(surface_texture_); - } } int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, @@ -312,6 +320,21 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { frames_received_ = 0; frames_decoded_ = 0; + jobject java_surface_texture_helper_ = nullptr; + if (use_surface_) { + java_surface_texture_helper_ = jni->CallStaticObjectMethod( + FindClass(jni, "org/webrtc/SurfaceTextureHelper"), + GetStaticMethodID(jni, + FindClass(jni, "org/webrtc/SurfaceTextureHelper"), + "create", + "(Lorg/webrtc/EglBase$Context;)" + "Lorg/webrtc/SurfaceTextureHelper;"), + render_egl_context_); + RTC_CHECK(java_surface_texture_helper_ != nullptr); + surface_texture_helper_ = new rtc::RefCountedObject<SurfaceTextureHelper>( + jni, java_surface_texture_helper_); + } + jobject j_video_codec_enum = JavaEnumFromIndex( jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_); bool success = jni->CallBooleanMethod( @@ -320,7 +343,7 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { j_video_codec_enum, codec_.width, codec_.height, - use_surface_ ? render_egl_context_ : nullptr); + java_surface_texture_helper_); if (CheckException(jni) || !success) { ALOGE << "Codec initialization error - fallback to SW codec."; sw_fallback_required_ = true; @@ -332,6 +355,9 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { case kVideoCodecVP8: max_pending_frames_ = kMaxPendingFramesVp8; break; + case kVideoCodecVP9: + max_pending_frames_ = kMaxPendingFramesVp9; + break; case kVideoCodecH264: max_pending_frames_ = kMaxPendingFramesH264; break; @@ -342,9 +368,6 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { current_frames_ = 0; current_bytes_ = 0; current_decoding_time_ms_ = 0; - timestamps_.clear(); - ntp_times_ms_.clear(); - frame_rtc_times_ms_.clear(); jobjectArray input_buffers = (jobjectArray)GetObjectField( jni, *j_media_codec_video_decoder_, j_input_buffers_field_); @@ -361,15 +384,6 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { } } - if (use_surface_) { - jobject surface_texture = GetObjectField( - jni, *j_media_codec_video_decoder_, j_surface_texture_field_); - if (previous_surface_texture_ != NULL) { - jni->DeleteGlobalRef(previous_surface_texture_); - } - previous_surface_texture_ = surface_texture_; - surface_texture_ = jni->NewGlobalRef(surface_texture); - } codec_thread_->PostDelayed(kMediaCodecPollMs, this); return WEBRTC_VIDEO_CODEC_OK; @@ -395,6 +409,7 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { } input_buffers_.clear(); jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_); + surface_texture_helper_ = nullptr; inited_ = false; rtc::MessageQueueManager::Clear(this); if (CheckException(jni)) { @@ -501,19 +516,21 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( // Try to drain the decoder and wait until output is not too // much behind the input. - if (frames_received_ > frames_decoded_ + max_pending_frames_) { + const int64 drain_start = GetCurrentTimeMs(); + while ((frames_received_ > frames_decoded_ + max_pending_frames_) && + (GetCurrentTimeMs() - drain_start) < kMediaCodecTimeoutMs) { ALOGV("Received: %d. Decoded: %d. Wait for output...", frames_received_, frames_decoded_); - if (!DeliverPendingOutputs(jni, kMediaCodecTimeoutMs * 1000)) { + if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) { ALOGE << "DeliverPendingOutputs error. Frames received: " << frames_received_ << ". Frames decoded: " << frames_decoded_; return ProcessHWErrorOnCodecThread(); } - if (frames_received_ > frames_decoded_ + max_pending_frames_) { - ALOGE << "Output buffer dequeue timeout. Frames received: " << - frames_received_ << ". Frames decoded: " << frames_decoded_; - return ProcessHWErrorOnCodecThread(); - } + } + if (frames_received_ > frames_decoded_ + max_pending_frames_) { + ALOGE << "Output buffer dequeue timeout. Frames received: " << + frames_received_ << ". Frames decoded: " << frames_decoded_; + return ProcessHWErrorOnCodecThread(); } // Get input buffer. @@ -535,11 +552,14 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( " is bigger than buffer size " << buffer_capacity; return ProcessHWErrorOnCodecThread(); } - jlong timestamp_us = (frames_received_ * 1000000) / codec_.maxFramerate; + jlong presentation_timestamp_us = + (frames_received_ * 1000000) / codec_.maxFramerate; if (frames_decoded_ < kMaxDecodedLogFrames) { ALOGD << "Decoder frame in # " << frames_received_ << ". Type: " << inputImage._frameType << ". Buffer # " << - j_input_buffer_index << ". TS: " << (int)(timestamp_us / 1000) + j_input_buffer_index << ". pTS: " + << (int)(presentation_timestamp_us / 1000) + << ". TS: " << inputImage._timeStamp << ". Size: " << inputImage._length; } memcpy(buffer, inputImage._buffer, inputImage._length); @@ -547,16 +567,16 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( // Save input image timestamps for later output. frames_received_++; current_bytes_ += inputImage._length; - timestamps_.push_back(inputImage._timeStamp); - ntp_times_ms_.push_back(inputImage.ntp_time_ms_); - frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); // Feed input to decoder. - bool success = jni->CallBooleanMethod(*j_media_codec_video_decoder_, - j_queue_input_buffer_method_, - j_input_buffer_index, - inputImage._length, - timestamp_us); + bool success = jni->CallBooleanMethod( + *j_media_codec_video_decoder_, + j_queue_input_buffer_method_, + j_input_buffer_index, + inputImage._length, + presentation_timestamp_us, + static_cast<int64_t> (inputImage._timeStamp), + inputImage.ntp_time_ms_); if (CheckException(jni) || !success) { ALOGE << "queueInputBuffer error"; return ProcessHWErrorOnCodecThread(); @@ -572,16 +592,18 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( } bool MediaCodecVideoDecoder::DeliverPendingOutputs( - JNIEnv* jni, int dequeue_timeout_us) { + JNIEnv* jni, int dequeue_timeout_ms) { if (frames_received_ <= frames_decoded_) { // No need to query for output buffers - decoder is drained. return true; } // Get decoder output. - jobject j_decoder_output_buffer = jni->CallObjectMethod( - *j_media_codec_video_decoder_, - j_dequeue_output_buffer_method_, - dequeue_timeout_us); + jobject j_decoder_output_buffer = + jni->CallObjectMethod(*j_media_codec_video_decoder_, + use_surface_ ? j_dequeue_texture_buffer_method_ + : j_dequeue_byte_buffer_method_, + dequeue_timeout_ms); + if (CheckException(jni)) { ALOGE << "dequeueOutputBuffer() error"; return false; @@ -601,19 +623,35 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( j_slice_height_field_); rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer; - long output_timestamps_ms = 0; + int64_t output_timestamps_ms = 0; + int64_t output_ntp_timestamps_ms = 0; + int decode_time_ms = 0; + int64_t frame_delayed_ms = 0; if (use_surface_) { // Extract data from Java DecodedTextureBuffer. const int texture_id = - GetIntField(jni, j_decoder_output_buffer, j_textureID_field_); - const int64_t timestamp_us = - GetLongField(jni, j_decoder_output_buffer, - j_texture_presentation_timestamp_us_field_); - output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec; - // Create webrtc::VideoFrameBuffer with native texture handle. - native_handle_.SetTextureObject(surface_texture_, texture_id); - frame_buffer = new rtc::RefCountedObject<JniNativeHandleBuffer>( - &native_handle_, width, height); + GetIntField(jni, j_decoder_output_buffer, j_texture_id_field_); + if (texture_id != 0) { // |texture_id| == 0 represents a dropped frame. + const jfloatArray j_transform_matrix = + reinterpret_cast<jfloatArray>(GetObjectField( + jni, j_decoder_output_buffer, j_transform_matrix_field_)); + const int64_t timestamp_us = + GetLongField(jni, j_decoder_output_buffer, + j_texture_timestamp_ms_field_); + output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer, + j_texture_timestamp_ms_field_); + output_ntp_timestamps_ms = + GetLongField(jni, j_decoder_output_buffer, + j_texture_ntp_timestamp_ms_field_); + decode_time_ms = GetLongField(jni, j_decoder_output_buffer, + j_texture_decode_time_ms_field_); + frame_delayed_ms = GetLongField(jni, j_decoder_output_buffer, + j_texture_frame_delay_ms_field_); + + // Create webrtc::VideoFrameBuffer with native texture handle. + frame_buffer = surface_texture_helper_->CreateTextureFrame( + width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix)); + } } else { // Extract data from Java ByteBuffer and create output yuv420 frame - // for non surface decoding only. @@ -623,9 +661,14 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_); const int output_buffer_size = GetIntField(jni, j_decoder_output_buffer, j_info_size_field_); - const int64_t timestamp_us = GetLongField( - jni, j_decoder_output_buffer, j_info_presentation_timestamp_us_field_); - output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec; + output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer, + j_info_timestamp_ms_field_); + output_ntp_timestamps_ms = + GetLongField(jni, j_decoder_output_buffer, + j_info_ntp_timestamp_ms_field_); + + decode_time_ms = GetLongField(jni, j_decoder_output_buffer, + j_byte_buffer_decode_time_ms_field_); if (output_buffer_size < width * height * 3 / 2) { ALOGE << "Insufficient output buffer size: " << output_buffer_size; @@ -683,41 +726,31 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( j_return_decoded_byte_buffer_method_, output_buffer_index); if (CheckException(jni)) { - ALOGE << "returnDecodedByteBuffer error"; + ALOGE << "returnDecodedOutputBuffer error"; return false; } } VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0); + decoded_frame.set_timestamp(output_timestamps_ms); + decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms); - // Get frame timestamps from a queue. - if (timestamps_.size() > 0) { - decoded_frame.set_timestamp(timestamps_.front()); - timestamps_.erase(timestamps_.begin()); - } - if (ntp_times_ms_.size() > 0) { - decoded_frame.set_ntp_time_ms(ntp_times_ms_.front()); - ntp_times_ms_.erase(ntp_times_ms_.begin()); - } - int64_t frame_decoding_time_ms = 0; - if (frame_rtc_times_ms_.size() > 0) { - frame_decoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front(); - frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin()); - } if (frames_decoded_ < kMaxDecodedLogFrames) { ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width << " x " << height << ". " << stride << " x " << slice_height << - ". Color: " << color_format << ". TS:" << (int)output_timestamps_ms << - ". DecTime: " << (int)frame_decoding_time_ms; + ". Color: " << color_format << ". TS:" << decoded_frame.timestamp() << + ". DecTime: " << (int)decode_time_ms << + ". DelayTime: " << (int)frame_delayed_ms; } // Calculate and print decoding statistics - every 3 seconds. frames_decoded_++; current_frames_++; - current_decoding_time_ms_ += frame_decoding_time_ms; + current_decoding_time_ms_ += decode_time_ms; int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_; if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && current_frames_ > 0) { - ALOGD << "Decoded frames: " << frames_decoded_ << ". Bitrate: " << + ALOGD << "Decoded frames: " << frames_decoded_ << ". Received frames: " + << frames_received_ << ". Bitrate: " << (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " << ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms) << ". decTime: " << (current_decoding_time_ms_ / current_frames_) << @@ -728,12 +761,15 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( current_decoding_time_ms_ = 0; } - // Callback - output decoded frame. - const int32_t callback_status = callback_->Decoded(decoded_frame); - if (callback_status > 0) { - ALOGE << "callback error"; + // |.IsZeroSize())| returns true when a frame has been dropped. + if (!decoded_frame.IsZeroSize()) { + // Callback - output decoded frame. + const int32_t callback_status = + callback_->Decoded(decoded_frame, decode_time_ms); + if (callback_status > 0) { + ALOGE << "callback error"; + } } - return true; } @@ -790,6 +826,17 @@ MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory() : supported_codec_types_.push_back(kVideoCodecVP8); } + bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( + j_decoder_class, + GetStaticMethodID(jni, j_decoder_class, "isVp9HwSupported", "()Z")); + if (CheckException(jni)) { + is_vp9_hw_supported = false; + } + if (is_vp9_hw_supported) { + ALOGD << "VP9 HW Decoder supported."; + supported_codec_types_.push_back(kVideoCodecVP9); + } + bool is_h264_hw_supported = jni->CallStaticBooleanMethod( j_decoder_class, GetStaticMethodID(jni, j_decoder_class, "isH264HwSupported", "()Z")); @@ -825,7 +872,7 @@ void MediaCodecVideoDecoderFactory::SetEGLContext( render_egl_context_ = NULL; } else { jclass j_egl_context_class = - FindClass(jni, "javax/microedition/khronos/egl/EGLContext"); + FindClass(jni, "org/webrtc/EglBase$Context"); if (!jni->IsInstanceOf(render_egl_context_, j_egl_context_class)) { ALOGE << "Wrong EGL Context."; jni->DeleteGlobalRef(render_egl_context_); @@ -841,7 +888,7 @@ void MediaCodecVideoDecoderFactory::SetEGLContext( webrtc::VideoDecoder* MediaCodecVideoDecoderFactory::CreateVideoDecoder( VideoCodecType type) { if (supported_codec_types_.empty()) { - ALOGE << "No HW video decoder for type " << (int)type; + ALOGW << "No HW video decoder for type " << (int)type; return NULL; } for (VideoCodecType codec_type : supported_codec_types_) { @@ -851,7 +898,7 @@ webrtc::VideoDecoder* MediaCodecVideoDecoderFactory::CreateVideoDecoder( AttachCurrentThreadIfNeeded(), type, render_egl_context_); } } - ALOGE << "Can not find HW video decoder for type " << (int)type; + ALOGW << "Can not find HW video decoder for type " << (int)type; return NULL; } @@ -861,5 +908,9 @@ void MediaCodecVideoDecoderFactory::DestroyVideoDecoder( delete decoder; } +const char* MediaCodecVideoDecoder::ImplementationName() const { + return "MediaCodec"; +} + } // namespace webrtc_jni diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc index ac349e7faf..64831c3174 100644 --- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc +++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc @@ -29,14 +29,16 @@ #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h" #include "talk/app/webrtc/java/jni/classreferenceholder.h" #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" +#include "talk/app/webrtc/java/jni/native_handle_impl.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/thread.h" +#include "webrtc/base/thread_checker.h" #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h" -#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" -#include "webrtc/modules/video_coding/utility/include/quality_scaler.h" -#include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h" +#include "webrtc/modules/video_coding/include/video_codec_interface.h" +#include "webrtc/modules/video_coding/utility/quality_scaler.h" +#include "webrtc/modules/video_coding/utility/vp8_header_parser.h" #include "webrtc/system_wrappers/include/field_trial.h" #include "webrtc/system_wrappers/include/logcat_trace_context.h" #include "third_party/libyuv/include/libyuv/convert.h" @@ -56,6 +58,7 @@ using webrtc::VideoCodec; using webrtc::VideoCodecType; using webrtc::kVideoCodecH264; using webrtc::kVideoCodecVP8; +using webrtc::kVideoCodecVP9; namespace webrtc_jni { @@ -79,7 +82,9 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, public rtc::MessageHandler { public: virtual ~MediaCodecVideoEncoder(); - explicit MediaCodecVideoEncoder(JNIEnv* jni, VideoCodecType codecType); + MediaCodecVideoEncoder(JNIEnv* jni, + VideoCodecType codecType, + jobject egl_context); // webrtc::VideoEncoder implementation. Everything trampolines to // |codec_thread_| for execution. @@ -103,13 +108,18 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, int GetTargetFramerate() override; + bool SupportsNativeHandle() const override { return true; } + const char* ImplementationName() const override; + private: // CHECK-fail if not running on |codec_thread_|. void CheckOnCodecThread(); - // Release() and InitEncode() in an attempt to restore the codec to an + private: + // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and + // InitEncodeOnCodecThread() in an attempt to restore the codec to an // operable state. Necessary after all manner of OMX-layer errors. - void ResetCodec(); + bool ResetCodecOnCodecThread(); // Implementation of webrtc::VideoEncoder methods above, all running on the // codec thread exclusively. @@ -117,10 +127,20 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // If width==0 then this is assumed to be a re-initialization and the // previously-current values are reused instead of the passed parameters // (makes it easier to reason about thread-safety). - int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps); + int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps, + bool use_surface); + // Reconfigure to match |frame| in width, height. Also reconfigures the + // encoder if |frame| is a texture/byte buffer and the encoder is initialized + // for byte buffer/texture. Returns false if reconfiguring fails. + bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame); int32_t EncodeOnCodecThread( const webrtc::VideoFrame& input_image, const std::vector<webrtc::FrameType>* frame_types); + bool EncodeByteBufferOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index); + bool EncodeTextureOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame); + int32_t RegisterEncodeCompleteCallbackOnCodecThread( webrtc::EncodedImageCallback* callback); int32_t ReleaseOnCodecThread(); @@ -150,11 +170,14 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // State that is constant for the lifetime of this object once the ctor // returns. scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. + rtc::ThreadChecker codec_thread_checker_; ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_; ScopedGlobalRef<jobject> j_media_codec_video_encoder_; jmethodID j_init_encode_method_; + jmethodID j_get_input_buffers_method_; jmethodID j_dequeue_input_buffer_method_; - jmethodID j_encode_method_; + jmethodID j_encode_buffer_method_; + jmethodID j_encode_texture_method_; jmethodID j_release_method_; jmethodID j_set_rates_method_; jmethodID j_dequeue_output_buffer_method_; @@ -170,6 +193,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, int width_; // Frame width in pixels. int height_; // Frame height in pixels. bool inited_; + bool use_surface_; uint16_t picture_id_; enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. int last_set_bitrate_kbps_; // Last-requested bitrate in kbps. @@ -205,6 +229,16 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // H264 bitstream parser, used to extract QP from encoded bitstreams. webrtc::H264BitstreamParser h264_bitstream_parser_; + + // VP9 variables to populate codec specific structure. + webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for + // non-flexible VP9 mode. + uint8_t tl0_pic_idx_; + size_t gof_idx_; + + // EGL context - owned by factory, should not be allocated/destroyed + // by MediaCodecVideoEncoder. + jobject egl_context_; }; MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { @@ -213,11 +247,9 @@ MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { } MediaCodecVideoEncoder::MediaCodecVideoEncoder( - JNIEnv* jni, VideoCodecType codecType) : + JNIEnv* jni, VideoCodecType codecType, jobject egl_context) : codecType_(codecType), callback_(NULL), - inited_(false), - picture_id_(0), codec_thread_(new Thread()), j_media_codec_video_encoder_class_( jni, @@ -228,7 +260,11 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder( GetMethodID(jni, *j_media_codec_video_encoder_class_, "<init>", - "()V"))) { + "()V"))), + inited_(false), + use_surface_(false), + picture_id_(0), + egl_context_(egl_context) { ScopedLocalRefFrame local_ref_frame(jni); // It would be nice to avoid spinning up a new thread per MediaCodec, and // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug @@ -239,19 +275,27 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder( // thread. codec_thread_->SetName("MediaCodecVideoEncoder", NULL); RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder"; - + codec_thread_checker_.DetachFromThread(); jclass j_output_buffer_info_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); j_init_encode_method_ = GetMethodID( jni, *j_media_codec_video_encoder_class_, "initEncode", - "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;IIII)" - "[Ljava/nio/ByteBuffer;"); + "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;" + "IIIILorg/webrtc/EglBase14$Context;)Z"); + j_get_input_buffers_method_ = GetMethodID( + jni, + *j_media_codec_video_encoder_class_, + "getInputBuffers", + "()[Ljava/nio/ByteBuffer;"); j_dequeue_input_buffer_method_ = GetMethodID( jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I"); - j_encode_method_ = GetMethodID( - jni, *j_media_codec_video_encoder_class_, "encode", "(ZIIJ)Z"); + j_encode_buffer_method_ = GetMethodID( + jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z"); + j_encode_texture_method_ = GetMethodID( + jni, *j_media_codec_video_encoder_class_, "encodeTexture", + "(ZI[FJ)Z"); j_release_method_ = GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V"); j_set_rates_method_ = GetMethodID( @@ -275,6 +319,7 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder( j_info_presentation_timestamp_us_field_ = GetFieldID( jni, j_output_buffer_info_class, "presentationTimestampUs", "J"); CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed"; + srand(time(NULL)); AllowBlockingCalls(); } @@ -295,8 +340,8 @@ int32_t MediaCodecVideoEncoder::InitEncode( << codecType_; ALOGD << "InitEncode request"; - scale_ = webrtc::field_trial::FindFullName( - "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"; + scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName( + "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"); ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); if (scale_) { if (codecType_ == kVideoCodecVP8) { @@ -331,7 +376,8 @@ int32_t MediaCodecVideoEncoder::InitEncode( codec_settings->width, codec_settings->height, codec_settings->startBitrate, - codec_settings->maxFramerate)); + codec_settings->maxFramerate, + false /* use_surface */)); } int32_t MediaCodecVideoEncoder::Encode( @@ -374,6 +420,7 @@ int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, } void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); @@ -381,7 +428,6 @@ void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { // functor), so expect no ID/data. RTC_CHECK(!msg->message_id) << "Unexpected message!"; RTC_CHECK(!msg->pdata) << "Unexpected message!"; - CheckOnCodecThread(); if (!inited_) { return; } @@ -393,26 +439,24 @@ void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { codec_thread_->PostDelayed(kMediaCodecPollMs, this); } -void MediaCodecVideoEncoder::CheckOnCodecThread() { - RTC_CHECK(codec_thread_ == ThreadManager::Instance()->CurrentThread()) - << "Running on wrong thread!"; -} - -void MediaCodecVideoEncoder::ResetCodec() { - ALOGE << "ResetCodec"; - if (Release() != WEBRTC_VIDEO_CODEC_OK || - codec_thread_->Invoke<int32_t>(Bind( - &MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, - width_, height_, 0, 0)) != WEBRTC_VIDEO_CODEC_OK) { +bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + ALOGE << "ResetOnCodecThread"; + if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK || + InitEncodeOnCodecThread(width_, height_, 0, 0, false) != + WEBRTC_VIDEO_CODEC_OK) { // TODO(fischman): wouldn't it be nice if there was a way to gracefully // degrade to a SW encoder at this point? There isn't one AFAICT :( // https://code.google.com/p/webrtc/issues/detail?id=2920 + return false; } + return true; } int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( - int width, int height, int kbps, int fps) { - CheckOnCodecThread(); + int width, int height, int kbps, int fps, bool use_surface) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set."; JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); @@ -448,52 +492,63 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( render_times_ms_.clear(); frame_rtc_times_ms_.clear(); drop_next_input_frame_ = false; + use_surface_ = use_surface; picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; + gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); + tl0_pic_idx_ = static_cast<uint8_t>(rand()); + gof_idx_ = 0; + // We enforce no extra stride/padding in the format creation step. jobject j_video_codec_enum = JavaEnumFromIndex( jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_); - jobjectArray input_buffers = reinterpret_cast<jobjectArray>( - jni->CallObjectMethod(*j_media_codec_video_encoder_, - j_init_encode_method_, - j_video_codec_enum, - width_, - height_, - kbps, - fps)); - CHECK_EXCEPTION(jni); - if (IsNull(jni, input_buffers)) { + const bool encode_status = jni->CallBooleanMethod( + *j_media_codec_video_encoder_, j_init_encode_method_, + j_video_codec_enum, width, height, kbps, fps, + (use_surface ? egl_context_ : nullptr)); + if (!encode_status) { + ALOGE << "Failed to configure encoder."; return WEBRTC_VIDEO_CODEC_ERROR; } + CHECK_EXCEPTION(jni); - inited_ = true; - switch (GetIntField(jni, *j_media_codec_video_encoder_, - j_color_format_field_)) { - case COLOR_FormatYUV420Planar: - encoder_fourcc_ = libyuv::FOURCC_YU12; - break; - case COLOR_FormatYUV420SemiPlanar: - case COLOR_QCOM_FormatYUV420SemiPlanar: - case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: - encoder_fourcc_ = libyuv::FOURCC_NV12; - break; - default: - LOG(LS_ERROR) << "Wrong color format."; - return WEBRTC_VIDEO_CODEC_ERROR; - } - size_t num_input_buffers = jni->GetArrayLength(input_buffers); - RTC_CHECK(input_buffers_.empty()) - << "Unexpected double InitEncode without Release"; - input_buffers_.resize(num_input_buffers); - for (size_t i = 0; i < num_input_buffers; ++i) { - input_buffers_[i] = - jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); - int64_t yuv_buffer_capacity = - jni->GetDirectBufferCapacity(input_buffers_[i]); + if (!use_surface) { + jobjectArray input_buffers = reinterpret_cast<jobjectArray>( + jni->CallObjectMethod(*j_media_codec_video_encoder_, + j_get_input_buffers_method_)); CHECK_EXCEPTION(jni); - RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; + if (IsNull(jni, input_buffers)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + switch (GetIntField(jni, *j_media_codec_video_encoder_, + j_color_format_field_)) { + case COLOR_FormatYUV420Planar: + encoder_fourcc_ = libyuv::FOURCC_YU12; + break; + case COLOR_FormatYUV420SemiPlanar: + case COLOR_QCOM_FormatYUV420SemiPlanar: + case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: + encoder_fourcc_ = libyuv::FOURCC_NV12; + break; + default: + LOG(LS_ERROR) << "Wrong color format."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + size_t num_input_buffers = jni->GetArrayLength(input_buffers); + RTC_CHECK(input_buffers_.empty()) + << "Unexpected double InitEncode without Release"; + input_buffers_.resize(num_input_buffers); + for (size_t i = 0; i < num_input_buffers; ++i) { + input_buffers_[i] = + jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); + int64_t yuv_buffer_capacity = + jni->GetDirectBufferCapacity(input_buffers_[i]); + CHECK_EXCEPTION(jni); + RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; + } } - CHECK_EXCEPTION(jni); + inited_ = true; codec_thread_->PostDelayed(kMediaCodecPollMs, this); return WEBRTC_VIDEO_CODEC_OK; } @@ -501,40 +556,53 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( const webrtc::VideoFrame& frame, const std::vector<webrtc::FrameType>* frame_types) { - CheckOnCodecThread(); + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); if (!inited_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } + frames_received_++; if (!DeliverPendingOutputs(jni)) { - ResetCodec(); - // Continue as if everything's fine. + if (!ResetCodecOnCodecThread()) + return WEBRTC_VIDEO_CODEC_ERROR; } if (drop_next_input_frame_) { - ALOGV("Encoder drop frame - failed callback."); + ALOGW << "Encoder drop frame - failed callback."; drop_next_input_frame_ = false; return WEBRTC_VIDEO_CODEC_OK; } RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; - // Check framerate before spatial resolution change. - if (scale_) - quality_scaler_.OnEncodeFrame(frame); - const VideoFrame& input_frame = - scale_ ? quality_scaler_.GetScaledFrame(frame) : frame; + VideoFrame input_frame = frame; + if (scale_) { + // Check framerate before spatial resolution change. + quality_scaler_.OnEncodeFrame(frame); + const webrtc::QualityScaler::Resolution scaled_resolution = + quality_scaler_.GetScaledResolution(); + if (scaled_resolution.width != frame.width() || + scaled_resolution.height != frame.height()) { + if (frame.native_handle() != nullptr) { + rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer( + static_cast<AndroidTextureBuffer*>( + frame.video_frame_buffer().get())->ScaleAndRotate( + scaled_resolution.width, + scaled_resolution.height, + webrtc::kVideoRotation_0)); + input_frame.set_video_frame_buffer(scaled_buffer); + } else { + input_frame = quality_scaler_.GetScaledFrame(frame); + } + } + } - if (input_frame.width() != width_ || input_frame.height() != height_) { - ALOGD << "Frame resolution change from " << width_ << " x " << height_ << - " to " << input_frame.width() << " x " << input_frame.height(); - width_ = input_frame.width(); - height_ = input_frame.height(); - ResetCodec(); - return WEBRTC_VIDEO_CODEC_OK; + if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) { + ALOGE << "Failed to reconfigure encoder."; + return WEBRTC_VIDEO_CODEC_ERROR; } // Check if we accumulated too many frames in encoder input buffers @@ -552,65 +620,138 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( } } - int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, - j_dequeue_input_buffer_method_); - CHECK_EXCEPTION(jni); - if (j_input_buffer_index == -1) { - // Video codec falls behind - no input buffer available. - ALOGV("Encoder drop frame - no input buffers available"); - frames_dropped_++; - // Report dropped frame to quality_scaler_. - OnDroppedFrame(); - return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. - } - if (j_input_buffer_index == -2) { - ResetCodec(); + const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta; + bool encode_status = true; + if (!input_frame.native_handle()) { + int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, + j_dequeue_input_buffer_method_); + CHECK_EXCEPTION(jni); + if (j_input_buffer_index == -1) { + // Video codec falls behind - no input buffer available. + ALOGW << "Encoder drop frame - no input buffers available"; + frames_dropped_++; + // Report dropped frame to quality_scaler_. + OnDroppedFrame(); + return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. + } + if (j_input_buffer_index == -2) { + ResetCodecOnCodecThread(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame, + j_input_buffer_index); + } else { + encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame); + } + + if (!encode_status) { + ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); + ResetCodecOnCodecThread(); return WEBRTC_VIDEO_CODEC_ERROR; } + last_input_timestamp_ms_ = + current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; + frames_in_queue_++; + + // Save input image timestamps for later output + timestamps_.push_back(input_frame.timestamp()); + render_times_ms_.push_back(input_frame.render_time_ms()); + frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); + current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; + + if (!DeliverPendingOutputs(jni)) { + ALOGE << "Failed deliver pending outputs."; + ResetCodecOnCodecThread(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread( + const webrtc::VideoFrame& frame) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + + const bool is_texture_frame = frame.native_handle() != nullptr; + const bool reconfigure_due_to_format = is_texture_frame != use_surface_; + const bool reconfigure_due_to_size = + frame.width() != width_ || frame.height() != height_; + + if (reconfigure_due_to_format) { + ALOGD << "Reconfigure encoder due to format change. " + << (use_surface_ ? + "Reconfiguring to encode from byte buffer." : + "Reconfiguring to encode from texture."); + } + if (reconfigure_due_to_size) { + ALOGD << "Reconfigure encoder due to frame resolution change from " + << width_ << " x " << height_ << " to " << frame.width() << " x " + << frame.height(); + width_ = frame.width(); + height_ = frame.height(); + } + + if (!reconfigure_due_to_format && !reconfigure_due_to_size) + return true; + + ReleaseOnCodecThread(); + + return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) == + WEBRTC_VIDEO_CODEC_OK; +} + +bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + RTC_CHECK(!use_surface_); + ALOGV("Encoder frame in # %d. TS: %lld. Q: %d", frames_received_ - 1, current_timestamp_us_ / 1000, frames_in_queue_); - jobject j_input_buffer = input_buffers_[j_input_buffer_index]; + jobject j_input_buffer = input_buffers_[input_buffer_index]; uint8_t* yuv_buffer = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); CHECK_EXCEPTION(jni); RTC_CHECK(yuv_buffer) << "Indirect buffer??"; RTC_CHECK(!libyuv::ConvertFromI420( - input_frame.buffer(webrtc::kYPlane), input_frame.stride(webrtc::kYPlane), - input_frame.buffer(webrtc::kUPlane), input_frame.stride(webrtc::kUPlane), - input_frame.buffer(webrtc::kVPlane), input_frame.stride(webrtc::kVPlane), + frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), + frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), + frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), yuv_buffer, width_, width_, height_, encoder_fourcc_)) << "ConvertFromI420 failed"; - last_input_timestamp_ms_ = current_timestamp_us_ / 1000; - frames_in_queue_++; - // Save input image timestamps for later output - timestamps_.push_back(input_frame.timestamp()); - render_times_ms_.push_back(input_frame.render_time_ms()); - frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); - - bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta; bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, - j_encode_method_, + j_encode_buffer_method_, key_frame, - j_input_buffer_index, + input_buffer_index, yuv_size_, current_timestamp_us_); CHECK_EXCEPTION(jni); - current_timestamp_us_ += 1000000 / last_set_fps_; + return encode_status; +} - if (!encode_status || !DeliverPendingOutputs(jni)) { - ResetCodec(); - return WEBRTC_VIDEO_CODEC_ERROR; - } +bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni, + bool key_frame, const webrtc::VideoFrame& frame) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); + RTC_CHECK(use_surface_); + NativeHandleImpl* handle = + static_cast<NativeHandleImpl*>(frame.native_handle()); + jfloatArray sampling_matrix = jni->NewFloatArray(16); + jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); - return WEBRTC_VIDEO_CODEC_OK; + bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, + j_encode_texture_method_, + key_frame, + handle->oes_texture_id, + sampling_matrix, + current_timestamp_us_); + CHECK_EXCEPTION(jni); + return encode_status; } int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( webrtc::EncodedImageCallback* callback) { - CheckOnCodecThread(); + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); callback_ = callback; @@ -618,10 +759,10 @@ int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( } int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); if (!inited_) { return WEBRTC_VIDEO_CODEC_OK; } - CheckOnCodecThread(); JNIEnv* jni = AttachCurrentThreadIfNeeded(); ALOGD << "EncoderReleaseOnCodecThread: Frames received: " << frames_received_ << ". Encoded: " << frames_encoded_ << @@ -634,13 +775,14 @@ int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { CHECK_EXCEPTION(jni); rtc::MessageQueueManager::Clear(this); inited_ = false; + use_surface_ = false; ALOGD << "EncoderReleaseOnCodecThread done."; return WEBRTC_VIDEO_CODEC_OK; } int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate) { - CheckOnCodecThread(); + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); if (last_set_bitrate_kbps_ == new_bit_rate && last_set_fps_ == frame_rate) { return WEBRTC_VIDEO_CODEC_OK; @@ -659,7 +801,7 @@ int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, last_set_fps_); CHECK_EXCEPTION(jni); if (!ret) { - ResetCodec(); + ResetCodecOnCodecThread(); return WEBRTC_VIDEO_CODEC_ERROR; } return WEBRTC_VIDEO_CODEC_OK; @@ -691,6 +833,7 @@ jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs( } bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { + RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); while (true) { jobject j_output_buffer_info = jni->CallObjectMethod( *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_); @@ -702,7 +845,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { int output_buffer_index = GetOutputBufferInfoIndex(jni, j_output_buffer_info); if (output_buffer_index == -1) { - ResetCodec(); + ResetCodecOnCodecThread(); return false; } @@ -786,19 +929,42 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { info.codecSpecific.VP8.layerSync = false; info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx; info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; - picture_id_ = (picture_id_ + 1) & 0x7FFF; + } else if (codecType_ == kVideoCodecVP9) { + if (key_frame) { + gof_idx_ = 0; + } + info.codecSpecific.VP9.picture_id = picture_id_; + info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; + info.codecSpecific.VP9.flexible_mode = false; + info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; + info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++; + info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx; + info.codecSpecific.VP9.temporal_up_switch = true; + info.codecSpecific.VP9.inter_layer_predicted = false; + info.codecSpecific.VP9.gof_idx = + static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); + info.codecSpecific.VP9.num_spatial_layers = 1; + info.codecSpecific.VP9.spatial_layer_resolution_present = false; + if (info.codecSpecific.VP9.ss_data_available) { + info.codecSpecific.VP9.spatial_layer_resolution_present = true; + info.codecSpecific.VP9.width[0] = width_; + info.codecSpecific.VP9.height[0] = height_; + info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); + } } + picture_id_ = (picture_id_ + 1) & 0x7FFF; // Generate a header describing a single fragment. webrtc::RTPFragmentationHeader header; memset(&header, 0, sizeof(header)); - if (codecType_ == kVideoCodecVP8) { + if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) { header.VerifyAndAllocateFragmentationHeader(1); header.fragmentationOffset[0] = 0; header.fragmentationLength[0] = image->_length; header.fragmentationPlType[0] = 0; header.fragmentationTimeDiff[0] = 0; - if (scale_) { + if (codecType_ == kVideoCodecVP8 && scale_) { int qp; if (webrtc::vp8::GetQp(payload, payload_size, &qp)) quality_scaler_.ReportQP(qp); @@ -829,7 +995,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1] << " " << image->_buffer[2] << " " << image->_buffer[3] << " " << image->_buffer[4] << " " << image->_buffer[5]; - ResetCodec(); + ResetCodecOnCodecThread(); return false; } scPositions[scPositionsLength] = payload_size; @@ -852,7 +1018,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { output_buffer_index); CHECK_EXCEPTION(jni); if (!success) { - ResetCodec(); + ResetCodecOnCodecThread(); return false; } @@ -907,7 +1073,12 @@ int MediaCodecVideoEncoder::GetTargetFramerate() { return scale_ ? quality_scaler_.GetTargetFramerate() : -1; } -MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { +const char* MediaCodecVideoEncoder::ImplementationName() const { + return "MediaCodec"; +} + +MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() + : egl_context_(nullptr) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder"); @@ -923,6 +1094,16 @@ MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); } + bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( + j_encoder_class, + GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z")); + CHECK_EXCEPTION(jni); + if (is_vp9_hw_supported) { + ALOGD << "VP9 HW Encoder supported."; + supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9", + MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); + } + bool is_h264_hw_supported = jni->CallStaticBooleanMethod( j_encoder_class, GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z")); @@ -936,9 +1117,37 @@ MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {} +void MediaCodecVideoEncoderFactory::SetEGLContext( + JNIEnv* jni, jobject render_egl_context) { + ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext"; + if (egl_context_) { + jni->DeleteGlobalRef(egl_context_); + egl_context_ = NULL; + } + if (!IsNull(jni, render_egl_context)) { + egl_context_ = jni->NewGlobalRef(render_egl_context); + if (CheckException(jni)) { + ALOGE << "error calling NewGlobalRef for EGL Context."; + egl_context_ = NULL; + } else { + jclass j_egl_context_class = + FindClass(jni, "org/webrtc/EglBase14$Context"); + if (!jni->IsInstanceOf(egl_context_, j_egl_context_class)) { + ALOGE << "Wrong EGL Context."; + jni->DeleteGlobalRef(egl_context_); + egl_context_ = NULL; + } + } + } + if (egl_context_ == NULL) { + ALOGW << "NULL VideoDecoder EGL context - HW surface encoding is disabled."; + } +} + webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( VideoCodecType type) { if (supported_codecs_.empty()) { + ALOGW << "No HW video encoder for type " << (int)type; return NULL; } for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin(); @@ -946,9 +1155,11 @@ webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( if (it->type == type) { ALOGD << "Create HW video encoder for type " << (int)type << " (" << it->name << ")."; - return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type); + return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type, + egl_context_); } } + ALOGW << "Can not find HW video encoder for type " << (int)type; return NULL; } diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.h b/talk/app/webrtc/java/jni/androidmediaencoder_jni.h index ff124aa146..8ff8164c3b 100644 --- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.h +++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.h @@ -43,6 +43,8 @@ class MediaCodecVideoEncoderFactory MediaCodecVideoEncoderFactory(); virtual ~MediaCodecVideoEncoderFactory(); + void SetEGLContext(JNIEnv* jni, jobject render_egl_context); + // WebRtcVideoEncoderFactory implementation. webrtc::VideoEncoder* CreateVideoEncoder(webrtc::VideoCodecType type) override; @@ -50,6 +52,7 @@ class MediaCodecVideoEncoderFactory void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override; private: + jobject egl_context_; // Empty if platform support is lacking, const after ctor returns. std::vector<VideoCodec> supported_codecs_; }; diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc index 02b9f22015..8813c89de4 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.cc @@ -29,8 +29,9 @@ #include "talk/app/webrtc/java/jni/androidvideocapturer_jni.h" #include "talk/app/webrtc/java/jni/classreferenceholder.h" #include "talk/app/webrtc/java/jni/native_handle_impl.h" +#include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h" +#include "third_party/libyuv/include/libyuv/convert.h" #include "webrtc/base/bind.h" -#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" namespace webrtc_jni { @@ -47,15 +48,19 @@ int AndroidVideoCapturerJni::SetAndroidObjects(JNIEnv* jni, return 0; } -AndroidVideoCapturerJni::AndroidVideoCapturerJni(JNIEnv* jni, - jobject j_video_capturer) - : j_capturer_global_(jni, j_video_capturer), +AndroidVideoCapturerJni::AndroidVideoCapturerJni( + JNIEnv* jni, + jobject j_video_capturer, + jobject j_surface_texture_helper) + : j_video_capturer_(jni, j_video_capturer), j_video_capturer_class_( jni, FindClass(jni, "org/webrtc/VideoCapturerAndroid")), j_observer_class_( jni, FindClass(jni, "org/webrtc/VideoCapturerAndroid$NativeObserver")), + surface_texture_helper_(new rtc::RefCountedObject<SurfaceTextureHelper>( + jni, j_surface_texture_helper)), capturer_(nullptr) { LOG(LS_INFO) << "AndroidVideoCapturerJni ctor"; thread_checker_.DetachFromThread(); @@ -64,7 +69,7 @@ AndroidVideoCapturerJni::AndroidVideoCapturerJni(JNIEnv* jni, AndroidVideoCapturerJni::~AndroidVideoCapturerJni() { LOG(LS_INFO) << "AndroidVideoCapturerJni dtor"; jni()->CallVoidMethod( - *j_capturer_global_, + *j_video_capturer_, GetMethodID(jni(), *j_video_capturer_class_, "release", "()V")); CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.release()"; } @@ -90,7 +95,7 @@ void AndroidVideoCapturerJni::Start(int width, int height, int framerate, jni(), *j_video_capturer_class_, "startCapture", "(IIILandroid/content/Context;" "Lorg/webrtc/VideoCapturerAndroid$CapturerObserver;)V"); - jni()->CallVoidMethod(*j_capturer_global_, + jni()->CallVoidMethod(*j_video_capturer_, m, width, height, framerate, application_context_, @@ -109,7 +114,7 @@ void AndroidVideoCapturerJni::Stop() { } jmethodID m = GetMethodID(jni(), *j_video_capturer_class_, "stopCapture", "()V"); - jni()->CallVoidMethod(*j_capturer_global_, m); + jni()->CallVoidMethod(*j_video_capturer_, m); CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.stopCapture"; LOG(LS_INFO) << "AndroidVideoCapturerJni stop done"; } @@ -127,19 +132,12 @@ void AndroidVideoCapturerJni::AsyncCapturerInvoke( invoker_->AsyncInvoke<void>(rtc::Bind(method, capturer_, args...)); } -void AndroidVideoCapturerJni::ReturnBuffer(int64_t time_stamp) { - jmethodID m = GetMethodID(jni(), *j_video_capturer_class_, - "returnBuffer", "(J)V"); - jni()->CallVoidMethod(*j_capturer_global_, m, time_stamp); - CHECK_EXCEPTION(jni()) << "error during VideoCapturerAndroid.returnBuffer"; -} - std::string AndroidVideoCapturerJni::GetSupportedFormats() { jmethodID m = GetMethodID(jni(), *j_video_capturer_class_, "getSupportedFormatsAsJson", "()Ljava/lang/String;"); jstring j_json_caps = - (jstring) jni()->CallObjectMethod(*j_capturer_global_, m); + (jstring) jni()->CallObjectMethod(*j_video_capturer_, m); CHECK_EXCEPTION(jni()) << "error during supportedFormatsAsJson"; return JavaToStdString(jni(), j_json_caps); } @@ -158,46 +156,33 @@ void AndroidVideoCapturerJni::OnMemoryBufferFrame(void* video_frame, int rotation, int64_t timestamp_ns) { const uint8_t* y_plane = static_cast<uint8_t*>(video_frame); - // Android guarantees that the stride is a multiple of 16. - // http://developer.android.com/reference/android/hardware/Camera.Parameters.html#setPreviewFormat%28int%29 - int y_stride; - int uv_stride; - webrtc::Calc16ByteAlignedStride(width, &y_stride, &uv_stride); - const uint8_t* v_plane = y_plane + y_stride * height; - const uint8_t* u_plane = - v_plane + uv_stride * webrtc::AlignInt(height, 2) / 2; - - // Wrap the Java buffer, and call ReturnBuffer() in the wrapped - // VideoFrameBuffer destructor. - rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer( - new rtc::RefCountedObject<webrtc::WrappedI420Buffer>( - width, height, y_plane, y_stride, u_plane, uv_stride, v_plane, - uv_stride, - rtc::Bind(&AndroidVideoCapturerJni::ReturnBuffer, this, - timestamp_ns))); + const uint8_t* vu_plane = y_plane + width * height; + + rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = + buffer_pool_.CreateBuffer(width, height); + libyuv::NV21ToI420( + y_plane, width, + vu_plane, width, + buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane), + buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane), + buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane), + width, height); AsyncCapturerInvoke("OnIncomingFrame", &webrtc::AndroidVideoCapturer::OnIncomingFrame, buffer, rotation, timestamp_ns); } -void AndroidVideoCapturerJni::OnTextureFrame( - int width, - int height, - int64_t timestamp_ns, - const NativeTextureHandleImpl& handle) { - // TODO(magjed): Fix this. See bug webrtc:4993. - RTC_NOTREACHED() - << "The rest of the stack for Android expects the native " - "handle to be a NativeHandleImpl with a SurfaceTexture, not a " - "NativeTextureHandleImpl"; +void AndroidVideoCapturerJni::OnTextureFrame(int width, + int height, + int rotation, + int64_t timestamp_ns, + const NativeHandleImpl& handle) { rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer( - new rtc::RefCountedObject<AndroidTextureBuffer>( - width, height, handle, - rtc::Bind(&AndroidVideoCapturerJni::ReturnBuffer, this, - timestamp_ns))); + surface_texture_helper_->CreateTextureFrame(width, height, handle)); + AsyncCapturerInvoke("OnIncomingFrame", &webrtc::AndroidVideoCapturer::OnIncomingFrame, - buffer, 0, timestamp_ns); + buffer, rotation, timestamp_ns); } void AndroidVideoCapturerJni::OnOutputFormatRequest(int width, @@ -216,13 +201,6 @@ JOW(void, jint width, jint height, jint rotation, jlong timestamp) { jboolean is_copy = true; jbyte* bytes = jni->GetByteArrayElements(j_frame, &is_copy); - // If this is a copy of the original frame, it means that the memory - // is not direct memory and thus VideoCapturerAndroid does not guarantee - // that the memory is valid when we have released |j_frame|. - // TODO(magjed): Move ReleaseByteArrayElements() into ReturnBuffer() and - // remove this check. - RTC_CHECK(!is_copy) - << "NativeObserver_nativeOnFrameCaptured: frame is a copy"; reinterpret_cast<AndroidVideoCapturerJni*>(j_capturer) ->OnMemoryBufferFrame(bytes, length, width, height, rotation, timestamp); jni->ReleaseByteArrayElements(j_frame, bytes, JNI_ABORT); @@ -231,11 +209,11 @@ JOW(void, JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeOnTextureFrameCaptured) (JNIEnv* jni, jclass, jlong j_capturer, jint j_width, jint j_height, jint j_oes_texture_id, jfloatArray j_transform_matrix, - jlong j_timestamp) { + jint j_rotation, jlong j_timestamp) { reinterpret_cast<AndroidVideoCapturerJni*>(j_capturer) - ->OnTextureFrame(j_width, j_height, j_timestamp, - NativeTextureHandleImpl(jni, j_oes_texture_id, - j_transform_matrix)); + ->OnTextureFrame(j_width, j_height, j_rotation, j_timestamp, + NativeHandleImpl(jni, j_oes_texture_id, + j_transform_matrix)); } JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeCapturerStarted) @@ -254,9 +232,11 @@ JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeOnOutputFormatRequest) } JOW(jlong, VideoCapturerAndroid_nativeCreateVideoCapturer) - (JNIEnv* jni, jclass, jobject j_video_capturer) { + (JNIEnv* jni, jclass, + jobject j_video_capturer, jobject j_surface_texture_helper) { rtc::scoped_refptr<webrtc::AndroidVideoCapturerDelegate> delegate = - new rtc::RefCountedObject<AndroidVideoCapturerJni>(jni, j_video_capturer); + new rtc::RefCountedObject<AndroidVideoCapturerJni>( + jni, j_video_capturer, j_surface_texture_helper); rtc::scoped_ptr<cricket::VideoCapturer> capturer( new webrtc::AndroidVideoCapturer(delegate)); // Caller takes ownership of the cricket::VideoCapturer* pointer. diff --git a/talk/app/webrtc/java/jni/androidvideocapturer_jni.h b/talk/app/webrtc/java/jni/androidvideocapturer_jni.h index d1eb3a0ad0..89ecacb3a5 100644 --- a/talk/app/webrtc/java/jni/androidvideocapturer_jni.h +++ b/talk/app/webrtc/java/jni/androidvideocapturer_jni.h @@ -36,10 +36,12 @@ #include "webrtc/base/asyncinvoker.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/thread_checker.h" +#include "webrtc/common_video/include/i420_buffer_pool.h" namespace webrtc_jni { -class NativeTextureHandleImpl; +struct NativeHandleImpl; +class SurfaceTextureHelper; // AndroidVideoCapturerJni implements AndroidVideoCapturerDelegate. // The purpose of the delegate is to hide the JNI specifics from the C++ only @@ -48,7 +50,9 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate { public: static int SetAndroidObjects(JNIEnv* jni, jobject appliction_context); - AndroidVideoCapturerJni(JNIEnv* jni, jobject j_video_capturer); + AndroidVideoCapturerJni(JNIEnv* jni, + jobject j_video_capturer, + jobject j_surface_texture_helper); void Start(int width, int height, int framerate, webrtc::AndroidVideoCapturer* capturer) override; @@ -60,15 +64,14 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate { void OnCapturerStarted(bool success); void OnMemoryBufferFrame(void* video_frame, int length, int width, int height, int rotation, int64_t timestamp_ns); - void OnTextureFrame(int width, int height, int64_t timestamp_ns, - const NativeTextureHandleImpl& handle); + void OnTextureFrame(int width, int height, int rotation, int64_t timestamp_ns, + const NativeHandleImpl& handle); void OnOutputFormatRequest(int width, int height, int fps); protected: ~AndroidVideoCapturerJni(); private: - void ReturnBuffer(int64_t time_stamp); JNIEnv* jni(); // To avoid deducing Args from the 3rd parameter of AsyncCapturerInvoke. @@ -85,10 +88,13 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate { void (webrtc::AndroidVideoCapturer::*method)(Args...), typename Identity<Args>::type... args); - const ScopedGlobalRef<jobject> j_capturer_global_; + const ScopedGlobalRef<jobject> j_video_capturer_; const ScopedGlobalRef<jclass> j_video_capturer_class_; const ScopedGlobalRef<jclass> j_observer_class_; + // Used on the Java thread running the camera. + webrtc::I420BufferPool buffer_pool_; + rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_; rtc::ThreadChecker thread_checker_; // |capturer| is a guaranteed to be a valid pointer between a call to diff --git a/talk/app/webrtc/java/jni/classreferenceholder.cc b/talk/app/webrtc/java/jni/classreferenceholder.cc index 4c836f8252..5fe8ec707c 100644 --- a/talk/app/webrtc/java/jni/classreferenceholder.cc +++ b/talk/app/webrtc/java/jni/classreferenceholder.cc @@ -72,20 +72,21 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) { LoadClass(jni, "org/webrtc/IceCandidate"); #if defined(ANDROID) && !defined(WEBRTC_CHROMIUM_BUILD) LoadClass(jni, "android/graphics/SurfaceTexture"); - LoadClass(jni, "javax/microedition/khronos/egl/EGLContext"); LoadClass(jni, "org/webrtc/CameraEnumerator"); LoadClass(jni, "org/webrtc/Camera2Enumerator"); LoadClass(jni, "org/webrtc/CameraEnumerationAndroid"); LoadClass(jni, "org/webrtc/VideoCapturerAndroid"); LoadClass(jni, "org/webrtc/VideoCapturerAndroid$NativeObserver"); LoadClass(jni, "org/webrtc/EglBase"); + LoadClass(jni, "org/webrtc/EglBase$Context"); + LoadClass(jni, "org/webrtc/EglBase14$Context"); LoadClass(jni, "org/webrtc/NetworkMonitor"); LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder"); LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$VideoCodecType"); LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder"); LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); - LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedByteBuffer"); + LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer"); LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType"); LoadClass(jni, "org/webrtc/SurfaceTextureHelper"); #endif diff --git a/talk/app/webrtc/java/jni/jni_helpers.cc b/talk/app/webrtc/java/jni/jni_helpers.cc index 755698e379..3a7ff21e77 100644 --- a/talk/app/webrtc/java/jni/jni_helpers.cc +++ b/talk/app/webrtc/java/jni/jni_helpers.cc @@ -1,4 +1,3 @@ - /* * libjingle * Copyright 2015 Google Inc. @@ -33,8 +32,6 @@ #include <sys/syscall.h> #include <unistd.h> -#include "unicode/unistr.h" - namespace webrtc_jni { static JavaVM* g_jvm = nullptr; @@ -46,8 +43,6 @@ static pthread_once_t g_jni_ptr_once = PTHREAD_ONCE_INIT; // were attached by the JVM because of a Java->native call. static pthread_key_t g_jni_ptr; -using icu::UnicodeString; - JavaVM *GetJVM() { RTC_CHECK(g_jvm) << "JNI_OnLoad failed to run?"; return g_jvm; @@ -232,22 +227,20 @@ bool IsNull(JNIEnv* jni, jobject obj) { // Given a UTF-8 encoded |native| string return a new (UTF-16) jstring. jstring JavaStringFromStdString(JNIEnv* jni, const std::string& native) { - UnicodeString ustr(UnicodeString::fromUTF8(native)); - jstring jstr = jni->NewString(ustr.getBuffer(), ustr.length()); - CHECK_EXCEPTION(jni) << "error during NewString"; + jstring jstr = jni->NewStringUTF(native.c_str()); + CHECK_EXCEPTION(jni) << "error during NewStringUTF"; return jstr; } // Given a (UTF-16) jstring return a new UTF-8 native string. std::string JavaToStdString(JNIEnv* jni, const jstring& j_string) { - const jchar* jchars = jni->GetStringChars(j_string, NULL); - CHECK_EXCEPTION(jni) << "Error during GetStringChars"; - UnicodeString ustr(jchars, jni->GetStringLength(j_string)); - CHECK_EXCEPTION(jni) << "Error during GetStringLength"; - jni->ReleaseStringChars(j_string, jchars); - CHECK_EXCEPTION(jni) << "Error during ReleaseStringChars"; - std::string ret; - return ustr.toUTF8String(ret); + const char* chars = jni->GetStringUTFChars(j_string, NULL); + CHECK_EXCEPTION(jni) << "Error during GetStringUTFChars"; + std::string str(chars, jni->GetStringUTFLength(j_string)); + CHECK_EXCEPTION(jni) << "Error during GetStringUTFLength"; + jni->ReleaseStringUTFChars(j_string, chars); + CHECK_EXCEPTION(jni) << "Error during ReleaseStringUTFChars"; + return str; } // Return the (singleton) Java Enum object corresponding to |index|; diff --git a/talk/app/webrtc/java/jni/jni_onload.cc b/talk/app/webrtc/java/jni/jni_onload.cc new file mode 100644 index 0000000000..9664ecdca6 --- /dev/null +++ b/talk/app/webrtc/java/jni/jni_onload.cc @@ -0,0 +1,55 @@ +/* + * libjingle + * Copyright 2015 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <jni.h> +#undef JNIEXPORT +#define JNIEXPORT __attribute__((visibility("default"))) + +#include "talk/app/webrtc/java/jni/classreferenceholder.h" +#include "talk/app/webrtc/java/jni/jni_helpers.h" +#include "webrtc/base/ssladapter.h" + +namespace webrtc_jni { + +extern "C" jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM *jvm, void *reserved) { + jint ret = InitGlobalJniVariables(jvm); + RTC_DCHECK_GE(ret, 0); + if (ret < 0) + return -1; + + RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()"; + LoadGlobalClassReferenceHolder(); + + return ret; +} + +extern "C" void JNIEXPORT JNICALL JNI_OnUnLoad(JavaVM *jvm, void *reserved) { + FreeGlobalClassReferenceHolder(); + RTC_CHECK(rtc::CleanupSSL()) << "Failed to CleanupSSL()"; +} + +} // namespace webrtc_jni diff --git a/talk/app/webrtc/java/jni/native_handle_impl.cc b/talk/app/webrtc/java/jni/native_handle_impl.cc index ac3e0455df..1757184154 100644 --- a/talk/app/webrtc/java/jni/native_handle_impl.cc +++ b/talk/app/webrtc/java/jni/native_handle_impl.cc @@ -27,14 +27,65 @@ #include "talk/app/webrtc/java/jni/native_handle_impl.h" +#include "talk/app/webrtc/java/jni/jni_helpers.h" +#include "webrtc/base/bind.h" #include "webrtc/base/checks.h" +#include "webrtc/base/keep_ref_until_done.h" +#include "webrtc/base/scoped_ptr.h" +#include "webrtc/base/scoped_ref_ptr.h" +#include "webrtc/base/logging.h" + +using webrtc::NativeHandleBuffer; + +namespace { + +void RotateMatrix(float a[16], webrtc::VideoRotation rotation) { + // Texture coordinates are in the range 0 to 1. The transformation of the last + // row in each rotation matrix is needed for proper translation, e.g, to + // mirror x, we don't replace x by -x, but by 1-x. + switch (rotation) { + case webrtc::kVideoRotation_0: + break; + case webrtc::kVideoRotation_90: { + const float ROTATE_90[16] = + { a[4], a[5], a[6], a[7], + -a[0], -a[1], -a[2], -a[3], + a[8], a[9], a[10], a[11], + a[0] + a[12], a[1] + a[13], a[2] + a[14], a[3] + a[15]}; + memcpy(a, ROTATE_90, sizeof(ROTATE_90)); + } break; + case webrtc::kVideoRotation_180: { + const float ROTATE_180[16] = + { -a[0], -a[1], -a[2], -a[3], + -a[4], -a[5], -a[6], -a[7], + a[8], a[9], a[10], a[11], + a[0] + a[4] + a[12], a[1] +a[5] + a[13], a[2] + a[6] + a[14], + a[3] + a[11]+ a[15]}; + memcpy(a, ROTATE_180, sizeof(ROTATE_180)); + } + break; + case webrtc::kVideoRotation_270: { + const float ROTATE_270[16] = + { -a[4], -a[5], -a[6], -a[7], + a[0], a[1], a[2], a[3], + a[8], a[9], a[10], a[11], + a[4] + a[12], a[5] + a[13], a[6] + a[14], a[7] + a[15]}; + memcpy(a, ROTATE_270, sizeof(ROTATE_270)); + } break; + } +} + +} // anonymouse namespace namespace webrtc_jni { -NativeTextureHandleImpl::NativeTextureHandleImpl(JNIEnv* jni, - jint j_oes_texture_id, - jfloatArray j_transform_matrix) - : oes_texture_id(j_oes_texture_id) { +// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD. +static const int kBufferAlignment = 64; + +NativeHandleImpl::NativeHandleImpl(JNIEnv* jni, + jint j_oes_texture_id, + jfloatArray j_transform_matrix) + : oes_texture_id(j_oes_texture_id) { RTC_CHECK_EQ(16, jni->GetArrayLength(j_transform_matrix)); jfloat* transform_matrix_ptr = jni->GetFloatArrayElements(j_transform_matrix, nullptr); @@ -44,41 +95,15 @@ NativeTextureHandleImpl::NativeTextureHandleImpl(JNIEnv* jni, jni->ReleaseFloatArrayElements(j_transform_matrix, transform_matrix_ptr, 0); } -NativeHandleImpl::NativeHandleImpl() : texture_object_(NULL), texture_id_(-1) {} - -void* NativeHandleImpl::GetHandle() { - return texture_object_; -} - -int NativeHandleImpl::GetTextureId() { - return texture_id_; -} - -void NativeHandleImpl::SetTextureObject(void* texture_object, int texture_id) { - texture_object_ = reinterpret_cast<jobject>(texture_object); - texture_id_ = texture_id; -} - -JniNativeHandleBuffer::JniNativeHandleBuffer(void* native_handle, - int width, - int height) - : NativeHandleBuffer(native_handle, width, height) {} - -rtc::scoped_refptr<webrtc::VideoFrameBuffer> -JniNativeHandleBuffer::NativeToI420Buffer() { - // TODO(pbos): Implement before using this in the encoder pipeline (or - // remove the RTC_CHECK() in VideoCapture). - RTC_NOTREACHED(); - return nullptr; -} - AndroidTextureBuffer::AndroidTextureBuffer( int width, int height, - const NativeTextureHandleImpl& native_handle, + const NativeHandleImpl& native_handle, + jobject surface_texture_helper, const rtc::Callback0<void>& no_longer_used) : webrtc::NativeHandleBuffer(&native_handle_, width, height), native_handle_(native_handle), + surface_texture_helper_(surface_texture_helper), no_longer_used_cb_(no_longer_used) {} AndroidTextureBuffer::~AndroidTextureBuffer() { @@ -87,9 +112,75 @@ AndroidTextureBuffer::~AndroidTextureBuffer() { rtc::scoped_refptr<webrtc::VideoFrameBuffer> AndroidTextureBuffer::NativeToI420Buffer() { - RTC_NOTREACHED() - << "AndroidTextureBuffer::NativeToI420Buffer not implemented."; - return nullptr; + int uv_width = (width()+7) / 8; + int stride = 8 * uv_width; + int uv_height = (height()+1)/2; + size_t size = stride * (height() + uv_height); + // The data is owned by the frame, and the normal case is that the + // data is deleted by the frame's destructor callback. + // + // TODO(nisse): Use an I420BufferPool. We then need to extend that + // class, and I420Buffer, to support our memory layout. + rtc::scoped_ptr<uint8_t, webrtc::AlignedFreeDeleter> yuv_data( + static_cast<uint8_t*>(webrtc::AlignedMalloc(size, kBufferAlignment))); + // See SurfaceTextureHelper.java for the required layout. + uint8_t* y_data = yuv_data.get(); + uint8_t* u_data = y_data + height() * stride; + uint8_t* v_data = u_data + stride/2; + + rtc::scoped_refptr<webrtc::VideoFrameBuffer> copy = + new rtc::RefCountedObject<webrtc::WrappedI420Buffer>( + width(), height(), + y_data, stride, + u_data, stride, + v_data, stride, + rtc::Bind(&webrtc::AlignedFree, yuv_data.release())); + + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedLocalRefFrame local_ref_frame(jni); + + jmethodID transform_mid = GetMethodID( + jni, + GetObjectClass(jni, surface_texture_helper_), + "textureToYUV", + "(Ljava/nio/ByteBuffer;IIII[F)V"); + + jobject byte_buffer = jni->NewDirectByteBuffer(y_data, size); + + // TODO(nisse): Keep java transform matrix around. + jfloatArray sampling_matrix = jni->NewFloatArray(16); + jni->SetFloatArrayRegion(sampling_matrix, 0, 16, + native_handle_.sampling_matrix); + + jni->CallVoidMethod(surface_texture_helper_, + transform_mid, + byte_buffer, width(), height(), stride, + native_handle_.oes_texture_id, sampling_matrix); + CHECK_EXCEPTION(jni) << "textureToYUV throwed an exception"; + + return copy; +} + +rtc::scoped_refptr<AndroidTextureBuffer> +AndroidTextureBuffer::ScaleAndRotate(int dst_widht, + int dst_height, + webrtc::VideoRotation rotation) { + if (width() == dst_widht && height() == dst_height && + rotation == webrtc::kVideoRotation_0) { + return this; + } + int rotated_width = (rotation % 180 == 0) ? dst_widht : dst_height; + int rotated_height = (rotation % 180 == 0) ? dst_height : dst_widht; + + // Here we use Bind magic to add a reference count to |this| until the newly + // created AndroidTextureBuffer is destructed + rtc::scoped_refptr<AndroidTextureBuffer> buffer( + new rtc::RefCountedObject<AndroidTextureBuffer>( + rotated_width, rotated_height, native_handle_, + surface_texture_helper_, rtc::KeepRefUntilDone(this))); + + RotateMatrix(buffer->native_handle_.sampling_matrix, rotation); + return buffer; } } // namespace webrtc_jni diff --git a/talk/app/webrtc/java/jni/native_handle_impl.h b/talk/app/webrtc/java/jni/native_handle_impl.h index dd04bc20b1..1d0f601d0d 100644 --- a/talk/app/webrtc/java/jni/native_handle_impl.h +++ b/talk/app/webrtc/java/jni/native_handle_impl.h @@ -31,56 +31,44 @@ #include <jni.h> -#include "webrtc/common_video/interface/video_frame_buffer.h" +#include "webrtc/common_video/include/video_frame_buffer.h" +#include "webrtc/common_video/rotation.h" namespace webrtc_jni { // Wrapper for texture object. -struct NativeTextureHandleImpl { - NativeTextureHandleImpl(JNIEnv* jni, - jint j_oes_texture_id, - jfloatArray j_transform_matrix); +struct NativeHandleImpl { + NativeHandleImpl(JNIEnv* jni, + jint j_oes_texture_id, + jfloatArray j_transform_matrix); const int oes_texture_id; float sampling_matrix[16]; }; -// Native handle for SurfaceTexture + texture id. -class NativeHandleImpl { - public: - NativeHandleImpl(); - - void* GetHandle(); - int GetTextureId(); - void SetTextureObject(void* texture_object, int texture_id); - - private: - jobject texture_object_; - int32_t texture_id_; -}; - -class JniNativeHandleBuffer : public webrtc::NativeHandleBuffer { - public: - JniNativeHandleBuffer(void* native_handle, int width, int height); - - // TODO(pbos): Override destructor to release native handle, at the moment the - // native handle is not released based on refcount. - - private: - rtc::scoped_refptr<webrtc::VideoFrameBuffer> NativeToI420Buffer() override; -}; - class AndroidTextureBuffer : public webrtc::NativeHandleBuffer { public: AndroidTextureBuffer(int width, int height, - const NativeTextureHandleImpl& native_handle, + const NativeHandleImpl& native_handle, + jobject surface_texture_helper, const rtc::Callback0<void>& no_longer_used); ~AndroidTextureBuffer(); rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override; + rtc::scoped_refptr<AndroidTextureBuffer> ScaleAndRotate( + int dst_widht, + int dst_height, + webrtc::VideoRotation rotation); + private: - NativeTextureHandleImpl native_handle_; + NativeHandleImpl native_handle_; + // Raw object pointer, relying on the caller, i.e., + // AndroidVideoCapturerJni or the C++ SurfaceTextureHelper, to keep + // a global reference. TODO(nisse): Make this a reference to the C++ + // SurfaceTextureHelper instead, but that requires some refactoring + // of AndroidVideoCapturerJni. + jobject surface_texture_helper_; rtc::Callback0<void> no_longer_used_cb_; }; diff --git a/talk/app/webrtc/java/jni/peerconnection_jni.cc b/talk/app/webrtc/java/jni/peerconnection_jni.cc index e75cd553b6..5ea63f74ae 100644 --- a/talk/app/webrtc/java/jni/peerconnection_jni.cc +++ b/talk/app/webrtc/java/jni/peerconnection_jni.cc @@ -57,6 +57,7 @@ #define JNIEXPORT __attribute__((visibility("default"))) #include <limits> +#include <utility> #include "talk/app/webrtc/java/jni/classreferenceholder.h" #include "talk/app/webrtc/java/jni/jni_helpers.h" @@ -74,10 +75,11 @@ #include "talk/media/webrtc/webrtcvideoencoderfactory.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" +#include "webrtc/base/event_tracer.h" #include "webrtc/base/logging.h" #include "webrtc/base/logsinks.h" -#include "webrtc/base/networkmonitor.h" #include "webrtc/base/messagequeue.h" +#include "webrtc/base/networkmonitor.h" #include "webrtc/base/ssladapter.h" #include "webrtc/base/stringutils.h" #include "webrtc/system_wrappers/include/field_trial_default.h" @@ -141,22 +143,6 @@ static bool factory_static_initialized = false; static bool video_hw_acceleration_enabled = true; #endif -extern "C" jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM *jvm, void *reserved) { - jint ret = InitGlobalJniVariables(jvm); - if (ret < 0) - return -1; - - RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()"; - LoadGlobalClassReferenceHolder(); - - return ret; -} - -extern "C" void JNIEXPORT JNICALL JNI_OnUnLoad(JavaVM *jvm, void *reserved) { - FreeGlobalClassReferenceHolder(); - RTC_CHECK(rtc::CleanupSSL()) << "Failed to CleanupSSL()"; -} - // Return the (singleton) Java Enum object corresponding to |index|; // |state_class_fragment| is something like "MediaSource$State". static jobject JavaEnumFromIndex( @@ -545,7 +531,7 @@ class SdpObserverWrapper : public T { protected: // Common implementation for failure of Set & Create types, distinguished by // |op| being "Set" or "Create". - void OnFailure(const std::string& op, const std::string& error) { + void DoOnFailure(const std::string& op, const std::string& error) { jmethodID m = GetMethodID(jni(), *j_observer_class_, "on" + op + "Failure", "(Ljava/lang/String;)V"); jstring j_error_string = JavaStringFromStdString(jni(), error); @@ -572,7 +558,7 @@ class CreateSdpObserverWrapper void OnFailure(const std::string& error) override { ScopedLocalRefFrame local_ref_frame(jni()); - SdpObserverWrapper::OnFailure(std::string("Create"), error); + SdpObserverWrapper::DoOnFailure(std::string("Create"), error); } }; @@ -585,7 +571,7 @@ class SetSdpObserverWrapper void OnFailure(const std::string& error) override { ScopedLocalRefFrame local_ref_frame(jni()); - SdpObserverWrapper::OnFailure(std::string("Set"), error); + SdpObserverWrapper::DoOnFailure(std::string("Set"), error); } }; @@ -773,7 +759,7 @@ class JavaVideoRendererWrapper : public VideoRendererInterface { jni, *j_frame_class_, "<init>", "(III[I[Ljava/nio/ByteBuffer;J)V")), j_texture_frame_ctor_id_(GetMethodID( jni, *j_frame_class_, "<init>", - "(IIILjava/lang/Object;IJ)V")), + "(IIII[FJ)V")), j_byte_buffer_class_(jni, FindClass(jni, "java/nio/ByteBuffer")) { CHECK_EXCEPTION(jni); } @@ -829,13 +815,13 @@ class JavaVideoRendererWrapper : public VideoRendererInterface { jobject CricketToJavaTextureFrame(const cricket::VideoFrame* frame) { NativeHandleImpl* handle = reinterpret_cast<NativeHandleImpl*>(frame->GetNativeHandle()); - jobject texture_object = reinterpret_cast<jobject>(handle->GetHandle()); - int texture_id = handle->GetTextureId(); + jfloatArray sampling_matrix = jni()->NewFloatArray(16); + jni()->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); return jni()->NewObject( *j_frame_class_, j_texture_frame_ctor_id_, frame->GetWidth(), frame->GetHeight(), static_cast<int>(frame->GetVideoRotation()), - texture_object, texture_id, javaShallowCopy(frame)); + handle->oes_texture_id, sampling_matrix, javaShallowCopy(frame)); } JNIEnv* jni() { @@ -1054,6 +1040,32 @@ JOW(void, PeerConnectionFactory_initializeFieldTrials)( webrtc::field_trial::InitFieldTrialsFromString(field_trials_init_string); } +JOW(void, PeerConnectionFactory_initializeInternalTracer)(JNIEnv* jni, jclass) { + rtc::tracing::SetupInternalTracer(); +} + +JOW(jboolean, PeerConnectionFactory_startInternalTracingCapture)( + JNIEnv* jni, jclass, jstring j_event_tracing_filename) { + if (!j_event_tracing_filename) + return false; + + const char* init_string = + jni->GetStringUTFChars(j_event_tracing_filename, NULL); + LOG(LS_INFO) << "Starting internal tracing to: " << init_string; + bool ret = rtc::tracing::StartInternalCapture(init_string); + jni->ReleaseStringUTFChars(j_event_tracing_filename, init_string); + return ret; +} + +JOW(void, PeerConnectionFactory_stopInternalTracingCapture)( + JNIEnv* jni, jclass) { + rtc::tracing::StopInternalCapture(); +} + +JOW(void, PeerConnectionFactory_shutdownInternalTracer)(JNIEnv* jni, jclass) { + rtc::tracing::ShutdownInternalTracer(); +} + // Helper struct for working around the fact that CreatePeerConnectionFactory() // comes in two flavors: either entirely automagical (constructing its own // threads and deleting them on teardown, but no external codec factory support) @@ -1251,6 +1263,46 @@ JOW(jlong, PeerConnectionFactory_nativeCreateAudioTrack)( return (jlong)track.release(); } +JOW(jboolean, PeerConnectionFactory_nativeStartAecDump)( + JNIEnv* jni, jclass, jlong native_factory, jint file) { +#if defined(ANDROID) + rtc::scoped_refptr<PeerConnectionFactoryInterface> factory( + factoryFromJava(native_factory)); + return factory->StartAecDump(file); +#else + return false; +#endif +} + +JOW(void, PeerConnectionFactory_nativeStopAecDump)( + JNIEnv* jni, jclass, jlong native_factory) { +#if defined(ANDROID) + rtc::scoped_refptr<PeerConnectionFactoryInterface> factory( + factoryFromJava(native_factory)); + factory->StopAecDump(); +#endif +} + +JOW(jboolean, PeerConnectionFactory_nativeStartRtcEventLog)( + JNIEnv* jni, jclass, jlong native_factory, jint file) { +#if defined(ANDROID) + rtc::scoped_refptr<PeerConnectionFactoryInterface> factory( + factoryFromJava(native_factory)); + return factory->StartRtcEventLog(file); +#else + return false; +#endif +} + +JOW(void, PeerConnectionFactory_nativeStopRtcEventLog)( + JNIEnv* jni, jclass, jlong native_factory) { +#if defined(ANDROID) + rtc::scoped_refptr<PeerConnectionFactoryInterface> factory( + factoryFromJava(native_factory)); + factory->StopRtcEventLog(); +#endif +} + JOW(void, PeerConnectionFactory_nativeSetOptions)( JNIEnv* jni, jclass, jlong native_factory, jobject options) { rtc::scoped_refptr<PeerConnectionFactoryInterface> factory( @@ -1292,21 +1344,35 @@ JOW(void, PeerConnectionFactory_nativeSetOptions)( } JOW(void, PeerConnectionFactory_nativeSetVideoHwAccelerationOptions)( - JNIEnv* jni, jclass, jlong native_factory, jobject render_egl_context) { + JNIEnv* jni, jclass, jlong native_factory, jobject local_egl_context, + jobject remote_egl_context) { #if defined(ANDROID) && !defined(WEBRTC_CHROMIUM_BUILD) OwnedFactoryAndThreads* owned_factory = reinterpret_cast<OwnedFactoryAndThreads*>(native_factory); + + jclass j_eglbase14_context_class = + FindClass(jni, "org/webrtc/EglBase14$Context"); + + MediaCodecVideoEncoderFactory* encoder_factory = + static_cast<MediaCodecVideoEncoderFactory*> + (owned_factory->encoder_factory()); + if (encoder_factory && + jni->IsInstanceOf(local_egl_context, j_eglbase14_context_class)) { + LOG(LS_INFO) << "Set EGL context for HW encoding."; + encoder_factory->SetEGLContext(jni, local_egl_context); + } + MediaCodecVideoDecoderFactory* decoder_factory = static_cast<MediaCodecVideoDecoderFactory*> (owned_factory->decoder_factory()); - if (decoder_factory) { - LOG(LS_INFO) << "Set EGL context for HW acceleration."; - decoder_factory->SetEGLContext(jni, render_egl_context); + if (decoder_factory && + jni->IsInstanceOf(remote_egl_context, j_eglbase14_context_class)) { + LOG(LS_INFO) << "Set EGL context for HW decoding."; + decoder_factory->SetEGLContext(jni, remote_egl_context); } #endif } - static std::string GetJavaEnumName(JNIEnv* jni, const std::string& className, jobject j_enum) { jclass enumClass = FindClass(jni, className.c_str()); @@ -1503,6 +1569,9 @@ static void JavaRTCConfigurationToJsepRTCConfiguration( jfieldID j_ice_connection_receiving_timeout_id = GetFieldID(jni, j_rtc_config_class, "iceConnectionReceivingTimeout", "I"); + jfieldID j_ice_backup_candidate_pair_ping_interval_id = GetFieldID( + jni, j_rtc_config_class, "iceBackupCandidatePairPingInterval", "I"); + jfieldID j_continual_gathering_policy_id = GetFieldID(jni, j_rtc_config_class, "continualGatheringPolicy", "Lorg/webrtc/PeerConnection$ContinualGatheringPolicy;"); @@ -1524,6 +1593,8 @@ static void JavaRTCConfigurationToJsepRTCConfiguration( jni, j_rtc_config, j_audio_jitter_buffer_fast_accelerate_id); rtc_config->ice_connection_receiving_timeout = GetIntField(jni, j_rtc_config, j_ice_connection_receiving_timeout_id); + rtc_config->ice_backup_candidate_pair_ping_interval = GetIntField( + jni, j_rtc_config, j_ice_backup_candidate_pair_ping_interval_id); rtc_config->continual_gathering_policy = JavaContinualGatheringPolicyToNativeType( jni, j_continual_gathering_policy); @@ -1550,7 +1621,7 @@ JOW(jlong, PeerConnectionFactory_nativeCreatePeerConnection)( rtc::SSLIdentity::Generate(webrtc::kIdentityName, rtc::KT_ECDSA)); if (ssl_identity.get()) { rtc_config.certificates.push_back( - rtc::RTCCertificate::Create(ssl_identity.Pass())); + rtc::RTCCertificate::Create(std::move(ssl_identity))); LOG(LS_INFO) << "ECDSA certificate created."; } else { // Failing to create certificate should not abort peer connection @@ -1704,6 +1775,29 @@ JOW(void, PeerConnection_nativeRemoveLocalStream)( reinterpret_cast<MediaStreamInterface*>(native_stream)); } +JOW(jobject, PeerConnection_nativeCreateSender)( + JNIEnv* jni, jobject j_pc, jstring j_kind, jstring j_stream_id) { + jclass j_rtp_sender_class = FindClass(jni, "org/webrtc/RtpSender"); + jmethodID j_rtp_sender_ctor = + GetMethodID(jni, j_rtp_sender_class, "<init>", "(J)V"); + + std::string kind = JavaToStdString(jni, j_kind); + std::string stream_id = JavaToStdString(jni, j_stream_id); + rtc::scoped_refptr<RtpSenderInterface> sender = + ExtractNativePC(jni, j_pc)->CreateSender(kind, stream_id); + if (!sender.get()) { + return nullptr; + } + jlong nativeSenderPtr = jlongFromPointer(sender.get()); + jobject j_sender = + jni->NewObject(j_rtp_sender_class, j_rtp_sender_ctor, nativeSenderPtr); + CHECK_EXCEPTION(jni) << "error during NewObject"; + // Sender is now owned by the Java object, and will be freed from + // RtpSender.dispose(), called by PeerConnection.dispose() or getSenders(). + sender->AddRef(); + return j_sender; +} + JOW(jobject, PeerConnection_nativeGetSenders)(JNIEnv* jni, jobject j_pc) { jclass j_array_list_class = FindClass(jni, "java/util/ArrayList"); jmethodID j_array_list_ctor = @@ -1723,7 +1817,8 @@ JOW(jobject, PeerConnection_nativeGetSenders)(JNIEnv* jni, jobject j_pc) { jobject j_sender = jni->NewObject(j_rtp_sender_class, j_rtp_sender_ctor, nativeSenderPtr); CHECK_EXCEPTION(jni) << "error during NewObject"; - // Sender is now owned by Java object, and will be freed from there. + // Sender is now owned by the Java object, and will be freed from + // RtpSender.dispose(), called by PeerConnection.dispose() or getSenders(). sender->AddRef(); jni->CallBooleanMethod(j_senders, j_array_list_add, j_sender); CHECK_EXCEPTION(jni) << "error during CallBooleanMethod"; @@ -1802,6 +1897,7 @@ JOW(jobject, VideoCapturer_nativeCreateVideoCapturer)( // Since we can't create platform specific java implementations in Java, we // defer the creation to C land. #if defined(ANDROID) + // TODO(nisse): This case is intended to be deleted. jclass j_video_capturer_class( FindClass(jni, "org/webrtc/VideoCapturerAndroid")); const int camera_id = jni->CallStaticIntMethod( @@ -1816,8 +1912,13 @@ JOW(jobject, VideoCapturer_nativeCreateVideoCapturer)( j_video_capturer_class, GetMethodID(jni, j_video_capturer_class, "<init>", "(I)V"), camera_id); CHECK_EXCEPTION(jni) << "error during creation of VideoCapturerAndroid"; + jfieldID helper_fid = GetFieldID(jni, j_video_capturer_class, "surfaceHelper", + "Lorg/webrtc/SurfaceTextureHelper;"); + rtc::scoped_refptr<webrtc::AndroidVideoCapturerDelegate> delegate = - new rtc::RefCountedObject<AndroidVideoCapturerJni>(jni, j_video_capturer); + new rtc::RefCountedObject<AndroidVideoCapturerJni>( + jni, j_video_capturer, + GetObjectField(jni, j_video_capturer, helper_fid)); rtc::scoped_ptr<cricket::VideoCapturer> capturer( new webrtc::AndroidVideoCapturer(delegate)); @@ -2003,11 +2104,11 @@ JOW(jbyteArray, CallSessionFileRotatingLogSink_nativeGetLogData)( return result; } -JOW(void, RtpSender_nativeSetTrack)(JNIEnv* jni, +JOW(jboolean, RtpSender_nativeSetTrack)(JNIEnv* jni, jclass, jlong j_rtp_sender_pointer, jlong j_track_pointer) { - reinterpret_cast<RtpSenderInterface*>(j_rtp_sender_pointer) + return reinterpret_cast<RtpSenderInterface*>(j_rtp_sender_pointer) ->SetTrack(reinterpret_cast<MediaStreamTrackInterface*>(j_track_pointer)); } diff --git a/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc index 05f1b23768..3e32b9a6fe 100644 --- a/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc +++ b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc @@ -35,25 +35,14 @@ namespace webrtc_jni { -SurfaceTextureHelper::SurfaceTextureHelper(JNIEnv* jni, - jobject egl_shared_context) - : j_surface_texture_helper_class_( - jni, - FindClass(jni, "org/webrtc/SurfaceTextureHelper")), - j_surface_texture_helper_( - jni, - jni->CallStaticObjectMethod( - *j_surface_texture_helper_class_, - GetStaticMethodID(jni, - *j_surface_texture_helper_class_, - "create", - "(Ljavax/microedition/khronos/egl/EGLContext;)" - "Lorg/webrtc/SurfaceTextureHelper;"), - egl_shared_context)), - j_return_texture_method_(GetMethodID(jni, - *j_surface_texture_helper_class_, - "returnTextureFrame", - "()V")) { +SurfaceTextureHelper::SurfaceTextureHelper( + JNIEnv* jni, jobject surface_texture_helper) + : j_surface_texture_helper_(jni, surface_texture_helper), + j_return_texture_method_( + GetMethodID(jni, + FindClass(jni, "org/webrtc/SurfaceTextureHelper"), + "returnTextureFrame", + "()V")) { CHECK_EXCEPTION(jni) << "error during initialization of SurfaceTextureHelper"; } @@ -70,9 +59,9 @@ void SurfaceTextureHelper::ReturnTextureFrame() const { rtc::scoped_refptr<webrtc::VideoFrameBuffer> SurfaceTextureHelper::CreateTextureFrame(int width, int height, - const NativeTextureHandleImpl& native_handle) { + const NativeHandleImpl& native_handle) { return new rtc::RefCountedObject<AndroidTextureBuffer>( - width, height, native_handle, + width, height, native_handle, *j_surface_texture_helper_, rtc::Bind(&SurfaceTextureHelper::ReturnTextureFrame, this)); } diff --git a/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h index dc9d2b853d..8dde2b54ed 100644 --- a/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h +++ b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h @@ -35,7 +35,7 @@ #include "talk/app/webrtc/java/jni/native_handle_impl.h" #include "webrtc/base/refcount.h" #include "webrtc/base/scoped_ref_ptr.h" -#include "webrtc/common_video/interface/video_frame_buffer.h" +#include "webrtc/common_video/include/video_frame_buffer.h" namespace webrtc_jni { @@ -49,24 +49,19 @@ namespace webrtc_jni { // destroyed while a VideoFrameBuffer is in use. // This class is the C++ counterpart of the java class SurfaceTextureHelper. // Usage: -// 1. Create an instance of this class. -// 2. Call GetJavaSurfaceTextureHelper to get the Java SurfaceTextureHelper. +// 1. Create an java instance of SurfaceTextureHelper. +// 2. Create an instance of this class. // 3. Register a listener to the Java SurfaceListener and start producing // new buffers. -// 3. Call CreateTextureFrame to wrap the Java texture in a VideoFrameBuffer. +// 4. Call CreateTextureFrame to wrap the Java texture in a VideoFrameBuffer. class SurfaceTextureHelper : public rtc::RefCountInterface { public: - SurfaceTextureHelper(JNIEnv* jni, jobject shared_egl_context); - - // Returns the Java SurfaceTextureHelper. - jobject GetJavaSurfaceTextureHelper() const { - return *j_surface_texture_helper_; - } + SurfaceTextureHelper(JNIEnv* jni, jobject surface_texture_helper); rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateTextureFrame( int width, int height, - const NativeTextureHandleImpl& native_handle); + const NativeHandleImpl& native_handle); protected: ~SurfaceTextureHelper(); @@ -75,7 +70,6 @@ class SurfaceTextureHelper : public rtc::RefCountInterface { // May be called on arbitrary thread. void ReturnTextureFrame() const; - const ScopedGlobalRef<jclass> j_surface_texture_helper_class_; const ScopedGlobalRef<jobject> j_surface_texture_helper_; const jmethodID j_return_texture_method_; }; diff --git a/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoDecoder.java b/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoDecoder.java index 42af9c7fd0..19002f70e1 100644 --- a/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoDecoder.java +++ b/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoDecoder.java @@ -33,23 +33,23 @@ import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecList; import android.media.MediaFormat; -import android.opengl.GLES11Ext; -import android.opengl.GLES20; import android.os.Build; +import android.os.SystemClock; import android.view.Surface; import org.webrtc.Logging; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; - -import javax.microedition.khronos.egl.EGLContext; +import java.util.concurrent.CountDownLatch; +import java.util.Queue; +import java.util.concurrent.TimeUnit; // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. // This class is an implementation detail of the Java PeerConnection API. -// MediaCodec is thread-hostile so this class must be operated on a single -// thread. +@SuppressWarnings("deprecation") public class MediaCodecVideoDecoder { // This class is constructed, operated, and destroyed by its C++ incarnation, // so the class and its methods have non-public visibility. The API this @@ -66,18 +66,26 @@ public class MediaCodecVideoDecoder { } private static final int DEQUEUE_INPUT_TIMEOUT = 500000; // 500 ms timeout. + private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; // Timeout for codec releasing. // Active running decoder instance. Set in initDecode() (called from native code) // and reset to null in release() call. private static MediaCodecVideoDecoder runningInstance = null; + private static MediaCodecVideoDecoderErrorCallback errorCallback = null; + private static int codecErrors = 0; + private Thread mediaCodecThread; private MediaCodec mediaCodec; private ByteBuffer[] inputBuffers; private ByteBuffer[] outputBuffers; private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; + private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; private static final String H264_MIME_TYPE = "video/avc"; // List of supported HW VP8 decoders. private static final String[] supportedVp8HwCodecPrefixes = {"OMX.qcom.", "OMX.Nvidia.", "OMX.Exynos.", "OMX.Intel." }; + // List of supported HW VP9 decoders. + private static final String[] supportedVp9HwCodecPrefixes = + {"OMX.qcom.", "OMX.Exynos." }; // List of supported HW H.264 decoders. private static final String[] supportedH264HwCodecPrefixes = {"OMX.qcom.", "OMX.Intel." }; @@ -96,13 +104,29 @@ public class MediaCodecVideoDecoder { private int height; private int stride; private int sliceHeight; + private boolean hasDecodedFirstFrame; + private final Queue<TimeStamps> decodeStartTimeMs = new LinkedList<TimeStamps>(); private boolean useSurface; - private int textureID = 0; - private SurfaceTexture surfaceTexture = null; + + // The below variables are only used when decoding to a Surface. + private TextureListener textureListener; + // Max number of output buffers queued before starting to drop decoded frames. + private static final int MAX_QUEUED_OUTPUTBUFFERS = 3; + private int droppedFrames; private Surface surface = null; - private EglBase eglBase; + private final Queue<DecodedOutputBuffer> + dequeuedSurfaceOutputBuffers = new LinkedList<DecodedOutputBuffer>(); + + // MediaCodec error handler - invoked when critical error happens which may prevent + // further use of media codec API. Now it means that one of media codec instances + // is hanging and can no longer be used in the next call. + public static interface MediaCodecVideoDecoderErrorCallback { + void onMediaCodecVideoDecoderCriticalError(int codecErrors); + } - private MediaCodecVideoDecoder() { + public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorCallback) { + Logging.d(TAG, "Set error callback"); + MediaCodecVideoDecoder.errorCallback = errorCallback; } // Helper struct for findVp8Decoder() below. @@ -120,6 +144,7 @@ public class MediaCodecVideoDecoder { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { return null; // MediaCodec.setParameters is missing. } + Logging.d(TAG, "Trying to find HW decoder for mime " + mime); for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i); if (info.isEncoder()) { @@ -135,7 +160,7 @@ public class MediaCodecVideoDecoder { if (name == null) { continue; // No HW support in this codec; try the next one. } - Logging.v(TAG, "Found candidate decoder " + name); + Logging.d(TAG, "Found candidate decoder " + name); // Check if this is supported decoder. boolean supportedCodec = false; @@ -166,6 +191,7 @@ public class MediaCodecVideoDecoder { } } } + Logging.d(TAG, "No HW decoder found for mime " + mime); return null; // No HW decoder. } @@ -173,6 +199,10 @@ public class MediaCodecVideoDecoder { return findDecoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null; } + public static boolean isVp9HwSupported() { + return findDecoder(VP9_MIME_TYPE, supportedVp9HwCodecPrefixes) != null; + } + public static boolean isH264HwSupported() { return findDecoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null; } @@ -197,17 +227,21 @@ public class MediaCodecVideoDecoder { } } - // Pass null in |sharedContext| to configure the codec for ByteBuffer output. - private boolean initDecode(VideoCodecType type, int width, int height, EGLContext sharedContext) { + // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output. + private boolean initDecode( + VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTextureHelper) { if (mediaCodecThread != null) { throw new RuntimeException("Forgot to release()?"); } - useSurface = (sharedContext != null); + useSurface = (surfaceTextureHelper != null); String mime = null; String[] supportedCodecPrefixes = null; if (type == VideoCodecType.VIDEO_CODEC_VP8) { mime = VP8_MIME_TYPE; supportedCodecPrefixes = supportedVp8HwCodecPrefixes; + } else if (type == VideoCodecType.VIDEO_CODEC_VP9) { + mime = VP9_MIME_TYPE; + supportedCodecPrefixes = supportedVp9HwCodecPrefixes; } else if (type == VideoCodecType.VIDEO_CODEC_H264) { mime = H264_MIME_TYPE; supportedCodecPrefixes = supportedH264HwCodecPrefixes; @@ -221,9 +255,6 @@ public class MediaCodecVideoDecoder { Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + ". Color: 0x" + Integer.toHexString(properties.colorFormat) + ". Use Surface: " + useSurface); - if (sharedContext != null) { - Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext); - } runningInstance = this; // Decoder is now running and can be queried for stack traces. mediaCodecThread = Thread.currentThread(); try { @@ -233,16 +264,8 @@ public class MediaCodecVideoDecoder { sliceHeight = height; if (useSurface) { - // Create shared EGL context. - eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER); - eglBase.createDummyPbufferSurface(); - eglBase.makeCurrent(); - - // Create output surface - textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); - Logging.d(TAG, "Video decoder TextureID = " + textureID); - surfaceTexture = new SurfaceTexture(textureID); - surface = new Surface(surfaceTexture); + textureListener = new TextureListener(surfaceTextureHelper); + surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); } MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); @@ -261,6 +284,10 @@ public class MediaCodecVideoDecoder { colorFormat = properties.colorFormat; outputBuffers = mediaCodec.getOutputBuffers(); inputBuffers = mediaCodec.getInputBuffers(); + decodeStartTimeMs.clear(); + hasDecodedFirstFrame = false; + dequeuedSurfaceOutputBuffers.clear(); + droppedFrames = 0; Logging.d(TAG, "Input buffers: " + inputBuffers.length + ". Output buffers: " + outputBuffers.length); return true; @@ -271,25 +298,45 @@ public class MediaCodecVideoDecoder { } private void release() { - Logging.d(TAG, "Java releaseDecoder"); + Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames); checkOnMediaCodecThread(); - try { - mediaCodec.stop(); - mediaCodec.release(); - } catch (IllegalStateException e) { - Logging.e(TAG, "release failed", e); + + // Run Mediacodec stop() and release() on separate thread since sometime + // Mediacodec.stop() may hang. + final CountDownLatch releaseDone = new CountDownLatch(1); + + Runnable runMediaCodecRelease = new Runnable() { + @Override + public void run() { + try { + Logging.d(TAG, "Java releaseDecoder on release thread"); + mediaCodec.stop(); + mediaCodec.release(); + Logging.d(TAG, "Java releaseDecoder on release thread done"); + } catch (Exception e) { + Logging.e(TAG, "Media decoder release failed", e); + } + releaseDone.countDown(); + } + }; + new Thread(runMediaCodecRelease).start(); + + if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { + Logging.e(TAG, "Media decoder release timeout"); + codecErrors++; + if (errorCallback != null) { + Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); + errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); + } } + mediaCodec = null; mediaCodecThread = null; runningInstance = null; if (useSurface) { surface.release(); surface = null; - Logging.d(TAG, "Delete video decoder TextureID " + textureID); - GLES20.glDeleteTextures(1, new int[] {textureID}, 0); - textureID = 0; - eglBase.release(); - eglBase = null; + textureListener.release(); } Logging.d(TAG, "Java releaseDecoder done"); } @@ -306,13 +353,15 @@ public class MediaCodecVideoDecoder { } } - private boolean queueInputBuffer( - int inputBufferIndex, int size, long timestampUs) { + private boolean queueInputBuffer(int inputBufferIndex, int size, long presentationTimeStamUs, + long timeStampMs, long ntpTimeStamp) { checkOnMediaCodecThread(); try { inputBuffers[inputBufferIndex].position(0); inputBuffers[inputBufferIndex].limit(size); - mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); + decodeStartTimeMs.add(new TimeStamps(SystemClock.elapsedRealtime(), timeStampMs, + ntpTimeStamp)); + mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, presentationTimeStamUs, 0); return true; } catch (IllegalStateException e) { @@ -321,56 +370,183 @@ public class MediaCodecVideoDecoder { } } - // Helper structs for dequeueOutputBuffer() below. - private static class DecodedByteBuffer { - public DecodedByteBuffer(int index, int offset, int size, long presentationTimestampUs) { + private static class TimeStamps { + public TimeStamps(long decodeStartTimeMs, long timeStampMs, long ntpTimeStampMs) { + this.decodeStartTimeMs = decodeStartTimeMs; + this.timeStampMs = timeStampMs; + this.ntpTimeStampMs = ntpTimeStampMs; + } + private final long decodeStartTimeMs; // Time when this frame was queued for decoding. + private final long timeStampMs; // Only used for bookkeeping in Java. Used in C++; + private final long ntpTimeStampMs; // Only used for bookkeeping in Java. Used in C++; + } + + // Helper struct for dequeueOutputBuffer() below. + private static class DecodedOutputBuffer { + public DecodedOutputBuffer(int index, int offset, int size, long timeStampMs, + long ntpTimeStampMs, long decodeTime, long endDecodeTime) { this.index = index; this.offset = offset; this.size = size; - this.presentationTimestampUs = presentationTimestampUs; + this.timeStampMs = timeStampMs; + this.ntpTimeStampMs = ntpTimeStampMs; + this.decodeTimeMs = decodeTime; + this.endDecodeTimeMs = endDecodeTime; } private final int index; private final int offset; private final int size; - private final long presentationTimestampUs; + private final long timeStampMs; + private final long ntpTimeStampMs; + // Number of ms it took to decode this frame. + private final long decodeTimeMs; + // System time when this frame finished decoding. + private final long endDecodeTimeMs; } + // Helper struct for dequeueTextureBuffer() below. private static class DecodedTextureBuffer { private final int textureID; - private final long presentationTimestampUs; + private final float[] transformMatrix; + private final long timeStampMs; + private final long ntpTimeStampMs; + private final long decodeTimeMs; + // Interval from when the frame finished decoding until this buffer has been created. + // Since there is only one texture, this interval depend on the time from when + // a frame is decoded and provided to C++ and until that frame is returned to the MediaCodec + // so that the texture can be updated with the next decoded frame. + private final long frameDelayMs; - public DecodedTextureBuffer(int textureID, long presentationTimestampUs) { + // A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame + // that was dropped. + public DecodedTextureBuffer(int textureID, float[] transformMatrix, long timeStampMs, + long ntpTimeStampMs, long decodeTimeMs, long frameDelay) { this.textureID = textureID; - this.presentationTimestampUs = presentationTimestampUs; + this.transformMatrix = transformMatrix; + this.timeStampMs = timeStampMs; + this.ntpTimeStampMs = ntpTimeStampMs; + this.decodeTimeMs = decodeTimeMs; + this.frameDelayMs = frameDelay; } } - // Returns null if no decoded buffer is available, and otherwise either a DecodedByteBuffer or - // DecodedTexturebuffer depending on |useSurface| configuration. + // Poll based texture listener. + private static class TextureListener + implements SurfaceTextureHelper.OnTextureFrameAvailableListener { + private final SurfaceTextureHelper surfaceTextureHelper; + // |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll(). + private final Object newFrameLock = new Object(); + // |bufferToRender| is non-null when waiting for transition between addBufferToRender() to + // onTextureFrameAvailable(). + private DecodedOutputBuffer bufferToRender; + private DecodedTextureBuffer renderedBuffer; + + public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { + this.surfaceTextureHelper = surfaceTextureHelper; + surfaceTextureHelper.setListener(this); + } + + public void addBufferToRender(DecodedOutputBuffer buffer) { + if (bufferToRender != null) { + Logging.e(TAG, + "Unexpected addBufferToRender() called while waiting for a texture."); + throw new IllegalStateException("Waiting for a texture."); + } + bufferToRender = buffer; + } + + public boolean isWaitingForTexture() { + synchronized (newFrameLock) { + return bufferToRender != null; + } + } + + // Callback from |surfaceTextureHelper|. May be called on an arbitrary thread. + @Override + public void onTextureFrameAvailable( + int oesTextureId, float[] transformMatrix, long timestampNs) { + synchronized (newFrameLock) { + if (renderedBuffer != null) { + Logging.e(TAG, + "Unexpected onTextureFrameAvailable() called while already holding a texture."); + throw new IllegalStateException("Already holding a texture."); + } + // |timestampNs| is always zero on some Android versions. + renderedBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix, + bufferToRender.timeStampMs, bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs, + SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs); + bufferToRender = null; + newFrameLock.notifyAll(); + } + } + + // Dequeues and returns a DecodedTextureBuffer if available, or null otherwise. + public DecodedTextureBuffer dequeueTextureBuffer(int timeoutMs) { + synchronized (newFrameLock) { + if (renderedBuffer == null && timeoutMs > 0 && isWaitingForTexture()) { + try { + newFrameLock.wait(timeoutMs); + } catch(InterruptedException e) { + // Restore the interrupted status by reinterrupting the thread. + Thread.currentThread().interrupt(); + } + } + DecodedTextureBuffer returnedBuffer = renderedBuffer; + renderedBuffer = null; + return returnedBuffer; + } + } + + public void release() { + // SurfaceTextureHelper.disconnect() will block until any onTextureFrameAvailable() in + // progress is done. Therefore, the call to disconnect() must be outside any synchronized + // statement that is also used in the onTextureFrameAvailable() above to avoid deadlocks. + surfaceTextureHelper.disconnect(); + synchronized (newFrameLock) { + if (renderedBuffer != null) { + surfaceTextureHelper.returnTextureFrame(); + renderedBuffer = null; + } + } + } + } + + // Returns null if no decoded buffer is available, and otherwise a DecodedByteBuffer. // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException // upon codec error. - private Object dequeueOutputBuffer(int dequeueTimeoutUs) - throws IllegalStateException, MediaCodec.CodecException { + private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { checkOnMediaCodecThread(); + if (decodeStartTimeMs.isEmpty()) { + return null; + } // Drain the decoder until receiving a decoded buffer or hitting // MediaCodec.INFO_TRY_AGAIN_LATER. final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); while (true) { - final int result = mediaCodec.dequeueOutputBuffer(info, dequeueTimeoutUs); + final int result = mediaCodec.dequeueOutputBuffer( + info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); switch (result) { - case MediaCodec.INFO_TRY_AGAIN_LATER: - return null; case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: outputBuffers = mediaCodec.getOutputBuffers(); Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.length); + if (hasDecodedFirstFrame) { + throw new RuntimeException("Unexpected output buffer change event."); + } break; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: MediaFormat format = mediaCodec.getOutputFormat(); Logging.d(TAG, "Decoder format changed: " + format.toString()); + int new_width = format.getInteger(MediaFormat.KEY_WIDTH); + int new_height = format.getInteger(MediaFormat.KEY_HEIGHT); + if (hasDecodedFirstFrame && (new_width != width || new_height != height)) { + throw new RuntimeException("Unexpected size change. Configured " + width + "*" + + height + ". New " + new_width + "*" + new_height); + } width = format.getInteger(MediaFormat.KEY_WIDTH); height = format.getInteger(MediaFormat.KEY_HEIGHT); + if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); @@ -388,18 +564,76 @@ public class MediaCodecVideoDecoder { stride = Math.max(width, stride); sliceHeight = Math.max(height, sliceHeight); break; + case MediaCodec.INFO_TRY_AGAIN_LATER: + return null; default: - // Output buffer decoded. - if (useSurface) { - mediaCodec.releaseOutputBuffer(result, true /* render */); - // TODO(magjed): Wait for SurfaceTexture.onFrameAvailable() before returning a texture - // frame. - return new DecodedTextureBuffer(textureID, info.presentationTimeUs); - } else { - return new DecodedByteBuffer(result, info.offset, info.size, info.presentationTimeUs); - } + hasDecodedFirstFrame = true; + TimeStamps timeStamps = decodeStartTimeMs.remove(); + return new DecodedOutputBuffer(result, info.offset, info.size, timeStamps.timeStampMs, + timeStamps.ntpTimeStampMs, + SystemClock.elapsedRealtime() - timeStamps.decodeStartTimeMs, + SystemClock.elapsedRealtime()); + } + } + } + + // Returns null if no decoded buffer is available, and otherwise a DecodedTextureBuffer. + // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an + // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException + // upon codec error. If |dequeueTimeoutMs| > 0, the oldest decoded frame will be dropped if + // a frame can't be returned. + private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { + checkOnMediaCodecThread(); + if (!useSurface) { + throw new IllegalStateException("dequeueTexture() called for byte buffer decoding."); + } + DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); + if (outputBuffer != null) { + dequeuedSurfaceOutputBuffers.add(outputBuffer); + } + + MaybeRenderDecodedTextureBuffer(); + // Check if there is texture ready now by waiting max |dequeueTimeoutMs|. + DecodedTextureBuffer renderedBuffer = textureListener.dequeueTextureBuffer(dequeueTimeoutMs); + if (renderedBuffer != null) { + MaybeRenderDecodedTextureBuffer(); + return renderedBuffer; + } + + if ((dequeuedSurfaceOutputBuffers.size() + >= Math.min(MAX_QUEUED_OUTPUTBUFFERS, outputBuffers.length) + || (dequeueTimeoutMs > 0 && !dequeuedSurfaceOutputBuffers.isEmpty()))) { + ++droppedFrames; + // Drop the oldest frame still in dequeuedSurfaceOutputBuffers. + // The oldest frame is owned by |textureListener| and can't be dropped since + // mediaCodec.releaseOutputBuffer has already been called. + final DecodedOutputBuffer droppedFrame = dequeuedSurfaceOutputBuffers.remove(); + if (dequeueTimeoutMs > 0) { + // TODO(perkj): Re-add the below log when VideoRenderGUI has been removed or fixed to + // return the one and only texture even if it does not render. + // Logging.w(TAG, "Draining decoder. Dropping frame with TS: " + // + droppedFrame.timeStampMs + ". Total number of dropped frames: " + droppedFrames); + } else { + Logging.w(TAG, "Too many output buffers. Dropping frame with TS: " + + droppedFrame.timeStampMs + ". Total number of dropped frames: " + droppedFrames); } + + mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */); + return new DecodedTextureBuffer(0, null, droppedFrame.timeStampMs, + droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs, + SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs); + } + return null; + } + + private void MaybeRenderDecodedTextureBuffer() { + if (dequeuedSurfaceOutputBuffers.isEmpty() || textureListener.isWaitingForTexture()) { + return; } + // Get the first frame in the queue and render to the decoder output surface. + final DecodedOutputBuffer buffer = dequeuedSurfaceOutputBuffers.remove(); + textureListener.addBufferToRender(buffer); + mediaCodec.releaseOutputBuffer(buffer.index, true /* render */); } // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for @@ -407,11 +641,11 @@ public class MediaCodecVideoDecoder { // Throws IllegalStateException if the call is made on the wrong thread, if codec is configured // for surface decoding, or if |mediaCodec| is not in the Executing state. Throws // MediaCodec.CodecException upon codec error. - private void returnDecodedByteBuffer(int index) + private void returnDecodedOutputBuffer(int index) throws IllegalStateException, MediaCodec.CodecException { checkOnMediaCodecThread(); if (useSurface) { - throw new IllegalStateException("returnDecodedByteBuffer() called for surface decoding."); + throw new IllegalStateException("returnDecodedOutputBuffer() called for surface decoding."); } mediaCodec.releaseOutputBuffer(index, false /* render */); } diff --git a/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoEncoder.java b/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoEncoder.java index f3f03c1d20..5c8f9dc77e 100644 --- a/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoEncoder.java +++ b/talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoEncoder.java @@ -27,24 +27,29 @@ package org.webrtc; +import android.annotation.TargetApi; import android.media.MediaCodec; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.MediaFormat; +import android.opengl.GLES20; import android.os.Build; import android.os.Bundle; +import android.view.Surface; import org.webrtc.Logging; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; // Java-side of peerconnection_jni.cc:MediaCodecVideoEncoder. // This class is an implementation detail of the Java PeerConnection API. -// MediaCodec is thread-hostile so this class must be operated on a single -// thread. +@TargetApi(19) +@SuppressWarnings("deprecation") public class MediaCodecVideoEncoder { // This class is constructed, operated, and destroyed by its C++ incarnation, // so the class and its methods have non-public visibility. The API this @@ -60,18 +65,31 @@ public class MediaCodecVideoEncoder { VIDEO_CODEC_H264 } + private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; // Timeout for codec releasing. private static final int DEQUEUE_TIMEOUT = 0; // Non-blocking, no wait. - // Active running encoder instance. Set in initDecode() (called from native code) + // Active running encoder instance. Set in initEncode() (called from native code) // and reset to null in release() call. private static MediaCodecVideoEncoder runningInstance = null; + private static MediaCodecVideoEncoderErrorCallback errorCallback = null; + private static int codecErrors = 0; + private Thread mediaCodecThread; private MediaCodec mediaCodec; private ByteBuffer[] outputBuffers; + private EglBase14 eglBase; + private int width; + private int height; + private Surface inputSurface; + private GlRectDrawer drawer; private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; + private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; private static final String H264_MIME_TYPE = "video/avc"; // List of supported HW VP8 codecs. private static final String[] supportedVp8HwCodecPrefixes = {"OMX.qcom.", "OMX.Intel." }; + // List of supported HW VP9 decoders. + private static final String[] supportedVp9HwCodecPrefixes = + {"OMX.qcom."}; // List of supported HW H.264 codecs. private static final String[] supportedH264HwCodecPrefixes = {"OMX.qcom." }; @@ -99,13 +117,25 @@ public class MediaCodecVideoEncoder { CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m }; - private int colorFormat; - // Video encoder type. + private static final int[] supportedSurfaceColorList = { + CodecCapabilities.COLOR_FormatSurface + }; private VideoCodecType type; + private int colorFormat; // Used by native code. + // SPS and PPS NALs (Config frame) for H.264. private ByteBuffer configData = null; - private MediaCodecVideoEncoder() { + // MediaCodec error handler - invoked when critical error happens which may prevent + // further use of media codec API. Now it means that one of media codec instances + // is hanging and can no longer be used in the next call. + public static interface MediaCodecVideoEncoderErrorCallback { + void onMediaCodecVideoEncoderCriticalError(int codecErrors); + } + + public static void setErrorCallback(MediaCodecVideoEncoderErrorCallback errorCallback) { + Logging.d(TAG, "Set error callback"); + MediaCodecVideoEncoder.errorCallback = errorCallback; } // Helper struct for findHwEncoder() below. @@ -119,7 +149,7 @@ public class MediaCodecVideoEncoder { } private static EncoderProperties findHwEncoder( - String mime, String[] supportedHwCodecPrefixes) { + String mime, String[] supportedHwCodecPrefixes, int[] colorList) { // MediaCodec.setParameters is missing for JB and below, so bitrate // can not be adjusted dynamically. if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { @@ -130,8 +160,7 @@ public class MediaCodecVideoEncoder { if (mime.equals(H264_MIME_TYPE)) { List<String> exceptionModels = Arrays.asList(H264_HW_EXCEPTION_MODELS); if (exceptionModels.contains(Build.MODEL)) { - Logging.w(TAG, "Model: " + Build.MODEL + - " has black listed H.264 encoder."); + Logging.w(TAG, "Model: " + Build.MODEL + " has black listed H.264 encoder."); return null; } } @@ -170,8 +199,7 @@ public class MediaCodecVideoEncoder { Logging.v(TAG, " Color: 0x" + Integer.toHexString(colorFormat)); } - // Check if codec supports either yuv420 or nv12. - for (int supportedColorFormat : supportedColorList) { + for (int supportedColorFormat : colorList) { for (int codecColorFormat : capabilities.colorFormats) { if (codecColorFormat == supportedColorFormat) { // Found supported HW encoder. @@ -182,15 +210,34 @@ public class MediaCodecVideoEncoder { } } } - return null; // No HW VP8 encoder. + return null; // No HW encoder. } public static boolean isVp8HwSupported() { - return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null; + return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, supportedColorList) != null; + } + + public static boolean isVp9HwSupported() { + return findHwEncoder(VP9_MIME_TYPE, supportedVp9HwCodecPrefixes, supportedColorList) != null; } public static boolean isH264HwSupported() { - return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null; + return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, supportedColorList) != null; + } + + public static boolean isVp8HwSupportedUsingTextures() { + return findHwEncoder( + VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, supportedSurfaceColorList) != null; + } + + public static boolean isVp9HwSupportedUsingTextures() { + return findHwEncoder( + VP9_MIME_TYPE, supportedVp9HwCodecPrefixes, supportedSurfaceColorList) != null; + } + + public static boolean isH264HwSupportedUsingTextures() { + return findHwEncoder( + H264_MIME_TYPE, supportedH264HwCodecPrefixes, supportedSurfaceColorList) != null; } private void checkOnMediaCodecThread() { @@ -223,32 +270,43 @@ public class MediaCodecVideoEncoder { } } - // Return the array of input buffers, or null on failure. - private ByteBuffer[] initEncode( - VideoCodecType type, int width, int height, int kbps, int fps) { + boolean initEncode(VideoCodecType type, int width, int height, int kbps, int fps, + EglBase14.Context sharedContext) { + final boolean useSurface = sharedContext != null; Logging.d(TAG, "Java initEncode: " + type + " : " + width + " x " + height + - ". @ " + kbps + " kbps. Fps: " + fps + - ". Color: 0x" + Integer.toHexString(colorFormat)); + ". @ " + kbps + " kbps. Fps: " + fps + ". Encode from texture : " + useSurface); + + this.width = width; + this.height = height; if (mediaCodecThread != null) { throw new RuntimeException("Forgot to release()?"); } - this.type = type; EncoderProperties properties = null; String mime = null; int keyFrameIntervalSec = 0; if (type == VideoCodecType.VIDEO_CODEC_VP8) { mime = VP8_MIME_TYPE; - properties = findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes); + properties = findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, + useSurface ? supportedSurfaceColorList : supportedColorList); + keyFrameIntervalSec = 100; + } else if (type == VideoCodecType.VIDEO_CODEC_VP9) { + mime = VP9_MIME_TYPE; + properties = findHwEncoder(VP9_MIME_TYPE, supportedH264HwCodecPrefixes, + useSurface ? supportedSurfaceColorList : supportedColorList); keyFrameIntervalSec = 100; } else if (type == VideoCodecType.VIDEO_CODEC_H264) { mime = H264_MIME_TYPE; - properties = findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes); + properties = findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, + useSurface ? supportedSurfaceColorList : supportedColorList); keyFrameIntervalSec = 20; } if (properties == null) { throw new RuntimeException("Can not find HW encoder for " + type); } runningInstance = this; // Encoder is now running and can be queried for stack traces. + colorFormat = properties.colorFormat; + Logging.d(TAG, "Color format: " + colorFormat); + mediaCodecThread = Thread.currentThread(); try { MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); @@ -259,26 +317,39 @@ public class MediaCodecVideoEncoder { format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec); Logging.d(TAG, " Format: " + format); mediaCodec = createByCodecName(properties.codecName); + this.type = type; if (mediaCodec == null) { Logging.e(TAG, "Can not create media encoder"); - return null; + return false; } mediaCodec.configure( format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + + if (useSurface) { + eglBase = new EglBase14(sharedContext, EglBase.CONFIG_RECORDABLE); + // Create an input surface and keep a reference since we must release the surface when done. + inputSurface = mediaCodec.createInputSurface(); + eglBase.createSurface(inputSurface); + drawer = new GlRectDrawer(); + } mediaCodec.start(); - colorFormat = properties.colorFormat; outputBuffers = mediaCodec.getOutputBuffers(); - ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers(); - Logging.d(TAG, "Input buffers: " + inputBuffers.length + - ". Output buffers: " + outputBuffers.length); - return inputBuffers; + Logging.d(TAG, "Output buffers: " + outputBuffers.length); + } catch (IllegalStateException e) { Logging.e(TAG, "initEncode failed", e); - return null; + return false; } + return true; + } + + ByteBuffer[] getInputBuffers() { + ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers(); + Logging.d(TAG, "Input buffers: " + inputBuffers.length); + return inputBuffers; } - private boolean encode( + boolean encodeBuffer( boolean isKeyframe, int inputBuffer, int size, long presentationTimestampUs) { checkOnMediaCodecThread(); @@ -298,22 +369,82 @@ public class MediaCodecVideoEncoder { return true; } catch (IllegalStateException e) { - Logging.e(TAG, "encode failed", e); + Logging.e(TAG, "encodeBuffer failed", e); return false; } } - private void release() { - Logging.d(TAG, "Java releaseEncoder"); + boolean encodeTexture(boolean isKeyframe, int oesTextureId, float[] transformationMatrix, + long presentationTimestampUs) { checkOnMediaCodecThread(); try { - mediaCodec.stop(); - mediaCodec.release(); - } catch (IllegalStateException e) { - Logging.e(TAG, "release failed", e); + if (isKeyframe) { + Logging.d(TAG, "Sync frame request"); + Bundle b = new Bundle(); + b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + mediaCodec.setParameters(b); + } + eglBase.makeCurrent(); + // TODO(perkj): glClear() shouldn't be necessary since every pixel is covered anyway, + // but it's a workaround for bug webrtc:5147. + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + drawer.drawOes(oesTextureId, transformationMatrix, 0, 0, width, height); + eglBase.swapBuffers(TimeUnit.MICROSECONDS.toNanos(presentationTimestampUs)); + return true; + } + catch (RuntimeException e) { + Logging.e(TAG, "encodeTexture failed", e); + return false; } + } + + void release() { + Logging.d(TAG, "Java releaseEncoder"); + checkOnMediaCodecThread(); + + // Run Mediacodec stop() and release() on separate thread since sometime + // Mediacodec.stop() may hang. + final CountDownLatch releaseDone = new CountDownLatch(1); + + Runnable runMediaCodecRelease = new Runnable() { + @Override + public void run() { + try { + Logging.d(TAG, "Java releaseEncoder on release thread"); + mediaCodec.stop(); + mediaCodec.release(); + Logging.d(TAG, "Java releaseEncoder on release thread done"); + } catch (Exception e) { + Logging.e(TAG, "Media encoder release failed", e); + } + releaseDone.countDown(); + } + }; + new Thread(runMediaCodecRelease).start(); + + if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { + Logging.e(TAG, "Media encoder release timeout"); + codecErrors++; + if (errorCallback != null) { + Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); + errorCallback.onMediaCodecVideoEncoderCriticalError(codecErrors); + } + } + mediaCodec = null; mediaCodecThread = null; + if (drawer != null) { + drawer.release(); + drawer = null; + } + if (eglBase != null) { + eglBase.release(); + eglBase = null; + } + if (inputSurface != null) { + inputSurface.release(); + inputSurface = null; + } runningInstance = null; Logging.d(TAG, "Java releaseEncoder done"); } @@ -336,7 +467,7 @@ public class MediaCodecVideoEncoder { // Dequeue an input buffer and return its index, -1 if no input buffer is // available, or -2 if the codec is no longer operative. - private int dequeueInputBuffer() { + int dequeueInputBuffer() { checkOnMediaCodecThread(); try { return mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT); @@ -347,7 +478,7 @@ public class MediaCodecVideoEncoder { } // Helper struct for dequeueOutputBuffer() below. - private static class OutputBufferInfo { + static class OutputBufferInfo { public OutputBufferInfo( int index, ByteBuffer buffer, boolean isKeyFrame, long presentationTimestampUs) { @@ -357,15 +488,15 @@ public class MediaCodecVideoEncoder { this.presentationTimestampUs = presentationTimestampUs; } - private final int index; - private final ByteBuffer buffer; - private final boolean isKeyFrame; - private final long presentationTimestampUs; + public final int index; + public final ByteBuffer buffer; + public final boolean isKeyFrame; + public final long presentationTimestampUs; } // Dequeue and return an output buffer, or null if no output is ready. Return // a fake OutputBufferInfo with index -1 if the codec is no longer operable. - private OutputBufferInfo dequeueOutputBuffer() { + OutputBufferInfo dequeueOutputBuffer() { checkOnMediaCodecThread(); try { MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); @@ -434,7 +565,7 @@ public class MediaCodecVideoEncoder { // Release a dequeued output buffer back to the codec for re-use. Return // false if the codec is no longer operable. - private boolean releaseOutputBuffer(int index) { + boolean releaseOutputBuffer(int index) { checkOnMediaCodecThread(); try { mediaCodec.releaseOutputBuffer(index, false); diff --git a/talk/app/webrtc/java/src/org/webrtc/PeerConnection.java b/talk/app/webrtc/java/src/org/webrtc/PeerConnection.java index 50023001d7..36cd07595c 100644 --- a/talk/app/webrtc/java/src/org/webrtc/PeerConnection.java +++ b/talk/app/webrtc/java/src/org/webrtc/PeerConnection.java @@ -28,7 +28,6 @@ package org.webrtc; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -151,6 +150,7 @@ public class PeerConnection { public int audioJitterBufferMaxPackets; public boolean audioJitterBufferFastAccelerate; public int iceConnectionReceivingTimeout; + public int iceBackupCandidatePairPingInterval; public KeyType keyType; public ContinualGatheringPolicy continualGatheringPolicy; @@ -163,6 +163,7 @@ public class PeerConnection { audioJitterBufferMaxPackets = 50; audioJitterBufferFastAccelerate = false; iceConnectionReceivingTimeout = -1; + iceBackupCandidatePairPingInterval = -1; keyType = KeyType.ECDSA; continualGatheringPolicy = ContinualGatheringPolicy.GATHER_ONCE; } @@ -223,6 +224,14 @@ public class PeerConnection { localStreams.remove(stream); } + public RtpSender createSender(String kind, String stream_id) { + RtpSender new_sender = nativeCreateSender(kind, stream_id); + if (new_sender != null) { + senders.add(new_sender); + } + return new_sender; + } + // Note that calling getSenders will dispose of the senders previously // returned (and same goes for getReceivers). public List<RtpSender> getSenders() { @@ -288,6 +297,8 @@ public class PeerConnection { private native boolean nativeGetStats( StatsObserver observer, long nativeTrack); + private native RtpSender nativeCreateSender(String kind, String stream_id); + private native List<RtpSender> nativeGetSenders(); private native List<RtpReceiver> nativeGetReceivers(); diff --git a/talk/app/webrtc/java/src/org/webrtc/PeerConnectionFactory.java b/talk/app/webrtc/java/src/org/webrtc/PeerConnectionFactory.java index 83999ece98..d759c69271 100644 --- a/talk/app/webrtc/java/src/org/webrtc/PeerConnectionFactory.java +++ b/talk/app/webrtc/java/src/org/webrtc/PeerConnectionFactory.java @@ -73,6 +73,15 @@ public class PeerConnectionFactory { // Field trial initialization. Must be called before PeerConnectionFactory // is created. public static native void initializeFieldTrials(String fieldTrialsInitString); + // Internal tracing initialization. Must be called before PeerConnectionFactory is created to + // prevent racing with tracing code. + public static native void initializeInternalTracer(); + // Internal tracing shutdown, called to prevent resource leaks. Must be called after + // PeerConnectionFactory is gone to prevent races with code performing tracing. + public static native void shutdownInternalTracer(); + // Start/stop internal capturing of internal tracing. + public static native boolean startInternalTracingCapture(String tracing_filename); + public static native void stopInternalTracingCapture(); public PeerConnectionFactory() { nativeFactory = nativeCreatePeerConnectionFactory(); @@ -131,12 +140,52 @@ public class PeerConnectionFactory { nativeFactory, id, source.nativeSource)); } + // Starts recording an AEC dump. Ownership of the file is transfered to the + // native code. If an AEC dump is already in progress, it will be stopped and + // a new one will start using the provided file. + public boolean startAecDump(int file_descriptor) { + return nativeStartAecDump(nativeFactory, file_descriptor); + } + + // Stops recording an AEC dump. If no AEC dump is currently being recorded, + // this call will have no effect. + public void stopAecDump() { + nativeStopAecDump(nativeFactory); + } + + // Starts recording an RTC event log. Ownership of the file is transfered to + // the native code. If an RTC event log is already being recorded, it will be + // stopped and a new one will start using the provided file. + public boolean startRtcEventLog(int file_descriptor) { + return nativeStartRtcEventLog(nativeFactory, file_descriptor); + } + + // Stops recording an RTC event log. If no RTC event log is currently being + // recorded, this call will have no effect. + public void StopRtcEventLog() { + nativeStopRtcEventLog(nativeFactory); + } + public void setOptions(Options options) { nativeSetOptions(nativeFactory, options); } + @Deprecated public void setVideoHwAccelerationOptions(Object renderEGLContext) { - nativeSetVideoHwAccelerationOptions(nativeFactory, renderEGLContext); + nativeSetVideoHwAccelerationOptions(nativeFactory, renderEGLContext, renderEGLContext); + } + + /** Set the EGL context used by HW Video encoding and decoding. + * + * + * @param localEGLContext An instance of javax.microedition.khronos.egl.EGLContext. + * Must be the same as used by VideoCapturerAndroid and any local + * video renderer. + * @param remoteEGLContext An instance of javax.microedition.khronos.egl.EGLContext. + * Must be the same as used by any remote video renderer. + */ + public void setVideoHwAccelerationOptions(Object localEGLContext, Object remoteEGLContext) { + nativeSetVideoHwAccelerationOptions(nativeFactory, localEGLContext, remoteEGLContext); } public void dispose() { @@ -201,10 +250,18 @@ public class PeerConnectionFactory { private static native long nativeCreateAudioTrack( long nativeFactory, String id, long nativeSource); + private static native boolean nativeStartAecDump(long nativeFactory, int file_descriptor); + + private static native void nativeStopAecDump(long nativeFactory); + + private static native boolean nativeStartRtcEventLog(long nativeFactory, int file_descriptor); + + private static native void nativeStopRtcEventLog(long nativeFactory); + public native void nativeSetOptions(long nativeFactory, Options options); private static native void nativeSetVideoHwAccelerationOptions( - long nativeFactory, Object renderEGLContext); + long nativeFactory, Object localEGLContext, Object remoteEGLContext); private static native void nativeThreadsCallbacks(long nativeFactory); diff --git a/talk/app/webrtc/java/src/org/webrtc/RtpSender.java b/talk/app/webrtc/java/src/org/webrtc/RtpSender.java index 37357c0657..9ac2e7034f 100644 --- a/talk/app/webrtc/java/src/org/webrtc/RtpSender.java +++ b/talk/app/webrtc/java/src/org/webrtc/RtpSender.java @@ -32,6 +32,7 @@ public class RtpSender { final long nativeRtpSender; private MediaStreamTrack cachedTrack; + private boolean ownsTrack = true; public RtpSender(long nativeRtpSender) { this.nativeRtpSender = nativeRtpSender; @@ -40,14 +41,22 @@ public class RtpSender { cachedTrack = (track == 0) ? null : new MediaStreamTrack(track); } - // NOTE: This should not be called with a track that's already used by - // another RtpSender, because then it would be double-disposed. - public void setTrack(MediaStreamTrack track) { - if (cachedTrack != null) { + // If |takeOwnership| is true, the RtpSender takes ownership of the track + // from the caller, and will auto-dispose of it when no longer needed. + // |takeOwnership| should only be used if the caller owns the track; it is + // not appropriate when the track is owned by, for example, another RtpSender + // or a MediaStream. + public boolean setTrack(MediaStreamTrack track, boolean takeOwnership) { + if (!nativeSetTrack(nativeRtpSender, + (track == null) ? 0 : track.nativeTrack)) { + return false; + } + if (cachedTrack != null && ownsTrack) { cachedTrack.dispose(); } cachedTrack = track; - nativeSetTrack(nativeRtpSender, (track == null) ? 0 : track.nativeTrack); + ownsTrack = takeOwnership; + return true; } public MediaStreamTrack track() { @@ -59,14 +68,14 @@ public class RtpSender { } public void dispose() { - if (cachedTrack != null) { + if (cachedTrack != null && ownsTrack) { cachedTrack.dispose(); } free(nativeRtpSender); } - private static native void nativeSetTrack(long nativeRtpSender, - long nativeTrack); + private static native boolean nativeSetTrack(long nativeRtpSender, + long nativeTrack); // This should increment the reference count of the track. // Will be released in dispose() or setTrack(). diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java index 3c255dd123..2e307fc54b 100644 --- a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java +++ b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java @@ -46,7 +46,11 @@ public class VideoRenderer { public final int[] yuvStrides; public ByteBuffer[] yuvPlanes; public final boolean yuvFrame; - public Object textureObject; + // Matrix that transforms standard coordinates to their proper sampling locations in + // the texture. This transform compensates for any properties of the video source that + // cause it to appear different from a normalized texture. This matrix does not take + // |rotationDegree| into account. + public final float[] samplingMatrix; public int textureId; // Frame pointer in C++. private long nativeFramePointer; @@ -70,19 +74,27 @@ public class VideoRenderer { if (rotationDegree % 90 != 0) { throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree); } + // The convention in WebRTC is that the first element in a ByteBuffer corresponds to the + // top-left corner of the image, but in glTexImage2D() the first element corresponds to the + // bottom-left corner. This discrepancy is corrected by setting a vertical flip as sampling + // matrix. + samplingMatrix = new float[] { + 1, 0, 0, 0, + 0, -1, 0, 0, + 0, 0, 1, 0, + 0, 1, 0, 1}; } /** * Construct a texture frame of the given dimensions with data in SurfaceTexture */ - I420Frame( - int width, int height, int rotationDegree, - Object textureObject, int textureId, long nativeFramePointer) { + I420Frame(int width, int height, int rotationDegree, int textureId, float[] samplingMatrix, + long nativeFramePointer) { this.width = width; this.height = height; this.yuvStrides = null; this.yuvPlanes = null; - this.textureObject = textureObject; + this.samplingMatrix = samplingMatrix; this.textureId = textureId; this.yuvFrame = false; this.rotationDegree = rotationDegree; @@ -125,7 +137,6 @@ public class VideoRenderer { */ public static void renderFrameDone(I420Frame frame) { frame.yuvPlanes = null; - frame.textureObject = null; frame.textureId = 0; if (frame.nativeFramePointer != 0) { releaseNativeFrame(frame.nativeFramePointer); |