diff options
author | Magnus Jedvert <magjed@webrtc.org> | 2015-09-03 12:40:38 +0200 |
---|---|---|
committer | Magnus Jedvert <magjed@webrtc.org> | 2015-09-03 10:40:50 +0000 |
commit | 7afc12fe91e97a3d68de3768a73f3604e5651504 (patch) | |
tree | 63d38ceefb83c5a0680e8079e0e22a7ba034c202 | |
parent | 4df08ff37459ee7c9d545c2574c1a84b5e968243 (diff) | |
download | webrtc-7afc12fe91e97a3d68de3768a73f3604e5651504.tar.gz |
VideoRendererGui: Move to async rendering and remove no longer needed code
BUG=webrtc:4742, webrtc:4909
R=glaznev@webrtc.org
Review URL: https://codereview.webrtc.org/1321853003 .
Cr-Commit-Position: refs/heads/master@{#9847}
4 files changed, 88 insertions, 156 deletions
diff --git a/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java b/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java index 08b82af1bc..71bd5e3561 100644 --- a/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java +++ b/talk/app/webrtc/androidtests/src/org/webrtc/GlRectDrawerTest.java @@ -86,7 +86,7 @@ public class GlRectDrawerTest extends ActivityTestCase { final GlRectDrawer drawer = new GlRectDrawer(); final float[] texMatrix = new float[16]; Matrix.setIdentityM(texMatrix, 0); - drawer.drawYuv(WIDTH, HEIGHT, yuvTextures, texMatrix); + drawer.drawYuv(yuvTextures, texMatrix); // Download the pixels in the pixel buffer as RGBA. final ByteBuffer data = ByteBuffer.allocateDirect(WIDTH * HEIGHT * 4); diff --git a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java index c14abb0219..6b3032121f 100644 --- a/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java +++ b/talk/app/webrtc/java/android/org/webrtc/GlRectDrawer.java @@ -120,6 +120,51 @@ public class GlRectDrawer { private GlShader currentShader; private float[] currentTexMatrix; private int texMatrixLocation; + // Intermediate copy buffer for uploading yuv frames that are not packed, i.e. stride > width. + // TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader that + // handles stride and compare performance with intermediate copy. + private ByteBuffer copyBuffer; + + /** + * Upload |planes| into |outputYuvTextures|, taking stride into consideration. |outputYuvTextures| + * must have been generated in advance. + */ + public void uploadYuvData( + int[] outputYuvTextures, int width, int height, int[] strides, ByteBuffer[] planes) { + // Make a first pass to see if we need a temporary copy buffer. + int copyCapacityNeeded = 0; + for (int i = 0; i < 3; ++i) { + final int planeWidth = (i == 0) ? width : width / 2; + final int planeHeight = (i == 0) ? height : height / 2; + if (strides[i] > planeWidth) { + copyCapacityNeeded = Math.max(copyCapacityNeeded, planeWidth * planeHeight); + } + } + // Allocate copy buffer if necessary. + if (copyCapacityNeeded > 0 + && (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) { + copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded); + } + // Upload each plane. + for (int i = 0; i < 3; ++i) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputYuvTextures[i]); + final int planeWidth = (i == 0) ? width : width / 2; + final int planeHeight = (i == 0) ? height : height / 2; + // GLES only accepts packed data, i.e. stride == planeWidth. + final ByteBuffer packedByteBuffer; + if (strides[i] == planeWidth) { + // Input is packed already. + packedByteBuffer = planes[i]; + } else { + VideoRenderer.nativeCopyPlane( + planes[i], planeWidth, planeHeight, strides[i], copyBuffer, planeWidth); + packedByteBuffer = copyBuffer; + } + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidth, planeHeight, 0, + GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); + } + } /** * Draw an OES texture frame with specified texture transformation matrix. Required resources are @@ -150,7 +195,7 @@ public class GlRectDrawer { * Draw a YUV frame with specified texture transformation matrix. Required resources are * allocated at the first call to this function. */ - public void drawYuv(int width, int height, int[] yuvTextures, float[] texMatrix) { + public void drawYuv(int[] yuvTextures, float[] texMatrix) { prepareShader(YUV_FRAGMENT_SHADER_STRING); // Bind the textures. for (int i = 0; i < 3; ++i) { @@ -219,5 +264,6 @@ public class GlRectDrawer { shader.release(); } shaders.clear(); + copyBuffer = null; } } diff --git a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java index 23da47bce3..85680073fd 100644 --- a/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java +++ b/talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java @@ -29,7 +29,6 @@ package org.webrtc; import java.util.ArrayList; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; @@ -102,14 +101,11 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private int[] yuvTextures = { 0, 0, 0 }; private int oesTexture = 0; - // Render frame queue - accessed by two threads. renderFrame() call does - // an offer (writing I420Frame to render) and early-returns (recording - // a dropped frame) if that queue is full. draw() call does a peek(), - // copies frame to texture and then removes it from a queue using poll(). - private final LinkedBlockingQueue<I420Frame> frameToRenderQueue; - // Local copy of incoming video frame. Synchronized on |frameToRenderQueue|. - private I420Frame yuvFrameToRender; - private I420Frame textureFrameToRender; + // Pending frame to render. Serves as a queue with size 1. |pendingFrame| is accessed by two + // threads - frames are received in renderFrame() and consumed in draw(). Frames are dropped in + // renderFrame() if the previous frame has not been rendered yet. + private I420Frame pendingFrame; + private final Object pendingFrameLock = new Object(); // Type of video frame used for recent frame rendering. private static enum RendererType { RENDERER_YUV, RENDERER_TEXTURE }; private RendererType rendererType; @@ -129,7 +125,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private long startTimeNs = -1; // Time in ns spent in draw() function. private long drawTimeNs; - // Time in ns spent in renderFrame() function - including copying frame + // Time in ns spent in draw() copying resources from |pendingFrame| - including uploading frame // data to rendering planes. private long copyTimeNs; // The allowed view area in percentage of screen size. @@ -163,7 +159,6 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { this.id = id; this.scalingType = scalingType; this.mirror = mirror; - frameToRenderQueue = new LinkedBlockingQueue<I420Frame>(1); layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height)); updateTextureProperties = false; rotationDegree = 0; @@ -171,10 +166,11 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { private synchronized void release() { surface = null; - synchronized (frameToRenderQueue) { - frameToRenderQueue.clear(); - yuvFrameToRender = null; - textureFrameToRender = null; + synchronized (pendingFrameLock) { + if (pendingFrame != null) { + VideoRenderer.renderFrameDone(pendingFrame); + pendingFrame = null; + } } } @@ -231,52 +227,47 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { GLES20.glViewport(displayLayout.left, screenHeight - displayLayout.bottom, displayLayout.width(), displayLayout.height()); - I420Frame frameFromQueue; - synchronized (frameToRenderQueue) { + final boolean isNewFrame; + synchronized (pendingFrameLock) { // Check if texture vertices/coordinates adjustment is required when // screen orientation changes or video frame size changes. checkAdjustTextureCoords(); - frameFromQueue = frameToRenderQueue.peek(); - if (frameFromQueue != null && startTimeNs == -1) { + isNewFrame = (pendingFrame != null); + if (isNewFrame && startTimeNs == -1) { startTimeNs = now; } - if (frameFromQueue != null) { - if (frameFromQueue.yuvFrame) { - // YUV textures rendering. Upload YUV data as textures. - for (int i = 0; i < 3; ++i) { - GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); - int w = (i == 0) ? frameFromQueue.width : frameFromQueue.width / 2; - int h = (i == 0) ? frameFromQueue.height : frameFromQueue.height / 2; - GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, - w, h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, - frameFromQueue.yuvPlanes[i]); - } + if (isNewFrame) { + if (pendingFrame.yuvFrame) { + rendererType = RendererType.RENDERER_YUV; + drawer.uploadYuvData(yuvTextures, pendingFrame.width, pendingFrame.height, + pendingFrame.yuvStrides, pendingFrame.yuvPlanes); } else { + rendererType = RendererType.RENDERER_TEXTURE; // External texture rendering. Copy texture id and update texture image to latest. // TODO(magjed): We should not make an unmanaged copy of texture id. Also, this is not // the best place to call updateTexImage. - oesTexture = frameFromQueue.textureId; - if (frameFromQueue.textureObject instanceof SurfaceTexture) { + oesTexture = pendingFrame.textureId; + if (pendingFrame.textureObject instanceof SurfaceTexture) { SurfaceTexture surfaceTexture = - (SurfaceTexture) frameFromQueue.textureObject; + (SurfaceTexture) pendingFrame.textureObject; surfaceTexture.updateTexImage(); } } - - frameToRenderQueue.poll(); + copyTimeNs += (System.nanoTime() - now); + VideoRenderer.renderFrameDone(pendingFrame); + pendingFrame = null; } } if (rendererType == RendererType.RENDERER_YUV) { - drawer.drawYuv(videoWidth, videoHeight, yuvTextures, texMatrix); + drawer.drawYuv(yuvTextures, texMatrix); } else { drawer.drawOes(oesTexture, texMatrix); } - if (frameFromQueue != null) { + if (isNewFrame) { framesRendered++; drawTimeNs += (System.nanoTime() - now); if ((framesRendered % 300) == 0) { @@ -342,25 +333,13 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation); } - // Frame re-allocation need to be synchronized with copying - // frame to textures in draw() function to avoid re-allocating - // the frame while it is being copied. - synchronized (frameToRenderQueue) { + synchronized (updateTextureLock) { Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " + videoWidth + " x " + videoHeight + " rotation " + rotation); this.videoWidth = videoWidth; this.videoHeight = videoHeight; rotationDegree = rotation; - int[] strides = { videoWidth, videoWidth / 2, videoWidth / 2 }; - - // Clear rendering queue. - frameToRenderQueue.poll(); - // Re-allocate / allocate the frame. - yuvFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree, - strides, null, 0); - textureFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree, - null, -1, 0); updateTextureProperties = true; Log.d(TAG, " YuvImageRenderer.setSize done."); } @@ -377,16 +356,8 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { Log.d(TAG, "ID: " + id + ". Reporting first rendered frame."); rendererEvents.onFirstFrameRendered(); } - setSize(frame.width, frame.height, frame.rotationDegree); - long now = System.nanoTime(); framesReceived++; - synchronized (frameToRenderQueue) { - // Skip rendering of this frame if setSize() was not called. - if (yuvFrameToRender == null || textureFrameToRender == null) { - framesDropped++; - VideoRenderer.renderFrameDone(frame); - return; - } + synchronized (pendingFrameLock) { // Check input frame parameters. if (frame.yuvFrame) { if (frame.yuvStrides[0] < frame.width || @@ -397,35 +368,18 @@ public class VideoRendererGui implements GLSurfaceView.Renderer { VideoRenderer.renderFrameDone(frame); return; } - // Check incoming frame dimensions. - if (frame.width != yuvFrameToRender.width || - frame.height != yuvFrameToRender.height) { - throw new RuntimeException("Wrong frame size " + - frame.width + " x " + frame.height); - } } - if (frameToRenderQueue.size() > 0) { + if (pendingFrame != null) { // Skip rendering of this frame if previous frame was not rendered yet. framesDropped++; VideoRenderer.renderFrameDone(frame); return; } - - // Create a local copy of the frame. - if (frame.yuvFrame) { - yuvFrameToRender.copyFrom(frame); - rendererType = RendererType.RENDERER_YUV; - frameToRenderQueue.offer(yuvFrameToRender); - } else { - textureFrameToRender.copyFrom(frame); - rendererType = RendererType.RENDERER_TEXTURE; - frameToRenderQueue.offer(textureFrameToRender); - } + pendingFrame = frame; } - copyTimeNs += (System.nanoTime() - now); + setSize(frame.width, frame.height, frame.rotationDegree); seenFrame = true; - VideoRenderer.renderFrameDone(frame); // Request rendering. surface.requestRender(); diff --git a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java index a7b88ba939..afda84897c 100644 --- a/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java +++ b/talk/app/webrtc/java/src/org/webrtc/VideoRenderer.java @@ -37,7 +37,7 @@ import java.nio.ByteBuffer; */ public class VideoRenderer { - /** Java version of cricket::VideoFrame. */ + /** Java version of cricket::VideoFrame. Frames are only constructed from native code. */ public static class I420Frame { public final int width; public final int height; @@ -46,7 +46,7 @@ public class VideoRenderer { public final boolean yuvFrame; public Object textureObject; public int textureId; - // If |nativeFramePointer| is non-zero, the memory is allocated on the C++ side. + // Frame pointer in C++. private long nativeFramePointer; // rotationDegree is the degree that the frame must be rotated clockwisely @@ -54,22 +54,14 @@ public class VideoRenderer { public int rotationDegree; /** - * Construct a frame of the given dimensions with the specified planar - * data. If |yuvPlanes| is null, new planes of the appropriate sizes are - * allocated. + * Construct a frame of the given dimensions with the specified planar data. */ - public I420Frame( + private I420Frame( int width, int height, int rotationDegree, int[] yuvStrides, ByteBuffer[] yuvPlanes, long nativeFramePointer) { this.width = width; this.height = height; this.yuvStrides = yuvStrides; - if (yuvPlanes == null) { - yuvPlanes = new ByteBuffer[3]; - yuvPlanes[0] = ByteBuffer.allocateDirect(yuvStrides[0] * height); - yuvPlanes[1] = ByteBuffer.allocateDirect(yuvStrides[1] * height / 2); - yuvPlanes[2] = ByteBuffer.allocateDirect(yuvStrides[2] * height / 2); - } this.yuvPlanes = yuvPlanes; this.yuvFrame = true; this.rotationDegree = rotationDegree; @@ -82,7 +74,7 @@ public class VideoRenderer { /** * Construct a texture frame of the given dimensions with data in SurfaceTexture */ - public I420Frame( + private I420Frame( int width, int height, int rotationDegree, Object textureObject, int textureId, long nativeFramePointer) { this.width = width; @@ -107,66 +99,6 @@ public class VideoRenderer { return (rotationDegree % 180 == 0) ? height : width; } - /** - * Copy the planes out of |source| into |this| and return |this|. Calling - * this with mismatched frame dimensions or frame type is a programming - * error and will likely crash. - */ - public I420Frame copyFrom(I420Frame source) { - // |nativeFramePointer| is not copied from |source|, because resources in this object are - // still allocated in Java. After copyFrom() is done, this object should not hold any - // references to |source|. This is violated for texture frames however, because |textureId| - // is copied without making a deep copy. - if (this.nativeFramePointer != 0) { - throw new RuntimeException("Trying to overwrite a frame allocated in C++"); - } - if (source.yuvFrame && yuvFrame) { - if (width != source.width || height != source.height) { - throw new RuntimeException("Mismatched dimensions! Source: " + - source.toString() + ", destination: " + toString()); - } - nativeCopyPlane(source.yuvPlanes[0], width, height, - source.yuvStrides[0], yuvPlanes[0], yuvStrides[0]); - nativeCopyPlane(source.yuvPlanes[1], width / 2, height / 2, - source.yuvStrides[1], yuvPlanes[1], yuvStrides[1]); - nativeCopyPlane(source.yuvPlanes[2], width / 2, height / 2, - source.yuvStrides[2], yuvPlanes[2], yuvStrides[2]); - rotationDegree = source.rotationDegree; - return this; - } else if (!source.yuvFrame && !yuvFrame) { - textureObject = source.textureObject; - textureId = source.textureId; - rotationDegree = source.rotationDegree; - return this; - } else { - throw new RuntimeException("Mismatched frame types! Source: " + - source.toString() + ", destination: " + toString()); - } - } - - public I420Frame copyFrom(byte[] yuvData, int rotationDegree) { - if (yuvData.length < width * height * 3 / 2) { - throw new RuntimeException("Wrong arrays size: " + yuvData.length); - } - if (!yuvFrame) { - throw new RuntimeException("Can not feed yuv data to texture frame"); - } - int planeSize = width * height; - ByteBuffer[] planes = new ByteBuffer[3]; - planes[0] = ByteBuffer.wrap(yuvData, 0, planeSize); - planes[1] = ByteBuffer.wrap(yuvData, planeSize, planeSize / 4); - planes[2] = ByteBuffer.wrap(yuvData, planeSize + planeSize / 4, - planeSize / 4); - for (int i = 0; i < 3; i++) { - yuvPlanes[i].position(0); - yuvPlanes[i].put(planes[i]); - yuvPlanes[i].position(0); - yuvPlanes[i].limit(yuvPlanes[i].capacity()); - } - this.rotationDegree = rotationDegree; - return this; - } - @Override public String toString() { return width + "x" + height + ":" + yuvStrides[0] + ":" + yuvStrides[1] + @@ -175,7 +107,7 @@ public class VideoRenderer { } // Helper native function to do a video frame plane copying. - private static native void nativeCopyPlane(ByteBuffer src, int width, + public static native void nativeCopyPlane(ByteBuffer src, int width, int height, int srcStride, ByteBuffer dst, int dstStride); /** The real meat of VideoRendererInterface. */ |