From e799badacc29a7567091f213a8b319f2581eff69 Mon Sep 17 00:00:00 2001 From: Jon Hjelle Date: Mon, 11 Jan 2016 13:47:11 -0800 Subject: Move Objective-C video renderers to webrtc/api/objc. BUG= R=tkchin@webrtc.org Review URL: https://codereview.webrtc.org/1542473003 . Patch from Jon Hjelle . Cr-Commit-Position: refs/heads/master@{#11209} --- webrtc/api/BUILD.gn | 26 ++ webrtc/api/api.gyp | 27 ++ webrtc/api/objc/RTCEAGLVideoView.h | 35 +++ webrtc/api/objc/RTCEAGLVideoView.m | 259 ++++++++++++++++ webrtc/api/objc/RTCNSGLVideoView.h | 34 +++ webrtc/api/objc/RTCNSGLVideoView.m | 141 +++++++++ webrtc/api/objc/RTCOpenGLVideoRenderer.h | 58 ++++ webrtc/api/objc/RTCOpenGLVideoRenderer.mm | 485 ++++++++++++++++++++++++++++++ webrtc/api/objc/RTCVideoRenderer.h | 30 ++ 9 files changed, 1095 insertions(+) create mode 100644 webrtc/api/objc/RTCEAGLVideoView.h create mode 100644 webrtc/api/objc/RTCEAGLVideoView.m create mode 100644 webrtc/api/objc/RTCNSGLVideoView.h create mode 100644 webrtc/api/objc/RTCNSGLVideoView.m create mode 100644 webrtc/api/objc/RTCOpenGLVideoRenderer.h create mode 100644 webrtc/api/objc/RTCOpenGLVideoRenderer.mm create mode 100644 webrtc/api/objc/RTCVideoRenderer.h diff --git a/webrtc/api/BUILD.gn b/webrtc/api/BUILD.gn index 157c79b3e9..c5ed3c804a 100644 --- a/webrtc/api/BUILD.gn +++ b/webrtc/api/BUILD.gn @@ -8,6 +8,15 @@ import("../build/webrtc.gni") +config("ios_config") { + libs = [ + "CoreGraphics.framework", + "GLKit.framework", + "OpenGLES.framework", + "QuartzCore.framework", + ] +} + if (is_ios) { source_set("rtc_api_objc") { deps = [ @@ -32,6 +41,8 @@ if (is_ios) { "objc/RTCMediaConstraints+Private.h", "objc/RTCMediaConstraints.h", "objc/RTCMediaConstraints.mm", + "objc/RTCOpenGLVideoRenderer.h", + "objc/RTCOpenGLVideoRenderer.mm", "objc/RTCSessionDescription+Private.h", "objc/RTCSessionDescription.h", "objc/RTCSessionDescription.mm", @@ -41,7 +52,22 @@ if (is_ios) { "objc/RTCVideoFrame+Private.h", "objc/RTCVideoFrame.h", "objc/RTCVideoFrame.mm", + "objc/RTCVideoRenderer.h", "objc/WebRTC-Prefix.pch", ] + + if (is_ios) { + sources += [ + "objc/RTCEAGLVideoView.h", + "objc/RTCEAGLVideoView.m", + ] + } + + if (is_mac) { + sources += [ + "objc/RTCNSGLVideoView.h", + "objc/RTCNSGLVideoView.m", + ] + } } } diff --git a/webrtc/api/api.gyp b/webrtc/api/api.gyp index ab1a047df2..430fee70d9 100644 --- a/webrtc/api/api.gyp +++ b/webrtc/api/api.gyp @@ -31,6 +31,8 @@ 'objc/RTCMediaStreamTrack+Private.h', 'objc/RTCMediaStreamTrack.h', 'objc/RTCMediaStreamTrack.mm', + 'objc/RTCOpenGLVideoRenderer.h', + 'objc/RTCOpenGLVideoRenderer.mm', 'objc/RTCSessionDescription+Private.h', 'objc/RTCSessionDescription.h', 'objc/RTCSessionDescription.mm', @@ -40,6 +42,31 @@ 'objc/RTCVideoFrame+Private.h', 'objc/RTCVideoFrame.h', 'objc/RTCVideoFrame.mm', + 'objc/RTCVideoRenderer.h', + ], + 'conditions': [ + ['OS=="ios"', { + 'sources': [ + 'objc/RTCEAGLVideoView.h', + 'objc/RTCEAGLVideoView.m', + ], + 'all_dependent_settings': { + 'xcode_settings': { + 'OTHER_LDFLAGS': [ + '-framework CoreGraphics', + '-framework GLKit', + '-framework OpenGLES', + '-framework QuartzCore', + ] + } + } + }], + ['OS=="mac"', { + 'sources': [ + 'objc/RTCNSGLVideoView.h', + 'objc/RTCNSGLVideoView.m', + ], + }], ], 'xcode_settings': { 'CLANG_ENABLE_OBJC_ARC': 'YES', diff --git a/webrtc/api/objc/RTCEAGLVideoView.h b/webrtc/api/objc/RTCEAGLVideoView.h new file mode 100644 index 0000000000..1a57df76bb --- /dev/null +++ b/webrtc/api/objc/RTCEAGLVideoView.h @@ -0,0 +1,35 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#import "RTCVideoRenderer.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTCEAGLVideoView; +@protocol RTCEAGLVideoViewDelegate + +- (void)videoView:(RTCEAGLVideoView *)videoView didChangeVideoSize:(CGSize)size; + +@end + +/** + * RTCEAGLVideoView is an RTCVideoRenderer which renders video frames in its + * bounds using OpenGLES 2.0. + */ +@interface RTCEAGLVideoView : UIView + +@property(nonatomic, weak) id delegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/api/objc/RTCEAGLVideoView.m b/webrtc/api/objc/RTCEAGLVideoView.m new file mode 100644 index 0000000000..e664ede455 --- /dev/null +++ b/webrtc/api/objc/RTCEAGLVideoView.m @@ -0,0 +1,259 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCEAGLVideoView.h" + +#import + +#import "RTCVideoFrame.h" +#import "RTCOpenGLVideoRenderer.h" + +// RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen +// refreshes, which should be 30fps. We wrap the display link in order to avoid +// a retain cycle since CADisplayLink takes a strong reference onto its target. +// The timer is paused by default. +@interface RTCDisplayLinkTimer : NSObject + +@property(nonatomic) BOOL isPaused; + +- (instancetype)initWithTimerHandler:(void (^)(void))timerHandler; +- (void)invalidate; + +@end + +@implementation RTCDisplayLinkTimer { + CADisplayLink *_displayLink; + void (^_timerHandler)(void); +} + +- (instancetype)initWithTimerHandler:(void (^)(void))timerHandler { + NSParameterAssert(timerHandler); + if (self = [super init]) { + _timerHandler = timerHandler; + _displayLink = + [CADisplayLink displayLinkWithTarget:self + selector:@selector(displayLinkDidFire:)]; + _displayLink.paused = YES; + // Set to half of screen refresh, which should be 30fps. + [_displayLink setFrameInterval:2]; + [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] + forMode:NSRunLoopCommonModes]; + } + return self; +} + +- (void)dealloc { + [self invalidate]; +} + +- (BOOL)isPaused { + return _displayLink.paused; +} + +- (void)setIsPaused:(BOOL)isPaused { + _displayLink.paused = isPaused; +} + +- (void)invalidate { + [_displayLink invalidate]; +} + +- (void)displayLinkDidFire:(CADisplayLink *)displayLink { + _timerHandler(); +} + +@end + +// RTCEAGLVideoView wraps a GLKView which is setup with +// enableSetNeedsDisplay = NO for the purpose of gaining control of +// exactly when to call -[GLKView display]. This need for extra +// control is required to avoid triggering method calls on GLKView +// that results in attempting to bind the underlying render buffer +// when the drawable size would be empty which would result in the +// error GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT. -[GLKView display] is +// the method that will trigger the binding of the render +// buffer. Because the standard behaviour of -[UIView setNeedsDisplay] +// is disabled for the reasons above, the RTCEAGLVideoView maintains +// its own |isDirty| flag. + +@interface RTCEAGLVideoView () +// |videoFrame| is set when we receive a frame from a worker thread and is read +// from the display link callback so atomicity is required. +@property(atomic, strong) RTCVideoFrame *videoFrame; +@property(nonatomic, readonly) GLKView *glkView; +@property(nonatomic, readonly) RTCOpenGLVideoRenderer *glRenderer; +@end + +@implementation RTCEAGLVideoView { + RTCDisplayLinkTimer *_timer; + // This flag should only be set and read on the main thread (e.g. by + // setNeedsDisplay) + BOOL _isDirty; +} + +@synthesize delegate = _delegate; +@synthesize videoFrame = _videoFrame; +@synthesize glkView = _glkView; +@synthesize glRenderer = _glRenderer; + +- (instancetype)initWithFrame:(CGRect)frame { + if (self = [super initWithFrame:frame]) { + [self configure]; + } + return self; +} + +- (instancetype)initWithCoder:(NSCoder *)aDecoder { + if (self = [super initWithCoder:aDecoder]) { + [self configure]; + } + return self; +} + +- (void)configure { + EAGLContext *glContext = + [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; + if (!glContext) { + glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2]; + } + _glRenderer = [[RTCOpenGLVideoRenderer alloc] initWithContext:glContext]; + + // GLKView manages a framebuffer for us. + _glkView = [[GLKView alloc] initWithFrame:CGRectZero + context:glContext]; + _glkView.drawableColorFormat = GLKViewDrawableColorFormatRGBA8888; + _glkView.drawableDepthFormat = GLKViewDrawableDepthFormatNone; + _glkView.drawableStencilFormat = GLKViewDrawableStencilFormatNone; + _glkView.drawableMultisample = GLKViewDrawableMultisampleNone; + _glkView.delegate = self; + _glkView.layer.masksToBounds = YES; + _glkView.enableSetNeedsDisplay = NO; + [self addSubview:_glkView]; + + // Listen to application state in order to clean up OpenGL before app goes + // away. + NSNotificationCenter *notificationCenter = + [NSNotificationCenter defaultCenter]; + [notificationCenter addObserver:self + selector:@selector(willResignActive) + name:UIApplicationWillResignActiveNotification + object:nil]; + [notificationCenter addObserver:self + selector:@selector(didBecomeActive) + name:UIApplicationDidBecomeActiveNotification + object:nil]; + + // Frames are received on a separate thread, so we poll for current frame + // using a refresh rate proportional to screen refresh frequency. This + // occurs on the main thread. + __weak RTCEAGLVideoView *weakSelf = self; + _timer = [[RTCDisplayLinkTimer alloc] initWithTimerHandler:^{ + RTCEAGLVideoView *strongSelf = weakSelf; + [strongSelf displayLinkTimerDidFire]; + }]; + [self setupGL]; +} + +- (void)dealloc { + [[NSNotificationCenter defaultCenter] removeObserver:self]; + UIApplicationState appState = + [UIApplication sharedApplication].applicationState; + if (appState == UIApplicationStateActive) { + [self teardownGL]; + } + [_timer invalidate]; +} + +#pragma mark - UIView + +- (void)setNeedsDisplay { + [super setNeedsDisplay]; + _isDirty = YES; +} + +- (void)setNeedsDisplayInRect:(CGRect)rect { + [super setNeedsDisplayInRect:rect]; + _isDirty = YES; +} + +- (void)layoutSubviews { + [super layoutSubviews]; + _glkView.frame = self.bounds; +} + +#pragma mark - GLKViewDelegate + +// This method is called when the GLKView's content is dirty and needs to be +// redrawn. This occurs on main thread. +- (void)glkView:(GLKView *)view drawInRect:(CGRect)rect { + // The renderer will draw the frame to the framebuffer corresponding to the + // one used by |view|. + [_glRenderer drawFrame:self.videoFrame]; +} + +#pragma mark - RTCVideoRenderer + +// These methods may be called on non-main thread. +- (void)setSize:(CGSize)size { + __weak RTCEAGLVideoView *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + RTCEAGLVideoView *strongSelf = weakSelf; + [strongSelf.delegate videoView:strongSelf didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(RTCVideoFrame *)frame { + self.videoFrame = frame; +} + +#pragma mark - Private + +- (void)displayLinkTimerDidFire { + // Don't render unless video frame have changed or the view content + // has explicitly been marked dirty. + if (!_isDirty && _glRenderer.lastDrawnFrame == self.videoFrame) { + return; + } + + // Always reset isDirty at this point, even if -[GLKView display] + // won't be called in the case the drawable size is empty. + _isDirty = NO; + + // Only call -[GLKView display] if the drawable size is + // non-empty. Calling display will make the GLKView setup its + // render buffer if necessary, but that will fail with error + // GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT if size is empty. + if (self.bounds.size.width > 0 && self.bounds.size.height > 0) { + [_glkView display]; + } +} + +- (void)setupGL { + self.videoFrame = nil; + [_glRenderer setupGL]; + _timer.isPaused = NO; +} + +- (void)teardownGL { + self.videoFrame = nil; + _timer.isPaused = YES; + [_glkView deleteDrawable]; + [_glRenderer teardownGL]; +} + +- (void)didBecomeActive { + [self setupGL]; +} + +- (void)willResignActive { + [self teardownGL]; +} + +@end diff --git a/webrtc/api/objc/RTCNSGLVideoView.h b/webrtc/api/objc/RTCNSGLVideoView.h new file mode 100644 index 0000000000..27eb31e9af --- /dev/null +++ b/webrtc/api/objc/RTCNSGLVideoView.h @@ -0,0 +1,34 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if TARGET_OS_IPHONE +#error "This file targets OSX." +#endif + +#import + +#import "RTCVideoRenderer.h" + +NS_ASSUME_NONNULL_BEGIN + +@class RTCNSGLVideoView; +@protocol RTCNSGLVideoViewDelegate + +- (void)videoView:(RTCNSGLVideoView *)videoView didChangeVideoSize:(CGSize)size; + +@end + +@interface RTCNSGLVideoView : NSOpenGLView + +@property(nonatomic, weak) id delegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/api/objc/RTCNSGLVideoView.m b/webrtc/api/objc/RTCNSGLVideoView.m new file mode 100644 index 0000000000..063e6f1330 --- /dev/null +++ b/webrtc/api/objc/RTCNSGLVideoView.m @@ -0,0 +1,141 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCNSGLVideoView.h" + +#import +#import +#import "RTCVideoFrame.h" +#import "RTCOpenGLVideoRenderer.h" + +@interface RTCNSGLVideoView () +// |videoFrame| is set when we receive a frame from a worker thread and is read +// from the display link callback so atomicity is required. +@property(atomic, strong) RTCVideoFrame *videoFrame; +@property(atomic, strong) RTCOpenGLVideoRenderer *glRenderer; +- (void)drawFrame; +@end + +static CVReturn OnDisplayLinkFired(CVDisplayLinkRef displayLink, + const CVTimeStamp *now, + const CVTimeStamp *outputTime, + CVOptionFlags flagsIn, + CVOptionFlags *flagsOut, + void *displayLinkContext) { + RTCNSGLVideoView *view = (__bridge RTCNSGLVideoView *)displayLinkContext; + [view drawFrame]; + return kCVReturnSuccess; +} + +@implementation RTCNSGLVideoView { + CVDisplayLinkRef _displayLink; +} + +@synthesize delegate = _delegate; +@synthesize videoFrame = _videoFrame; +@synthesize glRenderer = _glRenderer; + +- (void)dealloc { + [self teardownDisplayLink]; +} + +- (void)drawRect:(NSRect)rect { + [self drawFrame]; +} + +- (void)reshape { + [super reshape]; + NSRect frame = [self frame]; + CGLLockContext([[self openGLContext] CGLContextObj]); + glViewport(0, 0, frame.size.width, frame.size.height); + CGLUnlockContext([[self openGLContext] CGLContextObj]); +} + +- (void)lockFocus { + NSOpenGLContext *context = [self openGLContext]; + [super lockFocus]; + if ([context view] != self) { + [context setView:self]; + } + [context makeCurrentContext]; +} + +- (void)prepareOpenGL { + [super prepareOpenGL]; + if (!self.glRenderer) { + self.glRenderer = + [[RTCOpenGLVideoRenderer alloc] initWithContext:[self openGLContext]]; + } + [self.glRenderer setupGL]; + [self setupDisplayLink]; +} + +- (void)clearGLContext { + [self.glRenderer teardownGL]; + self.glRenderer = nil; + [super clearGLContext]; +} + +#pragma mark - RTCVideoRenderer + +// These methods may be called on non-main thread. +- (void)setSize:(CGSize)size { + dispatch_async(dispatch_get_main_queue(), ^{ + [self.delegate videoView:self didChangeVideoSize:size]; + }); +} + +- (void)renderFrame:(RTCVideoFrame *)frame { + self.videoFrame = frame; +} + +#pragma mark - Private + +- (void)drawFrame { + RTCVideoFrame *videoFrame = self.videoFrame; + if (self.glRenderer.lastDrawnFrame != videoFrame) { + // This method may be called from CVDisplayLink callback which isn't on the + // main thread so we have to lock the GL context before drawing. + CGLLockContext([[self openGLContext] CGLContextObj]); + [self.glRenderer drawFrame:videoFrame]; + CGLUnlockContext([[self openGLContext] CGLContextObj]); + } +} + +- (void)setupDisplayLink { + if (_displayLink) { + return; + } + // Synchronize buffer swaps with vertical refresh rate. + GLint swapInt = 1; + [[self openGLContext] setValues:&swapInt forParameter:NSOpenGLCPSwapInterval]; + + // Create display link. + CVDisplayLinkCreateWithActiveCGDisplays(&_displayLink); + CVDisplayLinkSetOutputCallback(_displayLink, + &OnDisplayLinkFired, + (__bridge void *)self); + // Set the display link for the current renderer. + CGLContextObj cglContext = [[self openGLContext] CGLContextObj]; + CGLPixelFormatObj cglPixelFormat = [[self pixelFormat] CGLPixelFormatObj]; + CVDisplayLinkSetCurrentCGDisplayFromOpenGLContext( + _displayLink, cglContext, cglPixelFormat); + CVDisplayLinkStart(_displayLink); +} + +- (void)teardownDisplayLink { + if (!_displayLink) { + return; + } + CVDisplayLinkRelease(_displayLink); + _displayLink = NULL; +} + +@end diff --git a/webrtc/api/objc/RTCOpenGLVideoRenderer.h b/webrtc/api/objc/RTCOpenGLVideoRenderer.h new file mode 100644 index 0000000000..729839c6a3 --- /dev/null +++ b/webrtc/api/objc/RTCOpenGLVideoRenderer.h @@ -0,0 +1,58 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#if TARGET_OS_IPHONE +#import +#else +#import +#endif + +NS_ASSUME_NONNULL_BEGIN + +@class RTCVideoFrame; + +// RTCOpenGLVideoRenderer issues appropriate OpenGL commands to draw a frame to +// the currently bound framebuffer. Supports OpenGL 3.2 and OpenGLES 2.0. OpenGL +// framebuffer creation and management should be handled elsewhere using the +// same context used to initialize this class. +@interface RTCOpenGLVideoRenderer : NSObject + +// The last successfully drawn frame. Used to avoid drawing frames unnecessarily +// hence saving battery life by reducing load. +@property(nonatomic, readonly) RTCVideoFrame *lastDrawnFrame; + +#if TARGET_OS_IPHONE +- (instancetype)initWithContext:(EAGLContext *)context + NS_DESIGNATED_INITIALIZER; +#else +- (instancetype)initWithContext:(NSOpenGLContext *)context + NS_DESIGNATED_INITIALIZER; +#endif + +// Draws |frame| onto the currently bound OpenGL framebuffer. |setupGL| must be +// called before this function will succeed. +- (BOOL)drawFrame:(RTCVideoFrame *)frame; + +// The following methods are used to manage OpenGL resources. On iOS +// applications should release resources when placed in background for use in +// the foreground application. In fact, attempting to call OpenGLES commands +// while in background will result in application termination. + +// Sets up the OpenGL state needed for rendering. +- (void)setupGL; +// Tears down the OpenGL state created by |setupGL|. +- (void)teardownGL; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/api/objc/RTCOpenGLVideoRenderer.mm b/webrtc/api/objc/RTCOpenGLVideoRenderer.mm new file mode 100644 index 0000000000..56a6431ffa --- /dev/null +++ b/webrtc/api/objc/RTCOpenGLVideoRenderer.mm @@ -0,0 +1,485 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCOpenGLVideoRenderer.h" + +#include + +#include "webrtc/base/scoped_ptr.h" + +#if TARGET_OS_IPHONE +#import +#else +#import +#endif + +#import "RTCVideoFrame.h" + +// TODO(tkchin): check and log openGL errors. Methods here return BOOLs in +// anticipation of that happening in the future. + +#if TARGET_OS_IPHONE +#define RTC_PIXEL_FORMAT GL_LUMINANCE +#define SHADER_VERSION +#define VERTEX_SHADER_IN "attribute" +#define VERTEX_SHADER_OUT "varying" +#define FRAGMENT_SHADER_IN "varying" +#define FRAGMENT_SHADER_OUT +#define FRAGMENT_SHADER_COLOR "gl_FragColor" +#define FRAGMENT_SHADER_TEXTURE "texture2D" +#else +#define RTC_PIXEL_FORMAT GL_RED +#define SHADER_VERSION "#version 150\n" +#define VERTEX_SHADER_IN "in" +#define VERTEX_SHADER_OUT "out" +#define FRAGMENT_SHADER_IN "in" +#define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" +#define FRAGMENT_SHADER_COLOR "fragColor" +#define FRAGMENT_SHADER_TEXTURE "texture" +#endif + +// Vertex shader doesn't do anything except pass coordinates through. +static const char kVertexShaderSource[] = + SHADER_VERSION + VERTEX_SHADER_IN " vec2 position;\n" + VERTEX_SHADER_IN " vec2 texcoord;\n" + VERTEX_SHADER_OUT " vec2 v_texcoord;\n" + "void main() {\n" + " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" + " v_texcoord = texcoord;\n" + "}\n"; + +// Fragment shader converts YUV values from input textures into a final RGB +// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. +static const char kFragmentShaderSource[] = + SHADER_VERSION + "precision highp float;" + FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" + "uniform lowp sampler2D s_textureY;\n" + "uniform lowp sampler2D s_textureU;\n" + "uniform lowp sampler2D s_textureV;\n" + FRAGMENT_SHADER_OUT + "void main() {\n" + " float y, u, v, r, g, b;\n" + " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" + " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" + " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" + " u = u - 0.5;\n" + " v = v - 0.5;\n" + " r = y + 1.403 * v;\n" + " g = y - 0.344 * u - 0.714 * v;\n" + " b = y + 1.770 * u;\n" + " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" + " }\n"; + +// Compiles a shader of the given |type| with GLSL source |source| and returns +// the shader handle or 0 on error. +GLuint CreateShader(GLenum type, const GLchar *source) { + GLuint shader = glCreateShader(type); + if (!shader) { + return 0; + } + glShaderSource(shader, 1, &source, NULL); + glCompileShader(shader); + GLint compileStatus = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); + if (compileStatus == GL_FALSE) { + glDeleteShader(shader); + shader = 0; + } + return shader; +} + +// Links a shader program with the given vertex and fragment shaders and +// returns the program handle or 0 on error. +GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) { + if (vertexShader == 0 || fragmentShader == 0) { + return 0; + } + GLuint program = glCreateProgram(); + if (!program) { + return 0; + } + glAttachShader(program, vertexShader); + glAttachShader(program, fragmentShader); + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (linkStatus == GL_FALSE) { + glDeleteProgram(program); + program = 0; + } + return program; +} + +// When modelview and projection matrices are identity (default) the world is +// contained in the square around origin with unit size 2. Drawing to these +// coordinates is equivalent to drawing to the entire screen. The texture is +// stretched over that square using texture coordinates (u, v) that range +// from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically +// here because the incoming frame has origin in upper left hand corner but +// OpenGL expects origin in bottom left corner. +const GLfloat gVertices[] = { + // X, Y, U, V. + -1, -1, 0, 1, // Bottom left. + 1, -1, 1, 1, // Bottom right. + 1, 1, 1, 0, // Top right. + -1, 1, 0, 0, // Top left. +}; + +// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets +// of 3 textures are used here, one for each of the Y, U and V planes. Having +// two sets alleviates CPU blockage in the event that the GPU is asked to render +// to a texture that is already in use. +static const GLsizei kNumTextureSets = 2; +static const GLsizei kNumTextures = 3 * kNumTextureSets; + +@implementation RTCOpenGLVideoRenderer { +#if TARGET_OS_IPHONE + EAGLContext *_context; +#else + NSOpenGLContext *_context; +#endif + BOOL _isInitialized; + NSUInteger _currentTextureSet; + // Handles for OpenGL constructs. + GLuint _textures[kNumTextures]; + GLuint _program; +#if !TARGET_OS_IPHONE + GLuint _vertexArray; +#endif + GLuint _vertexBuffer; + GLint _position; + GLint _texcoord; + GLint _ySampler; + GLint _uSampler; + GLint _vSampler; + // Used to create a non-padded plane for GPU upload when we receive padded + // frames. + rtc::scoped_ptr _planeBuffer; +} + +@synthesize lastDrawnFrame = _lastDrawnFrame; + ++ (void)initialize { + // Disable dithering for performance. + glDisable(GL_DITHER); +} + +#if TARGET_OS_IPHONE +- (instancetype)initWithContext:(EAGLContext *)context { +#else +- (instancetype)initWithContext:(NSOpenGLContext *)context { +#endif + NSAssert(context != nil, @"context cannot be nil"); + if (self = [super init]) { + _context = context; + } + return self; +} + +- (BOOL)drawFrame:(RTCVideoFrame *)frame { + if (!_isInitialized) { + return NO; + } + if (_lastDrawnFrame == frame) { + return NO; + } + [self ensureGLContext]; + glClear(GL_COLOR_BUFFER_BIT); + if (frame) { + if (![self updateTextureSizesForFrame:frame] || + ![self updateTextureDataForFrame:frame]) { + return NO; + } +#if !TARGET_OS_IPHONE + glBindVertexArray(_vertexArray); +#endif + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + } +#if !TARGET_OS_IPHONE + [_context flushBuffer]; +#endif + _lastDrawnFrame = frame; + return YES; +} + +- (void)setupGL { + if (_isInitialized) { + return; + } + [self ensureGLContext]; + if (![self setupProgram]) { + return; + } + if (![self setupTextures]) { + return; + } + if (![self setupVertices]) { + return; + } + glUseProgram(_program); + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + _isInitialized = YES; +} + +- (void)teardownGL { + if (!_isInitialized) { + return; + } + [self ensureGLContext]; + glDeleteProgram(_program); + _program = 0; + glDeleteTextures(kNumTextures, _textures); + glDeleteBuffers(1, &_vertexBuffer); + _vertexBuffer = 0; +#if !TARGET_OS_IPHONE + glDeleteVertexArrays(1, &_vertexArray); +#endif + _isInitialized = NO; +} + +#pragma mark - Private + +- (void)ensureGLContext { + NSAssert(_context, @"context shouldn't be nil"); +#if TARGET_OS_IPHONE + if ([EAGLContext currentContext] != _context) { + [EAGLContext setCurrentContext:_context]; + } +#else + if ([NSOpenGLContext currentContext] != _context) { + [_context makeCurrentContext]; + } +#endif +} + +- (BOOL)setupProgram { + NSAssert(!_program, @"program already set up"); + GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource); + NSAssert(vertexShader, @"failed to create vertex shader"); + GLuint fragmentShader = + CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource); + NSAssert(fragmentShader, @"failed to create fragment shader"); + _program = CreateProgram(vertexShader, fragmentShader); + // Shaders are created only to generate program. + if (vertexShader) { + glDeleteShader(vertexShader); + } + if (fragmentShader) { + glDeleteShader(fragmentShader); + } + if (!_program) { + return NO; + } + _position = glGetAttribLocation(_program, "position"); + _texcoord = glGetAttribLocation(_program, "texcoord"); + _ySampler = glGetUniformLocation(_program, "s_textureY"); + _uSampler = glGetUniformLocation(_program, "s_textureU"); + _vSampler = glGetUniformLocation(_program, "s_textureV"); + if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 || + _vSampler < 0) { + return NO; + } + return YES; +} + +- (BOOL)setupTextures { + glGenTextures(kNumTextures, _textures); + // Set parameters for each of the textures we created. + for (GLsizei i = 0; i < kNumTextures; i++) { + glActiveTexture(GL_TEXTURE0 + i); + glBindTexture(GL_TEXTURE_2D, _textures[i]); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + return YES; +} + +- (BOOL)updateTextureSizesForFrame:(RTCVideoFrame *)frame { + if (frame.height == _lastDrawnFrame.height && + frame.width == _lastDrawnFrame.width && + frame.chromaWidth == _lastDrawnFrame.chromaWidth && + frame.chromaHeight == _lastDrawnFrame.chromaHeight) { + return YES; + } + GLsizei lumaWidth = frame.width; + GLsizei lumaHeight = frame.height; + GLsizei chromaWidth = frame.chromaWidth; + GLsizei chromaHeight = frame.chromaHeight; + for (GLint i = 0; i < kNumTextureSets; i++) { + glActiveTexture(GL_TEXTURE0 + i * 3); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + lumaWidth, + lumaHeight, + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + 0); + glActiveTexture(GL_TEXTURE0 + i * 3 + 1); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + chromaWidth, + chromaHeight, + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + 0); + glActiveTexture(GL_TEXTURE0 + i * 3 + 2); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + chromaWidth, + chromaHeight, + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + 0); + } + if ((NSUInteger)frame.yPitch != frame.width || + (NSUInteger)frame.uPitch != frame.chromaWidth || + (NSUInteger)frame.vPitch != frame.chromaWidth) { + _planeBuffer.reset(new uint8_t[frame.width * frame.height]); + } else { + _planeBuffer.reset(); + } + return YES; +} + +- (void)uploadPlane:(const uint8_t *)plane + sampler:(GLint)sampler + offset:(NSUInteger)offset + width:(size_t)width + height:(size_t)height + stride:(int32_t)stride { + glActiveTexture(GL_TEXTURE0 + offset); + // When setting texture sampler uniforms, the texture index is used not + // the texture handle. + glUniform1i(sampler, offset); +#if TARGET_OS_IPHONE + BOOL hasUnpackRowLength = _context.API == kEAGLRenderingAPIOpenGLES3; +#else + BOOL hasUnpackRowLength = YES; +#endif + const uint8_t *uploadPlane = plane; + if ((size_t)stride != width) { + if (hasUnpackRowLength) { + // GLES3 allows us to specify stride. + glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + width, + height, + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + return; + } else { + // Make an unpadded copy and upload that instead. Quick profiling showed + // that this is faster than uploading row by row using glTexSubImage2D. + uint8_t *unpaddedPlane = _planeBuffer.get(); + for (size_t y = 0; y < height; ++y) { + memcpy(unpaddedPlane + y * width, plane + y * stride, width); + } + uploadPlane = unpaddedPlane; + } + } + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + width, + height, + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); +} + +- (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame { + NSUInteger textureOffset = _currentTextureSet * 3; + NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); + + [self uploadPlane:frame.yPlane + sampler:_ySampler + offset:textureOffset + width:frame.width + height:frame.height + stride:frame.yPitch]; + + [self uploadPlane:frame.uPlane + sampler:_uSampler + offset:textureOffset + 1 + width:frame.chromaWidth + height:frame.chromaHeight + stride:frame.uPitch]; + + [self uploadPlane:frame.vPlane + sampler:_vSampler + offset:textureOffset + 2 + width:frame.chromaWidth + height:frame.chromaHeight + stride:frame.vPitch]; + + _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; + return YES; +} + +- (BOOL)setupVertices { +#if !TARGET_OS_IPHONE + NSAssert(!_vertexArray, @"vertex array already set up"); + glGenVertexArrays(1, &_vertexArray); + if (!_vertexArray) { + return NO; + } + glBindVertexArray(_vertexArray); +#endif + NSAssert(!_vertexBuffer, @"vertex buffer already set up"); + glGenBuffers(1, &_vertexBuffer); + if (!_vertexBuffer) { +#if !TARGET_OS_IPHONE + glDeleteVertexArrays(1, &_vertexArray); + _vertexArray = 0; +#endif + return NO; + } + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); + + // Read position attribute from |gVertices| with size of 2 and stride of 4 + // beginning at the start of the array. The last argument indicates offset + // of data within |gVertices| as supplied to the vertex buffer. + glVertexAttribPointer( + _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); + glEnableVertexAttribArray(_position); + + // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 + // beginning at the first texcoord in the array. The last argument indicates + // offset of data within |gVertices| as supplied to the vertex buffer. + glVertexAttribPointer(_texcoord, + 2, + GL_FLOAT, + GL_FALSE, + 4 * sizeof(GLfloat), + (void *)(2 * sizeof(GLfloat))); + glEnableVertexAttribArray(_texcoord); + + return YES; +} + +@end diff --git a/webrtc/api/objc/RTCVideoRenderer.h b/webrtc/api/objc/RTCVideoRenderer.h new file mode 100644 index 0000000000..a97456275a --- /dev/null +++ b/webrtc/api/objc/RTCVideoRenderer.h @@ -0,0 +1,30 @@ +/* + * Copyright 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#if TARGET_OS_IPHONE +#import +#endif + +NS_ASSUME_NONNULL_BEGIN + +@class RTCVideoFrame; + +@protocol RTCVideoRenderer + +/** The size of the frame. */ +- (void)setSize:(CGSize)size; + +/** The frame to be displayed. */ +- (void)renderFrame:(RTCVideoFrame *)frame; + +@end + +NS_ASSUME_NONNULL_END -- cgit v1.2.3