video: OpenGL rendering of incoming frames
The previous system to display the incoming video frames was relying
on CGImage generation with raw framebuffer which were then set as the
content of the CallView. This way of doing it is not efficient as it
implies buffer copies and is discouraged by Apple for pictures that
change often. Moreover, this process was done by the
VideoReceiveThread from the daemon which was then blocked by those
copies without being able to decode further incoming frames. This is
why a lag was appearing and increasing on high resolution stream.
The new system now isolates frame delivering to the UI and their
rendering. The VideoReceiveThread just update the current frame buffer
and size without copy and another thread send those data to an OpenGL
texture on screen refresh which also enables to automatically skip
frames in case of heavy load.
Change-Id: I0b79ddce66f52a3db1eee19945733ff93e7ce34f
Reviewed-by: Kateryna Kostiuk <kateryna.kostiuk@savoirfairelinux.com>
diff --git a/src/views/CallLayer.mm b/src/views/CallLayer.mm
new file mode 100644
index 0000000..f1a6ea9
--- /dev/null
+++ b/src/views/CallLayer.mm
@@ -0,0 +1,206 @@
+/*
+ * Copyright (C) 2017 Savoir-faire Linux Inc.
+ * Author: Anthony Léonard <anthony.leonard@savoirfairelinux.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#import "CallLayer.h"
+#import <OpenGL/gl3.h>
+
+static const GLchar* vShaderSrc = R"glsl(
+#version 150
+
+in vec2 in_Pos;
+in vec2 in_TexCoord;
+uniform vec2 in_Scaling;
+
+out vec2 texCoord;
+
+void main()
+{
+ texCoord = in_TexCoord;
+ gl_Position = vec4(in_Pos.x*in_Scaling.x, in_Pos.y*in_Scaling.y, 0.0, 1.0);
+}
+)glsl";
+
+static const GLchar* fShaderSrc = R"glsl(
+#version 150
+
+out vec4 fragColor;
+in vec2 texCoord;
+
+uniform sampler2D tex;
+
+void main()
+{
+ fragColor = texture(tex, texCoord);
+}
+)glsl";
+
+@implementation CallLayer
+
+// OpenGL handlers
+GLuint tex, vbo, vShader, fShader, sProg, vao;
+
+// Last frame data and attributes
+Video::Frame currentFrame;
+QSize currentFrameSize;
+BOOL currentFrameDisplayed;
+NSLock* currentFrameLk;
+
+- (id) init
+{
+ self = [super init];
+ if (self) {
+ currentFrameLk = [[NSLock alloc] init];
+ [self setVideoRunning:NO];
+ }
+ return self;
+}
+
+// This setter is redefined so we can initialize the OpenGL context when this one is
+// setup by the UI (which seems to be done just before the first draw attempt and not in init method);
+- (void)setOpenGLContext:(NSOpenGLContext *)openGLContext
+{
+ [super setOpenGLContext:openGLContext];
+
+ if (openGLContext) {
+ GLfloat vertices[] = {
+ -1.0, 1.0, 0.0, 0.0, // Top-left
+ 1.0, 1.0, 1.0, 0.0, // Top-right
+ -1.0, -1.0, 0.0, 1.0, // Bottom-left
+ 1.0, -1.0, 1.0, 1.0 // Bottom-right
+ };
+
+ [openGLContext makeCurrentContext];
+
+ // VAO
+ glGenVertexArrays(1, &vao);
+ glBindVertexArray(vao);
+
+ // VBO
+ glGenBuffers(1, &vbo);
+ glBindBuffer(GL_ARRAY_BUFFER, vbo);
+ glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+
+ // Vertex shader
+ vShader = glCreateShader(GL_VERTEX_SHADER);
+ glShaderSource(vShader, 1, &vShaderSrc, NULL);
+ glCompileShader(vShader);
+
+ // Fragment shader
+ fShader = glCreateShader(GL_FRAGMENT_SHADER);
+ glShaderSource(fShader, 1, &fShaderSrc, NULL);
+ glCompileShader(fShader);
+
+ // Program
+ sProg = glCreateProgram();
+ glAttachShader(sProg, vShader);
+ glAttachShader(sProg, fShader);
+ glBindFragDataLocation(sProg, 0, "fragColor");
+ glLinkProgram(sProg);
+ glUseProgram(sProg);
+
+ // Vertices position attrib
+ GLuint inPosAttrib = glGetAttribLocation(sProg, "in_Pos");
+ glEnableVertexAttribArray(inPosAttrib);
+ glVertexAttribPointer(inPosAttrib, 2, GL_FLOAT, GL_FALSE, 4*sizeof(GLfloat), 0);
+
+ // Texture position attrib
+ GLuint inTexCoordAttrib = glGetAttribLocation(sProg, "in_TexCoord");
+ glEnableVertexAttribArray(inTexCoordAttrib);
+ glVertexAttribPointer(inTexCoordAttrib, 2, GL_FLOAT, GL_FALSE, 4*sizeof(GLfloat), (void*)(2*sizeof(GLfloat)));
+
+ // Texture
+ glGenTextures(1, &tex);
+ glBindTexture(GL_TEXTURE_2D, tex);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+ }
+}
+
+- (NSOpenGLPixelFormat *)openGLPixelFormatForDisplayMask:(uint32_t)mask
+{
+ NSOpenGLPixelFormatAttribute attrs[] = {
+ NSOpenGLPFANoRecovery,
+ NSOpenGLPFAColorSize, 24,
+ NSOpenGLPFAAlphaSize, 8,
+ NSOpenGLPFADoubleBuffer,
+ NSOpenGLPFAScreenMask,
+ mask,
+ NSOpenGLPFAAccelerated,
+ NSOpenGLPFAOpenGLProfile,
+ NSOpenGLProfileVersion3_2Core,
+ 0
+ };
+
+ NSOpenGLPixelFormat* pixelFormat = [[NSOpenGLPixelFormat alloc] initWithAttributes:attrs];
+
+ return pixelFormat;
+}
+
+- (BOOL)isAsynchronous
+{
+ return YES;
+}
+
+- (void)drawInOpenGLContext:(NSOpenGLContext *)context pixelFormat:(NSOpenGLPixelFormat *)pixelFormat forLayerTime:(CFTimeInterval)t displayTime:(const CVTimeStamp *)ts
+{
+ GLenum errEnum;
+ glBindTexture(GL_TEXTURE_2D, tex);
+
+ [currentFrameLk lock];
+ if(!currentFrameDisplayed) {
+ if(currentFrame.ptr)
+ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, currentFrameSize.width(), currentFrameSize.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, currentFrame.ptr);
+ currentFrameDisplayed = YES;
+ }
+ // To ensure that we will not divide by zero
+ if (!currentFrameSize.isEmpty()) {
+ // Compute scaling factor to keep the original aspect ratio of the video
+ CGSize viewSize = self.frame.size;
+ float viewRatio = viewSize.width/viewSize.height;
+ float frameRatio = ((float)currentFrameSize.width())/((float)currentFrameSize.height());
+ float ratio = viewRatio * (1/frameRatio);
+
+ GLint inScalingUniform = glGetUniformLocation(sProg, "in_Scaling");
+
+ if (ratio < 1.0)
+ glUniform2f(inScalingUniform, 1.0, ratio);
+ else
+ glUniform2f(inScalingUniform, 1.0/ratio, 1.0);
+ }
+ [currentFrameLk unlock];
+
+ glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+ glClear(GL_COLOR_BUFFER_BIT);
+
+ if([self videoRunning])
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+}
+
+- (void) setCurrentFrame:(Video::Frame)framePtr ofSize:(QSize)frameSize
+{
+ [currentFrameLk lock];
+ currentFrame = std::move(framePtr);
+ currentFrameSize = frameSize;
+ currentFrameDisplayed = NO;
+ [currentFrameLk unlock];
+}
+
+@end