[client] egl: make better use of the second thread for streaming

VERSION

@@ -1 +1 @@
-B1-203-g240d0ff263+1
+B1-204-gdc3e89e65c+1

client/include/interface/renderer.h

@@ -109,7 +109,9 @@ typedef struct LG_Renderer
   LG_RendererOnFrameEvent on_frame_event;
   LG_RendererOnAlert      on_alert;
   LG_RendererRender       render_startup;
+  LG_RendererRender       render_begin;
+  LG_RendererRender       render_end;
   LG_RendererRender       render;
   LG_RendererUpdateFPS    update_fps;
 }
-LG_Renderer;
+LG_Renderer;

client/renderers/EGL/desktop.c

@@ -18,6 +18,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 */
 
 #include "desktop.h"
+#include "egl.h"
 #include "common/debug.h"
 #include "common/option.h"
 #include "common/locking.h"
@@ -47,22 +48,19 @@ struct DesktopShader
 
 struct EGL_Desktop
 {
+  void * egl;
+
   EGL_Texture          * texture;
   struct DesktopShader * shader; // the active shader
   EGL_Model            * model;
 
+  // internals
+  int width, height;
+
   // shader instances
   struct DesktopShader shader_generic;
   struct DesktopShader shader_yuv;
 
-  // internals
-  LG_Lock              updateLock;
-  enum EGL_PixelFormat pixFmt;
-  unsigned int         width, height;
-  unsigned int         pitch;
-  const FrameBuffer  * frame;
-  bool                 update;
-
   // night vision
   KeybindHandle kbNV;
   int   nvMax;
@@ -97,7 +95,7 @@ static bool egl_init_desktop_shader(
   return true;
 }
 
-bool egl_desktop_init(EGL_Desktop ** desktop)
+bool egl_desktop_init(void * egl, EGL_Desktop ** desktop)
 {
   *desktop = (EGL_Desktop *)malloc(sizeof(EGL_Desktop));
   if (!*desktop)
@@ -141,8 +139,7 @@ bool egl_desktop_init(EGL_Desktop ** desktop)
   egl_model_set_default((*desktop)->model);
   egl_model_set_texture((*desktop)->model, (*desktop)->texture);
 
-  LG_LOCK_INIT((*desktop)->updateLock);
+  (*desktop)->egl  = egl;
-
   (*desktop)->kbNV = app_register_keybind(SDL_SCANCODE_N, egl_desktop_toggle_nv, *desktop);
 
   (*desktop)->nvMax  = option_get_int("egl", "nvGainMax");
@@ -168,8 +165,6 @@ void egl_desktop_free(EGL_Desktop ** desktop)
   if (!*desktop)
     return;
 
-  LG_LOCK_FREE((*desktop)->updateLock);
-
   egl_texture_free(&(*desktop)->texture              );
   egl_shader_free (&(*desktop)->shader_generic.shader);
   egl_shader_free (&(*desktop)->shader_yuv.shader    );
@@ -181,80 +176,74 @@ void egl_desktop_free(EGL_Desktop ** desktop)
   *desktop = NULL;
 }
 
-bool egl_desktop_prepare_update(EGL_Desktop * desktop, const bool sourceChanged, const LG_RendererFormat format, const FrameBuffer * frame)
+bool egl_desktop_update(EGL_Desktop * desktop, const bool sourceChanged, const LG_RendererFormat format, const FrameBuffer * frame)
 {
   if (sourceChanged)
   {
-    LG_LOCK(desktop->updateLock);
+    enum EGL_PixelFormat pixFmt;
     switch(format.type)
     {
       case FRAME_TYPE_BGRA:
-        desktop->pixFmt = EGL_PF_BGRA;
+        pixFmt = EGL_PF_BGRA;
         desktop->shader = &desktop->shader_generic;
         break;
 
       case FRAME_TYPE_RGBA:
-        desktop->pixFmt = EGL_PF_RGBA;
+        pixFmt = EGL_PF_RGBA;
         desktop->shader = &desktop->shader_generic;
         break;
 
       case FRAME_TYPE_RGBA10:
-        desktop->pixFmt = EGL_PF_RGBA10;
+        pixFmt = EGL_PF_RGBA10;
         desktop->shader = &desktop->shader_generic;
         break;
 
       case FRAME_TYPE_YUV420:
-        desktop->pixFmt = EGL_PF_YUV420;
+        pixFmt = EGL_PF_YUV420;
         desktop->shader = &desktop->shader_yuv;
         break;
 
       default:
         DEBUG_ERROR("Unsupported frame format");
-        LG_UNLOCK(desktop->updateLock);
         return false;
     }
 
     desktop->width  = format.width;
     desktop->height = format.height;
-    desktop->pitch  = format.pitch;
-    desktop->frame  = frame;
-    desktop->update = true;
 
-    /* defer the actual update as the format has changed and we need to issue GL commands first */
+    egl_lock(desktop->egl);
-    LG_UNLOCK(desktop->updateLock);
-    return true;
-  }
-
-  /* update the texture now */
-  return egl_texture_update_from_frame(desktop->texture, frame);
-}
-
-void egl_desktop_perform_update(EGL_Desktop * desktop, const bool sourceChanged)
-{
-  if (sourceChanged)
-  {
-    LG_LOCK(desktop->updateLock);
     if (!egl_texture_setup(
       desktop->texture,
-      desktop->pixFmt,
+      pixFmt,
-      desktop->width,
+      format.width,
-      desktop->height,
+      format.height,
-      desktop->pitch,
+      format.pitch,
       true // streaming texture
     ))
     {
+      egl_unlock(desktop->egl);
       DEBUG_ERROR("Failed to setup the desktop texture");
-      LG_UNLOCK(desktop->updateLock);
+      return false;
-      return;
     }
-    LG_UNLOCK(desktop->updateLock);
+    egl_unlock(desktop->egl);
   }
 
-  if (desktop->update)
+  if (!egl_texture_update_from_frame(desktop->texture, frame))
+    return false;
+
+  egl_lock(desktop->egl);
+  enum EGL_TexStatus status;
+  if ((status = egl_texture_process(desktop->texture)) != EGL_TEX_STATUS_OK)
   {
-    desktop->update = false;
+    if (status != EGL_TEX_STATUS_NOTREADY)
-    egl_texture_update_from_frame(desktop->texture, desktop->frame);
+    {
+      DEBUG_ERROR("Failed to process the desktop texture");
+      egl_unlock(desktop->egl);
+    }
   }
+  egl_unlock(desktop->egl);
+
+  return true;
 }
 
 bool egl_desktop_render(EGL_Desktop * desktop, const float x, const float y, const float scaleX, const float scaleY, const bool nearest)
@@ -262,9 +251,6 @@ bool egl_desktop_render(EGL_Desktop * desktop, const float x, const float y, con
   if (!desktop->shader)
     return false;
 
-  if (egl_texture_process(desktop->texture) != EGL_TEX_STATUS_OK)
-    return false;
-
   const struct DesktopShader * shader = desktop->shader;
   egl_shader_use(shader->shader);
   glUniform4f(shader->uDesktopPos , x, y, scaleX, scaleY);

client/renderers/EGL/desktop.h

@@ -25,9 +25,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 
 typedef struct EGL_Desktop EGL_Desktop;
 
-bool egl_desktop_init(EGL_Desktop ** desktop);
+bool egl_desktop_init(void * egl, EGL_Desktop ** desktop);
 void egl_desktop_free(EGL_Desktop ** desktop);
 
-bool egl_desktop_prepare_update(EGL_Desktop * desktop, const bool sourceChanged, const LG_RendererFormat format, const FrameBuffer * frame);
+bool egl_desktop_update(EGL_Desktop * desktop, const bool sourceChanged, const LG_RendererFormat format, const FrameBuffer * frame);
-void egl_desktop_perform_update(EGL_Desktop * desktop, const bool sourceChanged);
+bool egl_desktop_render(EGL_Desktop * desktop, const float x, const float y, const float scaleX, const float scaleY, const bool nearest);
-bool egl_desktop_render(EGL_Desktop * desktop, const float x, const float y, const float scaleX, const float scaleY, const bool nearest);

client/renderers/EGL/egl.c

@@ -23,6 +23,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "common/option.h"
 #include "common/sysinfo.h"
 #include "common/time.h"
+#include "common/locking.h"
 #include "utils.h"
 #include "dynamic/fonts.h"
 
@@ -58,7 +59,8 @@ struct Inst
   EGLDisplay           display;
   EGLConfig            configs;
   EGLSurface           surface;
-  EGLContext           context;
+  LG_Lock              lock;
+  EGLContext           context, frameContext;
 
   EGL_Desktop     * desktop; // the desktop
   EGL_Cursor      * cursor;  // the mouse cursor
@@ -67,7 +69,6 @@ struct Inst
   EGL_Alert       * alert;   // the alert display
 
   LG_RendererFormat    format;
-  bool                 sourceChanged;
   uint64_t             waitFadeTime;
   bool                 waitDone;
 
@@ -170,6 +171,7 @@ bool egl_create(void ** opaque, const LG_RendererParams params)
   this->scaleY       = 1.0f;
   this->screenScaleX = 1.0f;
   this->screenScaleY = 1.0f;
+  LG_LOCK_INIT(this->lock);
 
   this->font = LG_Fonts[0];
   if (!this->font->create(&this->fontObj, NULL, 16))
@@ -220,6 +222,8 @@ void egl_deinitialize(void * opaque)
   egl_splash_free (&this->splash);
   egl_alert_free  (&this->alert );
 
+  LG_LOCK_FREE(this->lock);
+
   free(this);
 }
 
@@ -296,25 +300,52 @@ bool egl_on_mouse_event(void * opaque, const bool visible, const int x, const in
   return true;
 }
 
+void egl_lock(void * opaque)
+{
+  struct Inst * this = (struct Inst *)opaque;
+  LG_LOCK(this->lock);
+  eglMakeCurrent(this->display, this->surface, this->surface, this->frameContext);
+}
+
+void egl_unlock(void * opaque)
+{
+  struct Inst * this = (struct Inst *)opaque;
+  eglMakeCurrent(this->display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+  LG_UNLOCK(this->lock);
+}
+
 bool egl_on_frame_event(void * opaque, const LG_RendererFormat format, const FrameBuffer * frame)
 {
   struct Inst * this = (struct Inst *)opaque;
-  this->sourceChanged = (
+  const bool sourceChanged = (
-    this->sourceChanged ||
     this->format.type   != format.type   ||
     this->format.width  != format.width  ||
     this->format.height != format.height ||
     this->format.pitch  != format.pitch
   );
 
-  if (this->sourceChanged)
+  if (sourceChanged)
     memcpy(&this->format, &format, sizeof(LG_RendererFormat));
 
   this->useNearest = this->width < format.width || this->height < format.height;
 
-  if (!egl_desktop_prepare_update(this->desktop, this->sourceChanged, format, frame))
+  if (!this->frameContext)
   {
-    DEBUG_INFO("Failed to prepare to update the desktop");
+    static EGLint attrs[] = {
+      EGL_CONTEXT_CLIENT_VERSION, 2,
+      EGL_NONE
+    };
+
+    if (!(this->frameContext = eglCreateContext(this->display, this->configs, this->context, attrs)))
+    {
+      DEBUG_ERROR("Failed to create the frame context");
+      return false;
+    }
+  }
+
+  if (!egl_desktop_update(this->desktop, sourceChanged, format, frame))
+  {
+    DEBUG_INFO("Failed to to update the desktop");
     return false;
   }
 
@@ -470,7 +501,7 @@ bool egl_render_startup(void * opaque, SDL_Window * window)
 
   eglSwapInterval(this->display, this->opt.vsync ? 1 : 0);
 
-  if (!egl_desktop_init(&this->desktop))
+  if (!egl_desktop_init(this, &this->desktop))
   {
     DEBUG_ERROR("Failed to initialize the desktop");
     return false;
@@ -503,6 +534,21 @@ bool egl_render_startup(void * opaque, SDL_Window * window)
   return true;
 }
 
+bool egl_render_begin(void * opaque, SDL_Window * window)
+{
+  struct Inst * this = (struct Inst *)opaque;
+  LG_LOCK(this->lock);
+  return eglMakeCurrent(this->display, this->surface, this->surface, this->context);
+}
+
+bool egl_render_end(void * opaque, SDL_Window * window)
+{
+  struct Inst * this = (struct Inst *)opaque;
+  bool ret = eglMakeCurrent(this->display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+  LG_UNLOCK(this->lock);
+  return ret;
+}
+
 bool egl_render(void * opaque, SDL_Window * window)
 {
   struct Inst * this = (struct Inst *)opaque;
@@ -554,11 +600,6 @@ bool egl_render(void * opaque, SDL_Window * window)
 
   egl_fps_render(this->fps, this->screenScaleX, this->screenScaleY);
   eglSwapBuffers(this->display, this->surface);
-
-  // defer texture uploads until after the flip to avoid stalling
-  egl_desktop_perform_update(this->desktop, this->sourceChanged);
-
-  this->sourceChanged = false;
   return true;
 }
 
@@ -584,6 +625,8 @@ struct LG_Renderer LGR_EGL =
   .on_frame_event = egl_on_frame_event,
   .on_alert       = egl_on_alert,
   .render_startup = egl_render_startup,
+  .render_begin   = egl_render_begin,
+  .render_end     = egl_render_end,
   .render         = egl_render,
   .update_fps     = egl_update_fps
 };

client/renderers/EGL/egl.h

@@ -0,0 +1,28 @@
+/*
+Looking Glass - KVM FrameRelay (KVMFR) Client
+Copyright (C) 2017-2020 Geoffrey McRae <geoff@hostfission.com>
+https://looking-glass.hostfission.com
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+#pragma once
+
+#include <stdbool.h>
+
+#include "interface/renderer.h"
+
+/* helpers to lock and make current the secondary context for the desktop */
+void egl_lock(void * opaque);
+void egl_unlock(void * opaque);

client/renderers/EGL/texture.c

@@ -119,6 +119,41 @@ void egl_texture_free(EGL_Texture ** texture)
   *texture = NULL;
 }
 
+static bool egl_texture_map(EGL_Texture * texture)
+{
+  // release old PBOs and delete and re-create the buffers
+  for(int i = 0; i < TEXTURE_COUNT; ++i)
+  {
+    glBindBuffer(GL_PIXEL_UNPACK_BUFFER, texture->tex[i].pbo);
+    texture->tex[i].map = glMapBufferRange(
+      GL_PIXEL_UNPACK_BUFFER,
+      0,
+      texture->pboBufferSize,
+      GL_MAP_WRITE_BIT             |
+      GL_MAP_UNSYNCHRONIZED_BIT    |
+      GL_MAP_INVALIDATE_BUFFER_BIT
+    );
+
+    if (!texture->tex[i].map)
+    {
+      EGL_ERROR("glMapBufferRange failed for %d of %lu bytes", i, texture->pboBufferSize);
+      return false;
+    }
+  }
+
+  glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+  return true;
+}
+
+static void egl_texture_unmap(EGL_Texture * texture)
+{
+  for(int i = 0; i < TEXTURE_COUNT; ++i)
+  {
+    glBindBuffer(GL_PIXEL_UNPACK_BUFFER, texture->tex[i].pbo);
+    glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
+  }
+}
+
 bool egl_texture_setup(EGL_Texture * texture, enum EGL_PixelFormat pixFmt, size_t width, size_t height, size_t stride, bool streaming)
 {
   int planeCount;
@@ -232,49 +267,33 @@ bool egl_texture_setup(EGL_Texture * texture, enum EGL_PixelFormat pixFmt, size_
   }
   glBindTexture(GL_TEXTURE_2D, 0);
 
-  if (streaming)
+  egl_texture_unmap(texture);
+
+  // release old PBOs and delete and re-create the buffers
+  for(int i = 0; i < TEXTURE_COUNT; ++i)
   {
-    // release old PBOs and delete and re-create the buffers
+    if (texture->tex[i].hasPBO)
-    for(int i = 0; i < TEXTURE_COUNT; ++i)
     {
-      if (texture->tex[i].hasPBO)
-      {
-        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, texture->tex[i].pbo);
-        glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
-        glDeleteBuffers(1, &texture->tex[i].pbo);
-      }
-
-      glGenBuffers(1, &texture->tex[i].pbo);
-      texture->tex[i].hasPBO = true;
-
       glBindBuffer(GL_PIXEL_UNPACK_BUFFER, texture->tex[i].pbo);
-      glBufferStorage(
+      glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
-        GL_PIXEL_UNPACK_BUFFER,
+      glDeleteBuffers(1, &texture->tex[i].pbo);
-        texture->pboBufferSize,
-        NULL,
-        GL_MAP_PERSISTENT_BIT |
-        GL_MAP_WRITE_BIT
-      );
-
-      texture->tex[i].map = glMapBufferRange(
-        GL_PIXEL_UNPACK_BUFFER,
-        0,
-        texture->pboBufferSize,
-        GL_MAP_PERSISTENT_BIT        |
-        GL_MAP_WRITE_BIT             |
-        GL_MAP_UNSYNCHRONIZED_BIT    |
-        GL_MAP_INVALIDATE_BUFFER_BIT
-      );
-
-      if (!texture->tex[i].map)
-      {
-        EGL_ERROR("glMapBufferRange failed for %d of %lu bytes", i, texture->pboBufferSize);
-        return false;
-      }
     }
-    glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+
+    glGenBuffers(1, &texture->tex[i].pbo);
+    texture->tex[i].hasPBO = true;
+
+    glBindBuffer(GL_PIXEL_UNPACK_BUFFER, texture->tex[i].pbo);
+    glBufferStorage(
+      GL_PIXEL_UNPACK_BUFFER,
+      texture->pboBufferSize,
+      NULL,
+      GL_MAP_WRITE_BIT
+    );
   }
 
+  if (!egl_texture_map(texture))
+    return false;
+
   return true;
 }
 
@@ -295,8 +314,6 @@ bool egl_texture_update(EGL_Texture * texture, const uint8_t * buffer)
 {
   if (texture->streaming)
   {
-    /* NOTE: DO NOT use any gl commands here as streaming must be thread safe */
-
     union TexState s;
     s.v = atomic_load_explicit(&texture->state.v, memory_order_acquire);
 
@@ -368,6 +385,7 @@ enum EGL_TexStatus egl_texture_process(EGL_Texture * texture)
     return texture->ready ? EGL_TEX_STATUS_OK : EGL_TEX_STATUS_NOTREADY;
 
   /* update the texture */
+  egl_texture_unmap(texture);
   glBindBuffer(GL_PIXEL_UNPACK_BUFFER, texture->tex[s.u].pbo);
   for(int p = 0; p < texture->planeCount; ++p)
   {
@@ -377,7 +395,6 @@ enum EGL_TexStatus egl_texture_process(EGL_Texture * texture)
         texture->format, texture->dataType, (const void *)texture->offsets[p]);
 
   }
-  glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
 
   /* create a fence to prevent usage before the update is complete */
   texture->tex[s.u].sync =
@@ -385,6 +402,9 @@ enum EGL_TexStatus egl_texture_process(EGL_Texture * texture)
 
   atomic_store_explicit(&texture->state.u, nextu, memory_order_release);
 
+  /* remap the for the next update */
+  egl_texture_map(texture);
+
   texture->ready = true;
   return EGL_TEX_STATUS_OK;
 }

client/src/main.c

@@ -180,6 +180,10 @@ static int renderThread(void * unused)
       }
     }
 
+    if (state.lgr->render_begin && !state.lgr->render_begin(state.lgrData,
+          state.window))
+      break;
+
     if (state.lgrResize)
     {
       if (state.lgr)
@@ -209,6 +213,10 @@ static int renderThread(void * unused)
       }
     }
 
+    if (state.lgr->render_end && !state.lgr->render_end(state.lgrData,
+          state.window))
+      break;
+
     if (!state.resizeDone && state.resizeTimeout < microtime())
     {
       SDL_SetWindowSize(