diff --git a/client/include/interface/renderer.h b/client/include/interface/renderer.h index 3435855e..11c73f7e 100644 --- a/client/include/interface/renderer.h +++ b/client/include/interface/renderer.h @@ -161,6 +161,51 @@ typedef enum LG_RendererCursor } LG_RendererCursor; +/* Current producers reserve at most a 512x512 32-bpp cursor payload. + * Monochrome wire heights contain vertically stacked AND and XOR masks. */ +#define LG_CURSOR_MAX_WIDTH 512 +#define LG_CURSOR_MAX_HEIGHT 512 +#define LG_CURSOR_MAX_DATA_SIZE \ + ((size_t)LG_CURSOR_MAX_WIDTH * LG_CURSOR_MAX_HEIGHT * 4U) + +static inline bool lg_rendererCursorValidate(LG_RendererCursor type, + int width, int height, int pitch, size_t * dataSize) +{ + if (width <= 0 || height <= 0 || pitch <= 0 || + width > LG_CURSOR_MAX_WIDTH) + return false; + + size_t rowBytes; + int decodedHeight; + switch (type) + { + case LG_CURSOR_COLOR: + case LG_CURSOR_MASKED_COLOR: + rowBytes = (size_t)width * 4U; + decodedHeight = height; + break; + + case LG_CURSOR_MONOCHROME: + if (height & 1) + return false; + rowBytes = ((size_t)width + 7U) / 8U; + decodedHeight = height / 2; + break; + + default: + return false; + } + + if (decodedHeight <= 0 || decodedHeight > LG_CURSOR_MAX_HEIGHT || + (size_t)pitch < rowBytes || + (size_t)pitch > LG_CURSOR_MAX_DATA_SIZE / (size_t)height) + return false; + + if (dataSize) + *dataSize = (size_t)height * (size_t)pitch; + return true; +} + typedef struct LG_Renderer LG_Renderer; typedef enum LG_RendererInteropType diff --git a/client/renderers/EGL/cursor.c b/client/renderers/EGL/cursor.c index 4577bd21..8669be3f 100644 --- a/client/renderers/EGL/cursor.c +++ b/client/renderers/EGL/cursor.c @@ -83,6 +83,9 @@ struct EGL_Cursor int stride; uint8_t * data; size_t dataSize; + uint32_t * normData; + uint32_t * monoData; + size_t decodedCapacity; bool update; // cursor state @@ -255,6 +258,8 @@ void egl_cursorFree(EGL_Cursor ** cursor) LG_LOCK_FREE((*cursor)->lock); if ((*cursor)->data) free((*cursor)->data); + free((*cursor)->normData); + free((*cursor)->monoData); cursorTexFree(&(*cursor)->norm); cursorTexFree(&(*cursor)->mono); @@ -266,34 +271,72 @@ void egl_cursorFree(EGL_Cursor ** cursor) *cursor = NULL; } +static bool cursorEnsureDecoded(EGL_Cursor * cursor, size_t pixels) +{ + if (pixels <= cursor->decodedCapacity) + return true; + + const size_t size = pixels * sizeof(*cursor->normData); + uint32_t * normData = malloc(size); + uint32_t * monoData = malloc(size); + if (!normData || !monoData) + { + free(normData); + free(monoData); + DEBUG_ERROR("Failed to allocate cursor conversion buffers"); + return false; + } + + free(cursor->normData); + free(cursor->monoData); + cursor->normData = normData; + cursor->monoData = monoData; + cursor->decodedCapacity = pixels; + return true; +} + bool egl_cursorSetShape(EGL_Cursor * cursor, const LG_RendererCursor type, const int width, const int height, const int stride, const uint8_t * data) { + size_t size; + if (!data || !lg_rendererCursorValidate( + type, width, height, stride, &size)) + { + DEBUG_ERROR("Invalid cursor shape geometry"); + return false; + } + + const int decodedHeight = + type == LG_CURSOR_MONOCHROME ? height / 2 : height; + const size_t pixels = (size_t)width * (size_t)decodedHeight; + LG_LOCK(cursor->lock); - cursor->type = type; - cursor->width = width; - cursor->height = (type == LG_CURSOR_MONOCHROME ? height / 2 : height); - cursor->stride = stride; + if (!cursorEnsureDecoded(cursor, pixels)) + { + LG_UNLOCK(cursor->lock); + return false; + } - const size_t size = height * stride; if (size > cursor->dataSize) { - if (cursor->data) - free(cursor->data); - - cursor->data = malloc(size); - if (!cursor->data) + uint8_t * resized = realloc(cursor->data, size); + if (!resized) { DEBUG_ERROR("Failed to malloc buffer for cursor shape"); LG_UNLOCK(cursor->lock); return false; } + cursor->data = resized; cursor->dataSize = size; } memcpy(cursor->data, data, size); + cursor->type = type; + cursor->width = width; + cursor->height = decodedHeight; + cursor->stride = stride; cursor->update = true; LG_UNLOCK(cursor->lock); @@ -367,67 +410,92 @@ struct CursorState egl_cursorRender(EGL_Cursor * cursor, if (updateShape) { - uint8_t * data = cursor->data; + const uint8_t * data = cursor->data; switch(cursor->type) { case LG_CURSOR_MASKED_COLOR: { - uint32_t xor[cursor->height][cursor->width]; for(int y = 0; y < cursor->height; ++y) + { + const uint8_t * row = data + (size_t)cursor->stride * (size_t)y; for(int x = 0; x < cursor->width; ++x) { - uint32_t * src = (uint32_t *)(data + (cursor->stride * y) + x * 4); - const bool masked = (*src & 0xFF000000) != 0; + uint32_t src; + memcpy(&src, row + (size_t)x * sizeof(src), sizeof(src)); + const size_t index = + (size_t)y * (size_t)cursor->width + (size_t)x; + const bool masked = (src & 0xFF000000) != 0; if (masked) - *src = xor[y][x] = *src & 0x00FFFFFF; + cursor->normData[index] = cursor->monoData[index] = + src & 0x00FFFFFF; else { - xor[y][x] = 0xFF000000; - *src |= 0xFF000000; + cursor->normData[index] = src | 0xFF000000; + cursor->monoData[index] = 0xFF000000; } } + } egl_textureSetup(cursor->mono.texture, EGL_PF_BGRA, - cursor->width, cursor->height, cursor->width, sizeof(xor[0])); - egl_textureUpdate(cursor->mono.texture, (uint8_t *)xor, true); - } - // fall through - - case LG_CURSOR_COLOR: - { + cursor->width, cursor->height, cursor->width, + (size_t)cursor->width * sizeof(*cursor->monoData)); + egl_textureUpdate(cursor->mono.texture, + (const uint8_t *)cursor->monoData, true); egl_textureSetup(cursor->norm.texture, EGL_PF_BGRA, - cursor->width, cursor->height, cursor->width, cursor->stride); - egl_textureUpdate(cursor->norm.texture, data, true); + cursor->width, cursor->height, cursor->width, + (size_t)cursor->width * sizeof(*cursor->normData)); + egl_textureUpdate(cursor->norm.texture, + (const uint8_t *)cursor->normData, true); break; } + case LG_CURSOR_COLOR: + for(int y = 0; y < cursor->height; ++y) + memcpy(cursor->normData + (size_t)y * (size_t)cursor->width, + data + (size_t)y * (size_t)cursor->stride, + (size_t)cursor->width * sizeof(*cursor->normData)); + + egl_textureSetup(cursor->norm.texture, EGL_PF_BGRA, + cursor->width, cursor->height, cursor->width, + (size_t)cursor->width * sizeof(*cursor->normData)); + egl_textureUpdate(cursor->norm.texture, + (const uint8_t *)cursor->normData, true); + break; + case LG_CURSOR_MONOCHROME: { - uint32_t and[cursor->height][cursor->width]; - uint32_t xor[cursor->height][cursor->width]; - for(int y = 0; y < cursor->height; ++y) { + const uint8_t * srcAnd = + data + (size_t)cursor->stride * (size_t)y; + const uint8_t * srcXor = data + + (size_t)cursor->stride * (size_t)(y + cursor->height); for(int x = 0; x < cursor->width; ++x) { - const uint8_t * srcAnd = data + (cursor->stride * y) + (x / 8); - const uint8_t * srcXor = srcAnd + cursor->stride * cursor->height; const uint8_t mask = 0x80 >> (x % 8); - const uint32_t andMask = (*srcAnd & mask) ? 0xFFFFFFFF : 0xFF000000; - const uint32_t xorMask = (*srcXor & mask) ? 0x00FFFFFF : 0x00000000; + const uint32_t andMask = + (srcAnd[x / 8] & mask) ? 0xFFFFFFFF : 0xFF000000; + const uint32_t xorMask = + (srcXor[x / 8] & mask) ? 0x00FFFFFF : 0x00000000; + const size_t index = + (size_t)y * (size_t)cursor->width + (size_t)x; - and[y][x] = andMask; - xor[y][x] = xorMask; + cursor->normData[index] = andMask; + cursor->monoData[index] = xorMask; } } // Monochrome cursors use AND/XOR mask textures - never convert to PQ egl_textureSetup(cursor->norm.texture, EGL_PF_BGRA, - cursor->width, cursor->height, cursor->width, sizeof(and[0])); + cursor->width, cursor->height, cursor->width, + (size_t)cursor->width * sizeof(*cursor->normData)); egl_textureSetup(cursor->mono.texture, EGL_PF_BGRA, - cursor->width, cursor->height, cursor->width, sizeof(xor[0])); - egl_textureUpdate(cursor->norm.texture, (uint8_t *)and, true); - egl_textureUpdate(cursor->mono.texture, (uint8_t *)xor, true); + cursor->width, cursor->height, cursor->width, + (size_t)cursor->width * sizeof(*cursor->monoData)); + egl_textureUpdate(cursor->norm.texture, + (const uint8_t *)cursor->normData, true); + egl_textureUpdate(cursor->mono.texture, + (const uint8_t *)cursor->monoData, true); break; } } diff --git a/client/renderers/OpenGL/opengl.c b/client/renderers/OpenGL/opengl.c index 21557b62..8eac8b70 100644 --- a/client/renderers/OpenGL/opengl.c +++ b/client/renderers/OpenGL/opengl.c @@ -157,6 +157,8 @@ struct Inst int mousePitch; uint8_t * mouseData; size_t mouseDataSize; + uint32_t * mouseRGBA; + size_t mouseRGBACapacity; bool mouseUpdate; bool newShape; @@ -269,6 +271,7 @@ void opengl_deinitialize(LG_Renderer * renderer) if (this->mouseData) free(this->mouseData); + free(this->mouseRGBA); free(this->swSurfaceBuffer); @@ -352,29 +355,52 @@ bool opengl_onMouseShape(LG_Renderer * renderer, const LG_RendererCursor cursor, { struct Inst * this = UPCAST(struct Inst, renderer); + size_t size; + if (!data || !lg_rendererCursorValidate( + cursor, width, height, pitch, &size)) + { + DEBUG_ERROR("Invalid cursor shape geometry"); + return false; + } + + const size_t pixels = (size_t)width * (size_t)height; + LG_LOCK(this->mouseLock); + + if (size > this->mouseDataSize) + { + uint8_t * resized = realloc(this->mouseData, size); + if (!resized) + { + DEBUG_ERROR("Failed to allocate cursor shape buffer"); + LG_UNLOCK(this->mouseLock); + return false; + } + + this->mouseData = resized; + this->mouseDataSize = size; + } + + if (pixels > this->mouseRGBACapacity) + { + uint32_t * resized = realloc( + this->mouseRGBA, pixels * sizeof(*this->mouseRGBA)); + if (!resized) + { + DEBUG_ERROR("Failed to allocate cursor conversion buffer"); + LG_UNLOCK(this->mouseLock); + return false; + } + + this->mouseRGBA = resized; + this->mouseRGBACapacity = pixels; + } + + memcpy(this->mouseData, data, size); this->mouseCursor = cursor; this->mouseWidth = width; this->mouseHeight = height; this->mousePitch = pitch; - - const size_t size = height * pitch; - if (size > this->mouseDataSize) - { - if (this->mouseData) - free(this->mouseData); - - this->mouseData = malloc(size); - if (!this->mouseData) - { - DEBUG_ERROR("out of memory"); - return false; - } - - this->mouseDataSize = size; - } - - memcpy(this->mouseData, data, size); this->newShape = true; LG_UNLOCK(this->mouseLock); @@ -1105,27 +1131,41 @@ static void updateMouseShape(struct Inst * this) const int height = this->mouseHeight; const int pitch = this->mousePitch; const uint8_t * data = this->mouseData; - - // tmp buffer for masked colour - uint32_t tmp[width * height]; + uint32_t * rgba = this->mouseRGBA; this->mouseType = cursor; switch(cursor) { case LG_CURSOR_MASKED_COLOR: - for(int i = 0; i < width * height; ++i) + for(int y = 0; y < height; ++y) { - const uint32_t c = ((uint32_t *)data)[i]; - tmp[i] = (c & ~0xFF000000) | (c & 0xFF000000 ? 0x0 : 0xFF000000); + const uint8_t * row = data + (size_t)pitch * (size_t)y; + for(int x = 0; x < width; ++x) + { + uint32_t c; + memcpy(&c, row + (size_t)x * sizeof(c), sizeof(c)); + rgba[(size_t)y * (size_t)width + (size_t)x] = + (c & ~0xFF000000) | + (c & 0xFF000000 ? 0x0 : 0xFF000000); + } } - data = (uint8_t *)tmp; - // fall through to LG_CURSOR_COLOR + data = (const uint8_t *)rgba; + // fall through // // technically we should also create an XOR texture from the data but this // usage seems very rare in modern software. case LG_CURSOR_COLOR: { + if (cursor == LG_CURSOR_COLOR) + { + for(int y = 0; y < height; ++y) + memcpy(rgba + (size_t)y * (size_t)width, + data + (size_t)y * (size_t)pitch, + (size_t)width * sizeof(*rgba)); + data = (const uint8_t *)rgba; + } + glBindTexture(GL_TEXTURE_2D, this->textures[MOUSE_TEXTURE]); glPixelStorei(GL_UNPACK_ALIGNMENT , 4 ); glPixelStorei(GL_UNPACK_ROW_LENGTH, width); @@ -1169,19 +1209,28 @@ static void updateMouseShape(struct Inst * this) case LG_CURSOR_MONOCHROME: { const int hheight = height / 2; - uint32_t d[width * height]; for(int y = 0; y < hheight; ++y) + { + const uint8_t * srcAnd = + data + (size_t)pitch * (size_t)y; + const uint8_t * srcXor = + data + (size_t)pitch * (size_t)(y + hheight); for(int x = 0; x < width; ++x) { - const uint8_t * srcAnd = data + (pitch * y) + (x / 8); - const uint8_t * srcXor = srcAnd + pitch * hheight; const uint8_t mask = 0x80 >> (x % 8); - const uint32_t andMask = (*srcAnd & mask) ? 0xFFFFFFFF : 0xFF000000; - const uint32_t xorMask = (*srcXor & mask) ? 0x00FFFFFF : 0x00000000; + const uint32_t andMask = + (srcAnd[x / 8] & mask) ? 0xFFFFFFFF : 0xFF000000; + const uint32_t xorMask = + (srcXor[x / 8] & mask) ? 0x00FFFFFF : 0x00000000; + const size_t andIndex = + (size_t)y * (size_t)width + (size_t)x; + const size_t xorIndex = + (size_t)(y + hheight) * (size_t)width + (size_t)x; - d[y * width + x ] = andMask; - d[y * width + x + width * hheight] = xorMask; + rgba[andIndex] = andMask; + rgba[xorIndex] = xorMask; } + } glBindTexture(GL_TEXTURE_2D, this->textures[MOUSE_TEXTURE]); glPixelStorei(GL_UNPACK_ALIGNMENT , 4 ); @@ -1196,7 +1245,7 @@ static void updateMouseShape(struct Inst * this) 0 , GL_RGBA, GL_UNSIGNED_BYTE, - d + rgba ); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); diff --git a/client/src/render_queue.c b/client/src/render_queue.c index 474679cf..40db2a54 100644 --- a/client/src/render_queue.c +++ b/client/src/render_queue.c @@ -494,16 +494,12 @@ static void setCommandSource(RenderCommand * cmd, RenderQueueSource source, static bool copyCursorImage(RenderCommand * cmd, const void * data) { - if (!data || cmd->cursorImage.width <= 0 || - cmd->cursorImage.height <= 0 || cmd->cursorImage.pitch <= 0) + size_t size; + if (!data || !lg_rendererCursorValidate(cmd->cursorImage.type, + cmd->cursorImage.width, cmd->cursorImage.height, + cmd->cursorImage.pitch, &size)) return false; - if ((size_t)cmd->cursorImage.height > - SIZE_MAX / (size_t)cmd->cursorImage.pitch) - return false; - - const size_t size = - (size_t)cmd->cursorImage.height * (size_t)cmd->cursorImage.pitch; cmd->cursorImage.data = malloc(size); if (!cmd->cursorImage.data) return false; diff --git a/client/tests/render_queue_test.c b/client/tests/render_queue_test.c index 2353eefb..72cf544e 100644 --- a/client/tests/render_queue_test.c +++ b/client/tests/render_queue_test.c @@ -426,8 +426,16 @@ static void testPayload(void) 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, }; - uint8_t shape[] = { 11, 12, 13, 14, 15, 16, 17, 18 }; - const uint8_t shapeExpected[] = { 11, 12, 13, 14, 15, 16, 17, 18 }; + uint8_t shape[] = + { + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + }; + const uint8_t shapeExpected[] = + { + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + }; LGColorTransform color = { .flags = LG_COLOR_TRANSFORM_MATRIX | LG_COLOR_TRANSFORM_LUT, @@ -439,7 +447,7 @@ static void testPayload(void) renderQueue_sourceSwSurfaceDrawBitmap(RENDER_QUEUE_SOURCE_PRIMARY, generation, 0, 0, 2, 2, 8, bitmap, true); renderQueue_sourceCursorImage(RENDER_QUEUE_SOURCE_PRIMARY, generation, - LG_CURSOR_COLOR, 2, 2, 4, shape); + LG_CURSOR_COLOR, 2, 2, 8, shape); renderQueue_sourceCursorColorTransform(RENDER_QUEUE_SOURCE_PRIMARY, generation, &color); renderQueue_sourceCursorState(RENDER_QUEUE_SOURCE_PRIMARY, generation, @@ -468,7 +476,7 @@ static void testPayload(void) CHECK(f.shapeType == LG_CURSOR_COLOR); CHECK(f.shapeWidth == 2); CHECK(f.shapeHeight == 2); - CHECK(f.shapePitch == 4); + CHECK(f.shapePitch == 8); CHECK(f.shapeSize == sizeof(shapeExpected)); CHECK(memcmp(f.shape, shapeExpected, sizeof(shapeExpected)) == 0); CHECK(f.colorCount == 1);