From 8336a27192135fbd14fb6fcfa5900cc03e27b368 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Mon, 24 Aug 2026 12:59:55 +1000 Subject: [PATCH] [common] rects: validate copy geometry --- common/include/common/rects.h | 7 +- common/src/rects.c | 134 +++++++++++++----- host/platform/Windows/capture/D12/d12.c | 20 ++- host/platform/Windows/capture/DXGI/src/dxgi.c | 14 +- 4 files changed, 124 insertions(+), 51 deletions(-) diff --git a/common/include/common/rects.h b/common/include/common/rects.h index 6d4f78d2..4988572d 100644 --- a/common/include/common/rects.h +++ b/common/include/common/rects.h @@ -33,16 +33,17 @@ extern void (*rectCopyUnaligned)( void rectScaleOutward(struct Rect * rect, double scale); -void rectsBufferToFramebuffer(FrameDamageRect * rects, int count, int bpp, +bool rectsBufferToFramebuffer(const FrameDamageRect * rects, int count, int bpp, FrameBuffer * frame, int dstPitch, int height, const uint8_t * src, int srcPitch); -bool rectsFramebufferToBuffer(FrameDamageRect * rects, int count, int bpp, +bool rectsFramebufferToBuffer(const FrameDamageRect * rects, int count, int bpp, uint8_t * dst, int dstPitch, int height, const FrameBuffer * frame, int srcPitch); /* As above, accumulating producer wait time in nanoseconds. */ -bool rectsFramebufferToBufferTimed(FrameDamageRect * rects, int count, int bpp, +bool rectsFramebufferToBufferTimed(const FrameDamageRect * rects, + int count, int bpp, uint8_t * dst, int dstPitch, int height, const FrameBuffer * frame, int srcPitch, uint64_t * waitTimeNs); diff --git a/common/src/rects.c b/common/src/rects.c index 83a7f63f..c47d3f64 100644 --- a/common/src/rects.c +++ b/common/src/rects.c @@ -99,18 +99,41 @@ static int cornerCompare(const void * a_, const void * b_) return 0; } -inline static bool rectsBufferCopy(FrameDamageRect * rects, int count, int bpp, +static bool rectsBufferCopy(const FrameDamageRect * rects, + int count, int bpp, uint8_t * dst, int dstStride, int height, const uint8_t * src, int srcStride, void * opaque, bool (*rowCopyStart)(int y, void * opaque), void (*rowCopyFinish)(int y, void * opaque)) { - const int cornerCount = 4 * count; - struct Corner corners[cornerCount]; + if (!rects || count <= 0 || bpp <= 0 || !dst || !src || + count > LG_MAX_FRAME_DAMAGE_RECTS || + dstStride <= 0 || srcStride <= 0 || height <= 0 || + (size_t)height > SIZE_MAX / (size_t)dstStride || + (size_t)height > SIZE_MAX / (size_t)srcStride || + (size_t)height * (size_t)dstStride > UINT_LEAST32_MAX || + (size_t)height * (size_t)srcStride > UINT_LEAST32_MAX) + return false; for (int i = 0; i < count; ++i) { - FrameDamageRect * rect = rects + i; + const FrameDamageRect * rect = rects + i; + const uint64_t right = (uint64_t)rect->x + rect->width; + const uint64_t bottom = (uint64_t)rect->y + rect->height; + if (!rect->width || !rect->height || bottom > (uint32_t)height || + right > (uint32_t)dstStride / (uint32_t)bpp || + right > (uint32_t)srcStride / (uint32_t)bpp) + return false; + } + + const size_t cornerCount = (size_t)count * 4U; + struct Corner corners[cornerCount]; + struct Edge active_[2][cornerCount]; + struct Edge change[cornerCount]; + + for (int i = 0; i < count; ++i) + { + const FrameDamageRect * rect = rects + i; corners[4 * i + 0] = (struct Corner) { .x = rect->x, .y = rect->y, .delta = 1 }; @@ -126,24 +149,20 @@ inline static bool rectsBufferCopy(FrameDamageRect * rects, int count, int bpp, } qsort(corners, cornerCount, sizeof(struct Corner), cornerCompare); - struct Edge active_[2][cornerCount]; - struct Edge change[cornerCount]; int prev_y = 0; int activeRow = 0; - int actives = 0; + size_t actives = 0; + bool result = true; - for (int rs = 0;;) + for (size_t rs = 0;;) { int y = corners[rs].y; - int re = rs; + size_t re = rs; while (re < cornerCount && corners[re].y == y) ++re; - if (y > height) - y = height; - - int changes = 0; - for (int i = rs; i < re; ) + size_t changes = 0; + for (size_t i = rs; i < re; ) { int x = corners[i].x; int delta = 0; @@ -153,12 +172,15 @@ inline static bool rectsBufferCopy(FrameDamageRect * rects, int count, int bpp, } if (rowCopyStart && !rowCopyStart(y, opaque)) - return false; + { + result = false; + break; + } struct Edge * active = active_[activeRow]; int x1 = 0; int in_rect = 0; - for (int i = 0; i < actives; ++i) + for (size_t i = 0; i < actives; ++i) { if (!in_rect) x1 = active[i].x; @@ -175,9 +197,9 @@ inline static bool rectsBufferCopy(FrameDamageRect * rects, int count, int bpp, rowCopyFinish(y, opaque); struct Edge * new = active_[activeRow ^ 1]; - int ai = 0; - int ci = 0; - int ni = 0; + size_t ai = 0; + size_t ci = 0; + size_t ni = 0; while (ai < actives && ci < changes) { @@ -206,7 +228,7 @@ inline static bool rectsBufferCopy(FrameDamageRect * rects, int count, int bpp, activeRow ^= 1; } - return true; + return result; } struct ToFramebufferData @@ -218,17 +240,33 @@ struct ToFramebufferData static void fbRowFinish(int y, void * opaque) { struct ToFramebufferData * data = opaque; - framebuffer_set_write_ptr(data->frame, y * data->pitch); + framebuffer_set_write_ptr( + data->frame, (size_t)y * (size_t)data->pitch); } -void rectsBufferToFramebuffer(FrameDamageRect * rects, int count, int bpp, +bool rectsBufferToFramebuffer(const FrameDamageRect * rects, + int count, int bpp, FrameBuffer * frame, int dstPitch, int height, const uint8_t * src, int srcPitch) { + if (!rects || !frame || count <= 0) + return false; + struct ToFramebufferData data = { .frame = frame, .pitch = dstPitch }; - rectsBufferCopy(rects, count, bpp, framebuffer_get_data(frame), dstPitch, - height, src, srcPitch, &data, NULL, fbRowFinish); - framebuffer_set_write_ptr(frame, height * dstPitch); + const bool publishRows = count <= LG_MAX_FRAME_DAMAGE_RECTS; + for (int offset = 0; offset < count; ) + { + const int batch = min(count - offset, LG_MAX_FRAME_DAMAGE_RECTS); + if (!rectsBufferCopy(rects + offset, batch, bpp, + framebuffer_get_data(frame), dstPitch, height, src, srcPitch, + &data, NULL, publishRows ? fbRowFinish : NULL)) + return false; + offset += batch; + } + + framebuffer_set_write_ptr( + frame, (size_t)height * (size_t)dstPitch); + return true; } struct FromFramebufferData @@ -242,24 +280,38 @@ static bool fbRowStart(int y, void * opaque) { struct FromFramebufferData * data = opaque; return framebuffer_wait_timed( - data->frame, y * data->pitch, data->waitTimeNs); + data->frame, (size_t)y * (size_t)data->pitch, data->waitTimeNs); } -bool rectsFramebufferToBufferTimed(FrameDamageRect * rects, int count, int bpp, +bool rectsFramebufferToBufferTimed(const FrameDamageRect * rects, + int count, int bpp, uint8_t * dst, int dstPitch, int height, const FrameBuffer * frame, int srcPitch, uint64_t * waitTimeNs) { + if (!rects || !frame || count <= 0) + return false; + struct FromFramebufferData data = { .frame = frame, .pitch = srcPitch, .waitTimeNs = waitTimeNs }; - return rectsBufferCopy(rects, count, bpp, dst, dstPitch, height, - framebuffer_get_buffer(frame), srcPitch, &data, fbRowStart, NULL); + for (int offset = 0; offset < count; ) + { + const int batch = min(count - offset, LG_MAX_FRAME_DAMAGE_RECTS); + if (!rectsBufferCopy(rects + offset, batch, bpp, dst, dstPitch, height, + framebuffer_get_buffer(frame), srcPitch, + &data, fbRowStart, NULL)) + return false; + offset += batch; + } + + return true; } -bool rectsFramebufferToBuffer(FrameDamageRect * rects, int count, int bpp, +bool rectsFramebufferToBuffer(const FrameDamageRect * rects, + int count, int bpp, uint8_t * dst, int dstPitch, int height, const FrameBuffer * frame, int srcPitch) { @@ -336,13 +388,16 @@ static void rectCopyUnaligned_memcpy( uint8_t *restrict dst, const uint8_t *restrict src, int ystart, int yend, int dx, int dstPitch, int srcPitch, int width) { - src += ystart * srcPitch + dx; - dst += ystart * dstPitch + dx; + src += (size_t)ystart * (size_t)srcPitch + (size_t)dx; + dst += (size_t)ystart * (size_t)dstPitch + (size_t)dx; for (int i = ystart; i < yend; ++i) { memcpy(dst, src, width); - src += srcPitch; - dst += dstPitch; + if (i + 1 < yend) + { + src += srcPitch; + dst += dstPitch; + } } } @@ -356,8 +411,8 @@ static void rectCopyUnaligned_avx( uint8_t *restrict dst, const uint8_t *restrict src, int ystart, int yend, int dx, int dstPitch, int srcPitch, int width) { - src += ystart * srcPitch + dx; - dst += ystart * dstPitch + dx; + src += (size_t)ystart * (size_t)srcPitch + (size_t)dx; + dst += (size_t)ystart * (size_t)dstPitch + (size_t)dx; bool streamed = false; for (int i = ystart; i < yend; ++i) @@ -399,8 +454,11 @@ static void rectCopyUnaligned_avx( for(int col = width - rem; col < width; ++col) dst[col] = src[col]; - src += srcPitch; - dst += dstPitch; + if (i + 1 < yend) + { + src += srcPitch; + dst += dstPitch; + } } if (streamed) diff --git a/host/platform/Windows/capture/D12/d12.c b/host/platform/Windows/capture/D12/d12.c index a538aeae..40d53080 100644 --- a/host/platform/Windows/capture/D12/d12.c +++ b/host/platform/Windows/capture/D12/d12.c @@ -1016,6 +1016,13 @@ static CaptureResult d12_getFrame( if (!*dst) goto exit; + const size_t frameHeight = this->dstFormat.desc.Height; + if (!this->pitch || !frameHeight || + frameHeight > maxFrameSize / this->pitch || + frameHeight * this->pitch > UINT_LEAST32_MAX) + goto exit; + const size_t frameSize = frameHeight * this->pitch; + // place a fence into the queue DEBUG_TRACE("d12_backendSync"); result = d12_backendSync(this->backend, @@ -1210,18 +1217,19 @@ static CaptureResult d12_getFrame( { if (rectCount == 0) { - framebuffer_write(frameBuffer, map, - this->pitch * this->dstFormat.desc.Height); + framebuffer_write(frameBuffer, map, frameSize); } else { /* copy all the rects */ - rectsBufferToFramebuffer(allRects, rectCount, this->bpp, frameBuffer, - this->pitch, this->dstFormat.desc.Height, map, this->pitch); + if (!rectsBufferToFramebuffer(allRects, rectCount, this->bpp, + frameBuffer, this->pitch, this->dstFormat.desc.Height, + map, this->pitch)) + framebuffer_write(frameBuffer, map, + frameSize); // signal the frame is complete - framebuffer_set_write_ptr(frameBuffer, - this->dstFormat.desc.Height * this->pitch); + framebuffer_set_write_ptr(frameBuffer, frameSize); } } else diff --git a/host/platform/Windows/capture/DXGI/src/dxgi.c b/host/platform/Windows/capture/DXGI/src/dxgi.c index 35eff1fa..a715e68d 100644 --- a/host/platform/Windows/capture/DXGI/src/dxgi.c +++ b/host/platform/Windows/capture/DXGI/src/dxgi.c @@ -1551,13 +1551,19 @@ static CaptureResult dxgi_getFrame( scaledDamageRects[i] = rect; } - rectsBufferToFramebuffer(scaledDamageRects, damage->count, this->bpp, frame, - this->pitch, this->dataHeight, tex->map, this->pitch); + if (!rectsBufferToFramebuffer(scaledDamageRects, damage->count, + this->bpp, frame, this->pitch, this->dataHeight, + tex->map, this->pitch)) + framebuffer_write(frame, tex->map, + this->pitch * this->dataHeight); } else { - rectsBufferToFramebuffer(damage->rects, damage->count, this->bpp, frame, - this->pitch, this->dataHeight, tex->map, this->pitch); + if (!rectsBufferToFramebuffer(damage->rects, damage->count, + this->bpp, frame, this->pitch, this->dataHeight, + tex->map, this->pitch)) + framebuffer_write(frame, tex->map, + this->pitch * this->dataHeight); } } }