[host] nvfbc: move NvFBCToSysCreate into nvfbc_init

When NvFBCToSysCapture reports recreation is required, we return
CAPTURE_RESULT_REINIT, which eventually calls nvfbc_deinit and then
nvfbc_init.

However, the NvFBC object is actually created in nvfbc_create, which
means the NvFBC object is never actually recreated. The result is an
endless cycle of NvFBC asking for recreation. This commonly manifests
as the client waiting endlessly for the host when the guest machine
reboots.

In this commit, the NvFBC object creation is moved into nvfbc_init,
and when recreation is required, it will actually be recreated.
This commit is contained in:
Quantum 2021-01-30 18:50:36 -05:00 committed by Geoffrey McRae
parent acc3298344
commit a702c912ae

View File

@ -119,6 +119,26 @@ static bool nvfbc_create(
if (!NvFBCInit())
return false;
this = (struct iface *)calloc(sizeof(struct iface), 1);
this->frameEvent = lgCreateEvent(true, 17);
if (!this->frameEvent)
{
DEBUG_ERROR("failed to create the frame event");
nvfbc_free();
return false;
}
this->seperateCursor = option_get_bool("nvfbc", "decoupleCursor");
this->getPointerBufferFn = getPointerBufferFn;
this->postPointerBufferFn = postPointerBufferFn;
return true;
}
static bool nvfbc_init(void)
{
this->stop = false;
int bufferLen = GetEnvironmentVariable("NVFBC_PRIV_DATA", NULL, 0);
uint8_t * privData = NULL;
int privDataLen = 0;
@ -140,33 +160,13 @@ static bool nvfbc_create(
free(buffer);
}
this = (struct iface *)calloc(sizeof(struct iface), 1);
if (!NvFBCToSysCreate(privData, privDataLen, &this->nvfbc, &this->maxWidth, &this->maxHeight))
{
free(privData);
nvfbc_free();
return false;
}
free(privData);
this->frameEvent = lgCreateEvent(true, 17);
if (!this->frameEvent)
{
DEBUG_ERROR("failed to create the frame event");
nvfbc_free();
return false;
}
this->seperateCursor = option_get_bool("nvfbc", "decoupleCursor");
this->getPointerBufferFn = getPointerBufferFn;
this->postPointerBufferFn = postPointerBufferFn;
return true;
}
static bool nvfbc_init(void)
{
this->stop = false;
getDesktopSize(&this->width, &this->height, &this->dpi);
lgResetEvent(this->frameEvent);
@ -235,13 +235,17 @@ static bool nvfbc_deinit(void)
this->cursorEvents[0] = NULL;
}
if (this->nvfbc)
{
NvFBCToSysRelease(&this->nvfbc);
this->nvfbc = NULL;
}
return true;
}
static void nvfbc_free(void)
{
NvFBCToSysRelease(&this->nvfbc);
if (this->frameEvent)
lgFreeEvent(this->frameEvent);