diff --git a/client/src/main.c b/client/src/main.c
index 66d04b6a..516dee5b 100644
--- a/client/src/main.c
+++ b/client/src/main.c
@@ -1020,6 +1020,8 @@ void app_handleButtonPress(int button)
   if (!app_inputEnabled() || !g_cursor.inView)
     return;
 
+  g_cursor.buttons |= (1U << button);
+
   if (!spice_mouse_press(button))
     DEBUG_ERROR("SDL_MOUSEBUTTONDOWN: failed to send message");
 }
@@ -1029,6 +1031,8 @@ void app_handleButtonRelease(int button)
   if (!app_inputEnabled())
     return;
 
+  g_cursor.buttons &= ~(1U << button);
+
   if (!spice_mouse_release(button))
     DEBUG_ERROR("SDL_MOUSEBUTTONUP: failed to send message");
 }
@@ -1107,6 +1111,10 @@ void app_handleMouseNormal(double ex, double ey)
   struct DoublePoint local;
   guestCurToLocal(&local);
 
+  /* if any buttons are held we should not allow exit to happen */
+  if (g_cursor.buttons)
+    testExit = false;
+
   /* check if the move would push the cursor outside the guest's viewport */
   if (testExit && (
       local.x + ex <  g_state.dstRect.x ||
diff --git a/client/src/main.h b/client/src/main.h
index 8c369da2..8935abb9 100644
--- a/client/src/main.h
+++ b/client/src/main.h
@@ -226,6 +226,9 @@ struct CursorState
   /* the local position */
   struct DoublePoint pos;
 
+  /* the button state */
+  unsigned int buttons;
+
   /* the delta since last warp when in auto capture mode */
   struct DoublePoint delta;