diff --git a/desktop-shell/shell.c b/desktop-shell/shell.c
index 137ec7a..e3ef66f 100644
--- a/desktop-shell/shell.c
+++ b/desktop-shell/shell.c
@@ -198,6 +198,7 @@
 
 struct weston_touch_move_grab {
 	struct shell_touch_grab base;
+	int active;
 	wl_fixed_t dx, dy;
 };
 
@@ -1313,6 +1314,9 @@
 		(struct weston_touch_move_grab *) container_of(
 			grab, struct shell_touch_grab, grab);
 
+	if (touch_id == 0)
+		move->active = 0;
+
 	if (grab->touch->num_tp == 0) {
 		shell_touch_grab_end(&move->base);
 		free(move);
@@ -1329,7 +1333,7 @@
 	int dx = wl_fixed_to_int(grab->touch->grab_x + move->dx);
 	int dy = wl_fixed_to_int(grab->touch->grab_y + move->dy);
 
-	if (!shsurf)
+	if (!shsurf || !move->active)
 		return;
 
 	es = shsurf->surface;
@@ -1374,6 +1378,7 @@
 	if (!move)
 		return -1;
 
+	move->active = 1;
 	move->dx = wl_fixed_from_double(shsurf->view->geometry.x) -
 			seat->touch->grab_x;
 	move->dy = wl_fixed_from_double(shsurf->view->geometry.y) -