diff --git a/gtk/gtklabel.c b/gtk/gtklabel.c
index 7933ab2af7..468ef2ae87 100644
--- a/gtk/gtklabel.c
+++ b/gtk/gtklabel.c
@@ -5222,7 +5222,8 @@ gtk_label_drag_gesture_begin (GtkGestureDrag *gesture,
     {
       if (min < max && min <= index && index <= max)
         {
-          info->in_drag = TRUE;
+          if (!info->select_words)
+            info->in_drag = TRUE;
           info->drag_start_x = start_x;
           info->drag_start_y = start_y;
         }