diff --git a/gtk/gtkheaderbar.c b/gtk/gtkheaderbar.c
index 30423b9d48..786dfb754e 100644
--- a/gtk/gtkheaderbar.c
+++ b/gtk/gtkheaderbar.c
@@ -743,8 +743,17 @@ gtk_header_bar_set_show_title_buttons (GtkHeaderBar *bar,
     create_window_controls (bar);
   else
     {
-      g_clear_pointer (&bar->start_window_controls, gtk_widget_unparent);
-      g_clear_pointer (&bar->end_window_controls, gtk_widget_unparent);
+      if (bar->start_box && bar->start_window_controls)
+        {
+          gtk_box_remove (GTK_BOX (bar->start_box), bar->start_window_controls);
+          bar->start_window_controls = NULL;
+        }
+
+      if (bar->end_box && bar->end_window_controls)
+        {
+          gtk_box_remove (GTK_BOX (bar->end_box), bar->end_window_controls);
+          bar->end_window_controls = NULL;
+        }
     }
 
   g_object_notify_by_pspec (G_OBJECT (bar), header_bar_props[PROP_SHOW_TITLE_BUTTONS]);