diff --git a/src/xfdesktop-icon-view.c b/src/xfdesktop-icon-view.c
index 6e2566bb..dde8831c 100644
--- a/src/xfdesktop-icon-view.c
+++ b/src/xfdesktop-icon-view.c
@@ -870,15 +870,13 @@ xfdesktop_icon_view_button_press(GtkWidget *widget,
             } else {
                 /* clicked a non-selected icon */
                 if(icon_view->priv->sel_mode != GTK_SELECTION_MULTIPLE
-                   || !(evt->state & GDK_CONTROL_MASK))
+                   || !(evt->state & GDK_CONTROL_MASK || evt->state & GDK_SHIFT_MASK))
                 {
                     /* unselect all of the other icons if we haven't held
-                     * down the ctrl key.  we'll handle shift in the next block,
-                     * but for shift we do need to unselect everything */
+                     * down the ctrl or shift key. */
                     xfdesktop_icon_view_unselect_all(icon_view);
 
-                    if(!(evt->state & GDK_SHIFT_MASK))
-                        icon_view->priv->first_clicked_item = NULL;
+                    icon_view->priv->first_clicked_item = NULL;
                 }
 
                 icon_view->priv->cursor = icon;
@@ -913,7 +911,7 @@ xfdesktop_icon_view_button_press(GtkWidget *widget,
             /* Button press wasn't over any icons */
             /* unselect previously selected icons if we didn't click one */
             if(icon_view->priv->sel_mode != GTK_SELECTION_MULTIPLE
-               || !(evt->state & GDK_CONTROL_MASK))
+               || !(evt->state & GDK_CONTROL_MASK || evt->state & GDK_SHIFT_MASK))
             {
                 xfdesktop_icon_view_unselect_all(icon_view);
             }
@@ -921,9 +919,7 @@ xfdesktop_icon_view_button_press(GtkWidget *widget,
             icon_view->priv->cursor = NULL;
             icon_view->priv->first_clicked_item = NULL;
 
-            if(icon_view->priv->allow_rubber_banding && evt->button == 1
-               && !(evt->state & GDK_SHIFT_MASK))
-            {
+            if(icon_view->priv->allow_rubber_banding && evt->button == 1) {
                 icon_view->priv->maybe_begin_drag = TRUE;
                 icon_view->priv->definitely_dragging = FALSE;
                 icon_view->priv->press_start_x = evt->x;
@@ -932,7 +928,7 @@ xfdesktop_icon_view_button_press(GtkWidget *widget,
 
             /* Since we're not over any icons this won't be the start of a
              * drag so we can pop up menu */
-            if(evt->button == 3 || (evt->button == 1 && (evt->state & GDK_SHIFT_MASK))) {
+            if(evt->button == 3) {
                 xfce_desktop_popup_root_menu(XFCE_DESKTOP(widget), evt->button, evt->time);
                 return TRUE;
             }
@@ -1023,7 +1019,7 @@ xfdesktop_icon_view_button_release(GtkWidget *widget,
         }
     }
 
-    if((evt->button == 3 || (evt->button == 1 && (evt->state & GDK_SHIFT_MASK))) &&
+    if((evt->button == 3) &&
        icon_view->priv->definitely_dragging == FALSE &&
        icon_view->priv->definitely_rubber_banding == FALSE &&
        icon_view->priv->maybe_begin_drag == TRUE)
@@ -1241,6 +1237,7 @@ xfdesktop_icon_view_motion_notify(GtkWidget *widget,
         GdkRectangle old_rect, *new_rect, intersect;
         cairo_region_t *region;
         GList *l;
+        gboolean ctrl_click = evt->state & GDK_CONTROL_MASK;
 
         /* we're dragging with no icon under the cursor -> rubber band start
          * OR, we're already doin' the band -> update it */
@@ -1285,13 +1282,11 @@ xfdesktop_icon_view_motion_notify(GtkWidget *widget,
         /* update list of selected icons */
 
         /* first pass: if the rubber band area got smaller at least in
-         * one dimension, we can try first to just remove icons that
-         * aren't in the band anymore */
+         * one dimension */
         if(old_rect.width > new_rect->width
            || old_rect.height > new_rect->height)
         {
-            l = icon_view->priv->selected_icons;
-            while(l) {
+            for(l = icon_view->priv->icons; l; l = l->next) {
                 GdkRectangle extents, dummy;
                 XfdesktopIcon *icon = l->data;
 
@@ -1302,16 +1297,15 @@ xfdesktop_icon_view_motion_notify(GtkWidget *widget,
                    && gdk_rectangle_intersect(&extents, &old_rect, NULL)
                    && !gdk_rectangle_intersect(&extents, new_rect, &dummy))
                 {
-                    /* remove the icon from the selected list */
-                    l = l->next;
-                    xfdesktop_icon_view_unselect_item(icon_view, icon);
-                } else
-                    l = l->next;
+                    if(ctrl_click && !xfdesktop_icon_view_is_icon_selected(icon_view, icon))
+                        xfdesktop_icon_view_select_item(icon_view, icon);
+                    else
+                        xfdesktop_icon_view_unselect_item(icon_view, icon);
+                }
             }
         }
 
-        /* second pass: if at least one dimension got larger, unfortunately
-         * we have to figure out what icons to add to the selected list */
+        /* second pass: if at least one dimension got larger */
         if(old_rect.width < new_rect->width
            || old_rect.height < new_rect->height)
         {
@@ -1320,12 +1314,13 @@ xfdesktop_icon_view_motion_notify(GtkWidget *widget,
                 XfdesktopIcon *icon = l->data;
 
                 if(xfdesktop_icon_get_extents(icon, NULL, NULL, &extents)
-                   && gdk_rectangle_intersect(&extents, new_rect, &dummy)
-                   && !xfdesktop_icon_view_is_icon_selected(icon_view, icon))
+                   && !gdk_rectangle_intersect(&extents, &old_rect, NULL)
+                   && gdk_rectangle_intersect(&extents, new_rect, &dummy))
                 {
-                    /* since _select_item() prepends to the list, we
-                     * should be ok just calling this */
-                    xfdesktop_icon_view_select_item(icon_view, icon);
+                    if(ctrl_click && xfdesktop_icon_view_is_icon_selected(icon_view, icon))
+                        xfdesktop_icon_view_unselect_item(icon_view, icon);
+                    else
+                        xfdesktop_icon_view_select_item(icon_view, icon);
                 }
             }
         }