diff --git a/editor/src/messages/tool/tool_messages/select_tool.rs b/editor/src/messages/tool/tool_messages/select_tool.rs
index 96b5a5737c..598523f121 100644
--- a/editor/src/messages/tool/tool_messages/select_tool.rs
+++ b/editor/src/messages/tool/tool_messages/select_tool.rs
@@ -251,7 +251,7 @@ impl ToolTransition for SelectTool {
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 enum SelectToolFsmState {
 	Ready { selection: NestedSelectionBehavior },
-	DrawingBox { selection: NestedSelectionBehavior, previous_selected: bool },
+	DrawingBox { selection: NestedSelectionBehavior },
 	Dragging,
 	ResizingBounds,
 	RotatingBounds,
@@ -283,6 +283,7 @@ struct SelectToolData {
 	snap_candidates: Vec<SnapCandidatePoint>,
 	auto_panning: AutoPanning,
 	previously_selected_layers: Vec<LayerNodeIdentifier>,
+	previous_selected: bool,
 }
 
 impl SelectToolData {
@@ -650,12 +651,12 @@ impl Fsm for SelectToolFsmState {
 				// Dragging a selection box
 				else {
 					tool_data.layers_dragging = selected;
-					let mut previous_selected = true;
+					tool_data.previous_selected = true;
 
 					if !input.keyboard.key(extend_selection) {
 						responses.add(DocumentMessage::DeselectAllLayers);
 						tool_data.layers_dragging.clear();
-						previous_selected = false
+						tool_data.previous_selected = false;
 					}
 
 					if let Some(intersection) = intersection {
@@ -673,7 +674,7 @@ impl Fsm for SelectToolFsmState {
 						} else {
 							// Make a box selection, preserving previously selected layers
 							let selection = tool_data.nested_selection_behavior;
-							SelectToolFsmState::DrawingBox { selection, previous_selected }
+							SelectToolFsmState::DrawingBox { selection }
 						}
 					};
 				tool_data.non_duplicated_layers = None;
@@ -820,8 +821,10 @@ impl Fsm for SelectToolFsmState {
 
 				SelectToolFsmState::DraggingPivot
 			}
-			(SelectToolFsmState::DrawingBox { selection: _, previous_selected }, SelectToolMessage::PointerMove(modifier_keys)) => {
+			(SelectToolFsmState::DrawingBox { .. }, SelectToolMessage::PointerMove(modifier_keys)) => {
+				tool_data.previous_selected = true;
 				tool_data.drag_current = input.mouse.position;
+
 				responses.add(OverlaysMessage::Draw);
 
 				// AutoPanning
@@ -832,7 +835,7 @@ impl Fsm for SelectToolFsmState {
 				tool_data.auto_panning.setup_by_mouse_position(input, &messages, responses);
 
 				let selection = tool_data.nested_selection_behavior;
-				SelectToolFsmState::DrawingBox { selection, previous_selected }
+				SelectToolFsmState::DrawingBox { selection }
 			}
 			(SelectToolFsmState::Ready { .. }, SelectToolMessage::PointerMove(_)) => {
 				let mut cursor = tool_data.bounding_box_manager.as_ref().map_or(MouseCursorIcon::Default, |bounds| bounds.get_cursor(input, true));
@@ -1102,14 +1105,14 @@ impl Fsm for SelectToolFsmState {
 
 				self
 			}
-			(SelectToolFsmState::DrawingBox { selection, mut previous_selected }, SelectToolMessage::RestoreSelection) => {
+			(SelectToolFsmState::DrawingBox { selection }, SelectToolMessage::RestoreSelection) => {
 				let selection_set: HashSet<_> = tool_data.previously_selected_layers.iter().collect();
 
 				if input.keyboard.key(Key::Shift) {
 					tool_data.layers_dragging.extend(&tool_data.previously_selected_layers);
-					previous_selected = false;
+					tool_data.previous_selected = false;
 				} else {
-					if previous_selected {
+					if tool_data.previous_selected {
 						tool_data.layers_dragging.extend(&tool_data.previously_selected_layers);
 					} else {
 						tool_data.layers_dragging.retain(|layer| !selection_set.contains(layer));
@@ -1133,7 +1136,7 @@ impl Fsm for SelectToolFsmState {
 
 				responses.add(OverlaysMessage::Draw);
 
-				SelectToolFsmState::DrawingBox { selection, previous_selected }
+				SelectToolFsmState::DrawingBox { selection }
 			}
 			_ => self,
 		}