diff --git a/src/pathfinding.rs b/src/pathfinding.rs
index 5056ccc..b36722c 100644
--- a/src/pathfinding.rs
+++ b/src/pathfinding.rs
@@ -108,7 +108,7 @@ fn calculate_path<D: 'static + Clone + Default + Send + Sync>(
     mut commands: Commands,
     pool: Res<AsyncComputeTaskPool>,
     rapier_context: Res<RapierContext>,
-    obstructions: Query<&MapObstruction>,
+    obstructions: Query<&RapierColliderHandle, With<MapObstruction>>,
     query: Query<
         (
             Entity,
@@ -138,7 +138,24 @@ fn calculate_path<D: 'static + Clone + Default + Send + Sync>(
             let query_pipeline = rapier_context.query_pipeline.clone();
             let map_clone = map.clone();
             let handle_clone = *handle;
-            let collider_set = rapier_context.colliders.clone();
+            let mut collider_set = rapier_context.colliders.clone();
+            let obstructions = obstructions
+                .iter()
+                .map(|v| v.0)
+                .collect::<Vec<ColliderHandle>>();
+            let mut to_remove = vec![];
+            for handle in collider_set.iter() {
+                if !obstructions.contains(&handle.0) {
+                    to_remove.push(handle.0);
+                }
+            }
+            if !to_remove.is_empty() {
+                let mut islands = rapier_context.islands.clone();
+                let mut bodies = rapier_context.bodies.clone();
+                for handle in to_remove {
+                    collider_set.remove(handle, &mut islands, &mut bodies, false);
+                }
+            }
             let shape_clone = (*shape).clone();
             let task = pool.spawn(async move {
                 find_path_for_shape(