diff --git a/manimlib/scene/interactive_scene.py b/manimlib/scene/interactive_scene.py
index fe794bc3..855eb1c4 100644
--- a/manimlib/scene/interactive_scene.py
+++ b/manimlib/scene/interactive_scene.py
@@ -126,7 +126,7 @@ class InteractiveScene(Scene):
 
     def update_selection_rectangle(self, rect: Rectangle):
         p1 = rect.fixed_corner
-        p2 = self.mouse_point.get_center()
+        p2 = self.frame.to_fixed_frame_point(self.mouse_point.get_center())
         rect.set_points_as_corners([
             p1, np.array([p2[0], p1[1], 0]),
             p2, np.array([p1[0], p2[1], 0]),
@@ -377,7 +377,9 @@ class InteractiveScene(Scene):
     def enable_selection(self):
         self.is_selecting = True
         self.add(self.selection_rectangle)
-        self.selection_rectangle.fixed_corner = self.mouse_point.get_center().copy()
+        self.selection_rectangle.fixed_corner = self.frame.to_fixed_frame_point(
+            self.mouse_point.get_center()
+        )
 
     def gather_new_selection(self):
         self.is_selecting = False
@@ -568,7 +570,8 @@ class InteractiveScene(Scene):
 
     def on_mouse_motion(self, point: np.ndarray, d_point: np.ndarray) -> None:
         super().on_mouse_motion(point, d_point)
-        self.crosshair.move_to(point)
+        ff_point = self.frame.to_fixed_frame_point(point)
+        self.crosshair.move_to(ff_point)
         if self.is_grabbing:
             self.handle_grabbing(point)
         elif self.window.is_key_pressed(ord(RESIZE_KEY)):
diff --git a/manimlib/scene/scene.py b/manimlib/scene/scene.py
index ba617027..a4592c48 100644
--- a/manimlib/scene/scene.py
+++ b/manimlib/scene/scene.py
@@ -58,7 +58,7 @@ QUIT_KEY = 'q'
 
 class Scene(object):
     random_seed: int = 0
-    pan_sensitivity: float = 3.0
+    pan_sensitivity: float = 0.5
     max_num_saved_states: int = 50
     default_camera_config: dict = dict()
     default_window_config: dict = dict()
@@ -813,16 +813,13 @@ class Scene(object):
         frame = self.camera.frame
         # Handle perspective changes
         if self.window.is_key_pressed(ord(PAN_3D_KEY)):
-            frame.increment_theta(-self.pan_sensitivity * d_point[0])
-            frame.increment_phi(self.pan_sensitivity * d_point[1])
+            ff_d_point = frame.to_fixed_frame_point(d_point)
+            ff_d_point *= self.pan_sensitivity
+            frame.increment_theta(-ff_d_point[0])
+            frame.increment_phi(ff_d_point[1])
         # Handle frame movements
         elif self.window.is_key_pressed(ord(FRAME_SHIFT_KEY)):
-            shift = -d_point
-            shift[0] *= frame.get_width() / 2
-            shift[1] *= frame.get_height() / 2
-            transform = frame.get_inverse_camera_rotation_matrix()
-            shift = np.dot(np.transpose(transform), shift)
-            frame.shift(shift)
+            frame.shift(-d_point)
 
     def on_mouse_drag(
         self,
diff --git a/manimlib/window.py b/manimlib/window.py
index 8006b382..178250c1 100644
--- a/manimlib/window.py
+++ b/manimlib/window.py
@@ -77,17 +77,17 @@ class Window(PygletWindow):
         py: int,
         relative: bool = False
     ) -> np.ndarray:
-        pw, ph = self.size
-        fw, fh = self.scene.camera.get_frame_shape()
-        fc = self.scene.camera.get_frame_center()
+        pixel_shape = np.array(self.size)
+        frame_shape = np.array(self.scene.frame.get_shape())
+
+        coords = (frame_shape / pixel_shape) * np.array([px, py])
+        view = self.scene.frame.get_view_matrix()
+
         if relative:
-            return np.array([px / pw, py / ph, 0])
-        else:
-            return np.array([
-                fc[0] + px * fw / pw - fw / 2,
-                fc[1] + py * fh / ph - fh / 2,
-                0
-            ])
+            return np.dot([*coords, 0], view[:3, :3])
+
+        coords -= 0.5 * frame_shape
+        return np.dot([*coords, 0, 1], np.linalg.inv(view).T)[:3]
 
     def on_mouse_motion(self, x: int, y: int, dx: int, dy: int) -> None:
         super().on_mouse_motion(x, y, dx, dy)