Eye Tracking Support, Minecraft Without Hands

Add support for eye tracking hardware, eg the Tobii 5.
Add settings to toggle eye tracking in the settings menu.
Add settings to adjust the deadzone while looking at the crosshair.
Disable mouse raw input while eye tracking is in use, as these two modes are not compatible.

src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsConfig.java

@@ -64,6 +64,8 @@ public class MidnightControlsConfig extends MidnightConfig {
     @Entry(category = "controller", name = "midnightcontrols.menu.y_axis_rotation_speed", isSlider = true, min = 0, max = 100, precision = 10) public static double yAxisRotationSpeed = rotationSpeed;
     @Entry(category = "screens", name = "midnightcontrols.menu.mouse_speed", isSlider = true, min = 0, max = 150, precision = 10) public static double mouseSpeed = 25.0;
     @Entry(category = "screens", name = "midnightcontrols.menu.joystick_as_mouse") public static boolean joystickAsMouse = false;
+    @Entry(category = "screens", name = "midnightcontrols.menu.eye_tracker_as_mouse") public static boolean eyeTrackerAsMouse = false;
+    @Entry(category = "screens", name = "midnightcontrols.menu.eye_tracker_deadzone", isSlider = true, min = 0, max = 0.4) public static double eyeTrackerDeadzone = 0.05;
     @Entry(category = "controller", name = "midnightcontrols.menu.unfocused_input") public static boolean unfocusedInput = false;
     @Entry(category = "screens", name = "midnightcontrols.menu.virtual_mouse") public static boolean virtualMouse = false;
     @Entry(category = "screens", name = "midnightcontrols.menu.virtual_mouse.skin") public static VirtualMouseSkin virtualMouseSkin = VirtualMouseSkin.DEFAULT_LIGHT;

src/main/java/eu/midnightdust/midnightcontrols/client/gui/MidnightControlsSettingsScreen.java

@@ -59,6 +59,8 @@ public class MidnightControlsSettingsScreen extends SpruceScreen {
     private final SpruceOption yAxisRotationSpeedOption;
     private final SpruceOption mouseSpeedOption;
     private final SpruceOption joystickAsMouseOption;
+    private final SpruceOption eyeTrackingAsMouseOption;
+    private final SpruceOption eyeTrackingDeadzone;
     private final SpruceOption virtualMouseOption;
     private final SpruceOption hideCursorOption;
     private final SpruceOption resetOption;
@@ -193,6 +195,12 @@ public class MidnightControlsSettingsScreen extends SpruceScreen {
         this.joystickAsMouseOption = new SpruceToggleBooleanOption("midnightcontrols.menu.joystick_as_mouse",
                 () -> MidnightControlsConfig.joystickAsMouse, value -> MidnightControlsConfig.joystickAsMouse = value,
                 Text.translatable("midnightcontrols.menu.joystick_as_mouse.tooltip"));
+        this.eyeTrackingAsMouseOption = new SpruceToggleBooleanOption("midnightcontrols.menu.eye_tracker_as_mouse",
+                () -> MidnightControlsConfig.eyeTrackerAsMouse, value -> MidnightControlsConfig.eyeTrackerAsMouse = value,
+                Text.translatable("midnightcontrols.menu.eye_tracker_as_mouse.tooltip"));
+        this.eyeTrackingDeadzone = new SpruceDoubleInputOption("midnightcontrols.menu.eye_tracker_deadzone",
+                () -> MidnightControlsConfig.eyeTrackerDeadzone, value -> MidnightControlsConfig.eyeTrackerDeadzone = value,
+                Text.translatable("midnightcontrols.menu.eye_tracker_deadzone.tooltip"));
         this.resetOption = SpruceSimpleActionOption.reset(btn -> {
             MidnightControlsConfig.reset();
             var client = MinecraftClient.getInstance();
@@ -344,6 +352,7 @@ public class MidnightControlsSettingsScreen extends SpruceScreen {
         list.addSingleOptionEntry(this.virtualMouseOption);
         list.addSingleOptionEntry(this.hideCursorOption);
         list.addSingleOptionEntry(this.joystickAsMouseOption);
+        list.addSingleOptionEntry(this.eyeTrackingAsMouseOption);
         list.addSingleOptionEntry(this.advancedConfigOption);
         return list;
     }

src/main/java/eu/midnightdust/midnightcontrols/client/mixin/InputUtilMixin.java

@@ -0,0 +1,39 @@
+package eu.midnightdust.midnightcontrols.client.mixin;
+
+import net.minecraft.client.util.InputUtil;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.Shadow;
+import org.spongepowered.asm.mixin.Overwrite;
+import org.spongepowered.asm.mixin.Final;
+import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
+import java.lang.invoke.MethodHandle;
+
+
+@Mixin(InputUtil.class)
+public abstract class InputUtilMixin {
+
+    @Final
+    @Shadow
+    private static MethodHandle GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE;
+
+    /**
+     * @author kabliz
+     * @reason This method is static, and there is a terrible UX issue if raw input is turned on at the same time as
+     * eye tracking. Raw input only tracks literal mice and not other devices, leading to the game appearing to be
+     * unresponsive and the player not understanding why. This overwrite preserves the user's mouse preferences,
+     * while not interfering with eye tracking, and the two modes can be switched between during a play session.
+     */
+    @Overwrite
+    public static boolean isRawMouseMotionSupported(){
+        if(MidnightControlsConfig.eyeTrackerAsMouse){
+             return false;
+        } else { //Paste original implementation from InputUtil below.
+            try {
+                return GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE != null &&
+                        (boolean) GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE.invokeExact();
+            } catch (Throwable var1) {
+                throw new RuntimeException(var1);
+            }
+        }
+    }
+}

src/main/java/eu/midnightdust/midnightcontrols/client/mixin/MouseMixin.java

@@ -15,7 +15,9 @@ import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
 import eu.midnightdust.midnightcontrols.client.util.MouseAccessor;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.client.Mouse;
-import net.minecraft.client.gui.screen.Screen;
+import net.minecraft.client.option.KeyBinding;
+import net.minecraft.client.util.GlfwUtil;
+import net.minecraft.client.util.SmoothUtil;
 import org.lwjgl.glfw.GLFW;
 import org.spongepowered.asm.mixin.Final;
 import org.spongepowered.asm.mixin.Mixin;
@@ -26,6 +28,10 @@ import org.spongepowered.asm.mixin.injection.At;
 import org.spongepowered.asm.mixin.injection.Inject;
 import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
 import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
+import eu.midnightdust.midnightcontrols.client.mouse.EyeTrackerHandler;
+
+import static org.lwjgl.glfw.GLFW.GLFW_CURSOR;
+import static org.lwjgl.glfw.GLFW.GLFW_CURSOR_HIDDEN;
 
 /**
  * Adds extra access to the mouse.
@@ -36,6 +42,37 @@ public abstract class MouseMixin implements MouseAccessor {
     @Final
     private MinecraftClient client;
 
+    @Shadow
+    private double y;
+
+    @Shadow
+    private double cursorDeltaX;
+
+    @Shadow
+    private double cursorDeltaY;
+
+    @Shadow
+    private double x;
+
+    @Shadow
+    private boolean cursorLocked;
+
+    @Shadow
+    private boolean hasResolutionChanged;
+
+    @Shadow
+    private double lastMouseUpdateTime;
+
+    @Shadow
+    @Final
+    private SmoothUtil cursorXSmoother;
+
+    @Shadow
+    @Final
+    private SmoothUtil cursorYSmoother;
+
+    @Shadow private boolean leftButtonClicked;
+
     @Accessor
     public abstract void setLeftButtonClicked(boolean value);
 
@@ -67,4 +104,37 @@ public abstract class MouseMixin implements MouseAccessor {
                 ||*/ (MidnightControlsConfig.controlsMode == ControlsMode.CONTROLLER && MidnightControlsConfig.virtualMouse))
             ci.cancel();
     }
+
+    @Inject(method = "updateMouse", at = @At("HEAD"), cancellable = true)
+    private void updateMouse(CallbackInfo ci) {
+        if (MidnightControlsConfig.eyeTrackerAsMouse && cursorLocked && client.isWindowFocused()) {
+            //Eye Tracking is only for the camera controlling cursor, we need the normal cursor everywhere else.
+            if (!client.options.smoothCameraEnabled) {
+                cursorXSmoother.clear();
+                cursorYSmoother.clear();
+            }
+            EyeTrackerHandler.updateMouseWithEyeTracking(x + cursorDeltaX, y + cursorDeltaY, client,
+                    lastMouseUpdateTime, leftButtonClicked, cursorXSmoother, cursorYSmoother);
+            lastMouseUpdateTime = GlfwUtil.getTime();
+            cursorDeltaX = 0.0;
+            cursorDeltaY = 0.0;
+            ci.cancel();
+        }
+    }
+
+    @Inject(method = "lockCursor", at = @At("HEAD"), cancellable = true)
+    private void lockCursor(CallbackInfo ci) {
+        if (MidnightControlsConfig.eyeTrackerAsMouse && client.isWindowFocused() && !this.cursorLocked) {
+            if (!MinecraftClient.IS_SYSTEM_MAC) {
+                KeyBinding.updatePressedStates();
+            }
+            //In eye tracking mode, we cannot have the cursor locked to the center.
+            GLFW.glfwSetInputMode(client.getWindow().getHandle(), GLFW_CURSOR, GLFW_CURSOR_HIDDEN);
+            cursorLocked = true; //The game uses this flag for other gameplay checks
+            client.setScreen(null);
+            hasResolutionChanged = true;
+            ci.cancel();
+        }
+    }
+
 }

src/main/java/eu/midnightdust/midnightcontrols/client/mouse/EyeTrackerHandler.java

@@ -0,0 +1,76 @@
+package eu.midnightdust.midnightcontrols.client.mouse;
+
+import net.minecraft.client.MinecraftClient;
+import net.minecraft.client.util.GlfwUtil;
+import net.minecraft.client.util.SmoothUtil;
+import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
+
+public class EyeTrackerHandler {
+
+    /**
+     * Based on the updateMouse method in the Mouse.class, this changes the mouse algorithm to suit eye tracking.
+     * This requires the cursor to not be locked, and the raw input setting to be turned off.
+     */
+    public static void updateMouseWithEyeTracking(double mouseX, double mouseY,
+                                                  MinecraftClient client,
+                                                  double lastMouseUpdateTime,
+                                                  boolean holdingLeftMouseButton,
+                                                  SmoothUtil smoothX,
+                                                  SmoothUtil smoothY
+                                             ) {
+        // The player wants objects of interest to be moved under the crosshair that is always center of screen.
+        // Normal mouse controls operate with the delta values from the direction of mouse movement,
+        // but in eye tracking we want to use the cursor's actual x,y values (their point of gaze), relative to
+        // the screen center (where the crosshair is). This new eye tracking delta creates a vector that points
+        // from the crosshair to the gaze point. As the player keeps their eyes on the object of interest, we pull
+        // that object into the center until the object is underneath the crosshair.
+        double timestamp = GlfwUtil.getTime();
+        double deltaTime = timestamp - lastMouseUpdateTime;
+
+        // The center of screen is the new (0,0)
+        double centerX = client.getWindow().getWidth() / 2.0;
+        double centerY = client.getWindow().getHeight() / 2.0;
+        double gazeRawX = mouseX - centerX;
+        double gazeRawY = mouseY - centerY;
+
+        //This part follows the original mouse.java somewhat closely, with different constants
+        double feeling = 2.5;
+        double sensitivity = client.options.getMouseSensitivity().getValue() * feeling;
+        double spyglass = sensitivity * sensitivity * sensitivity;
+        double moveScalar = spyglass * 8.0;
+
+        double frameScalar;
+        if(client.options.getPerspective().isFirstPerson() && client.player.isUsingSpyglass()) {
+            frameScalar = spyglass;
+        } else {
+            frameScalar = moveScalar;
+        }
+        if(holdingLeftMouseButton){
+            frameScalar *= 0.5; //Don't move the camera so much while mining. It's annoying.
+        }
+
+        // The longest vector connects the center to the corner of the screen, so that is our maximum magnitude for
+        // normalization. We use normalized screen size vector for resolution independent control
+        double magnitudeMax = Math.sqrt(centerX*centerX + centerY*centerY);
+        double normalizedX = gazeRawX / magnitudeMax;
+        double normalizedY = gazeRawY / magnitudeMax;
+
+        double moveX = normalizedX * frameScalar;
+        double moveY = normalizedY * frameScalar;
+        if (client.options.smoothCameraEnabled) {
+            moveX = smoothX.smooth(moveX, moveScalar*deltaTime);
+            moveY = smoothY.smooth(moveY, moveScalar*deltaTime);
+        }
+
+        // The player entity's needs their facing rotated.
+        double invertY = 1.0;
+        double moveMagnitude = Math.sqrt(normalizedX*normalizedX + normalizedY*normalizedY);
+        if (client.options.getInvertYMouse().getValue()) {
+            invertY = -1.0;
+        }
+        if (client.player != null && moveMagnitude > MidnightControlsConfig.eyeTrackerDeadzone) {
+            client.player.changeLookDirection(moveX, moveY * invertY);
+            client.getTutorialManager().onUpdateMouse(moveX, moveY);
+        }
+    }
+}

src/main/resources/assets/midnightcontrols/lang/en_us.json

@@ -153,6 +153,10 @@
   "midnightcontrols.menu.invert_right_y_axis": "Invert Right Y",
   "midnightcontrols.menu.joystick_as_mouse": "Always use left stick as mouse",
   "midnightcontrols.menu.joystick_as_mouse.tooltip": "Make the joystick behave like a mouse in every menu.",
+  "midnightcontrols.menu.eye_tracker_as_mouse": "Use Eye Tracker as Mouse",
+  "midnightcontrols.menu.eye_tracker_as_mouse.tooltip": "Replace the mouse with an eye tracking device, (for example) the Tobii 5.",
+  "midnightcontrols.menu.eye_tracker_deadzone": "Eye Tracker Deadzone Size",
+  "midnightcontrols.menu.eye_tracker_deadzone.tooltip": "Stops camera movement when looking near the cross hair",
   "midnightcontrols.menu.keyboard_controls": "Keyboard Controls...",
   "midnightcontrols.menu.left_dead_zone": "Left Stick Dead Zone",
   "midnightcontrols.menu.left_dead_zone.tooltip": "The dead zone for the controller's left analog stick.",

src/main/resources/midnightcontrols.mixins.json

@@ -13,6 +13,7 @@
     "GameRendererMixin",
     "HandledScreenMixin",
     "KeyBindingMixin",
+    "InputUtilMixin",
     "MinecraftClientMixin",
     "MouseMixin",
     "ChatScreenMixin",