Merge pull request #208 from kabliz/eyetracking

Eye Tracking Support, No-Hands Minecraft

src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsConfig.java

@@ -64,6 +64,8 @@ public class MidnightControlsConfig extends MidnightConfig {
     @Entry(category = "controller", name = "midnightcontrols.menu.y_axis_rotation_speed", isSlider = true, min = 0, max = 100, precision = 10) public static double yAxisRotationSpeed = rotationSpeed;
     @Entry(category = "screens", name = "midnightcontrols.menu.mouse_speed", isSlider = true, min = 0, max = 150, precision = 10) public static double mouseSpeed = 25.0;
     @Entry(category = "screens", name = "midnightcontrols.menu.joystick_as_mouse") public static boolean joystickAsMouse = false;
+    @Entry(category = "screens", name = "midnightcontrols.menu.eye_tracker_as_mouse") public static boolean eyeTrackerAsMouse = false;
+    @Entry(category = "screens", name = "midnightcontrols.menu.eye_tracker_deadzone", isSlider = true, min = 0, max = 0.4) public static double eyeTrackerDeadzone = 0.05;
     @Entry(category = "controller", name = "midnightcontrols.menu.unfocused_input") public static boolean unfocusedInput = false;
     @Entry(category = "screens", name = "midnightcontrols.menu.virtual_mouse") public static boolean virtualMouse = false;
     @Entry(category = "screens", name = "midnightcontrols.menu.virtual_mouse.skin") public static VirtualMouseSkin virtualMouseSkin = VirtualMouseSkin.DEFAULT_LIGHT;

src/main/java/eu/midnightdust/midnightcontrols/client/gui/MidnightControlsSettingsScreen.java

@@ -59,6 +59,8 @@ public class MidnightControlsSettingsScreen extends SpruceScreen {
     private final SpruceOption yAxisRotationSpeedOption;
     private final SpruceOption mouseSpeedOption;
     private final SpruceOption joystickAsMouseOption;
+    private final SpruceOption eyeTrackingAsMouseOption;
+    private final SpruceOption eyeTrackingDeadzone;
     private final SpruceOption virtualMouseOption;
     private final SpruceOption hideCursorOption;
     private final SpruceOption resetOption;
@@ -193,6 +195,12 @@ public class MidnightControlsSettingsScreen extends SpruceScreen {
         this.joystickAsMouseOption = new SpruceToggleBooleanOption("midnightcontrols.menu.joystick_as_mouse",
                 () -> MidnightControlsConfig.joystickAsMouse, value -> MidnightControlsConfig.joystickAsMouse = value,
                 Text.translatable("midnightcontrols.menu.joystick_as_mouse.tooltip"));
+        this.eyeTrackingAsMouseOption = new SpruceToggleBooleanOption("midnightcontrols.menu.eye_tracker_as_mouse",
+                () -> MidnightControlsConfig.eyeTrackerAsMouse, value -> MidnightControlsConfig.eyeTrackerAsMouse = value,
+                Text.translatable("midnightcontrols.menu.eye_tracker_as_mouse.tooltip"));
+        this.eyeTrackingDeadzone = new SpruceDoubleInputOption("midnightcontrols.menu.eye_tracker_deadzone",
+                () -> MidnightControlsConfig.eyeTrackerDeadzone, value -> MidnightControlsConfig.eyeTrackerDeadzone = value,
+                Text.translatable("midnightcontrols.menu.eye_tracker_deadzone.tooltip"));
         this.resetOption = SpruceSimpleActionOption.reset(btn -> {
             MidnightControlsConfig.reset();
             var client = MinecraftClient.getInstance();
@@ -344,6 +352,7 @@ public class MidnightControlsSettingsScreen extends SpruceScreen {
         list.addSingleOptionEntry(this.virtualMouseOption);
         list.addSingleOptionEntry(this.hideCursorOption);
         list.addSingleOptionEntry(this.joystickAsMouseOption);
+        list.addSingleOptionEntry(this.eyeTrackingAsMouseOption);
         list.addSingleOptionEntry(this.advancedConfigOption);
         return list;
     }

src/main/java/eu/midnightdust/midnightcontrols/client/mixin/InputUtilMixin.java

@@ -0,0 +1,39 @@
+package eu.midnightdust.midnightcontrols.client.mixin;
+
+import net.minecraft.client.util.InputUtil;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.Shadow;
+import org.spongepowered.asm.mixin.Overwrite;
+import org.spongepowered.asm.mixin.Final;
+import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
+import java.lang.invoke.MethodHandle;
+
+
+@Mixin(InputUtil.class)
+public abstract class InputUtilMixin {
+
+    @Final
+    @Shadow
+    private static MethodHandle GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE;
+
+    /**
+     * @author kabliz
+     * @reason This method is static, and there is a terrible UX issue if raw input is turned on at the same time as
+     * eye tracking. Raw input only tracks literal mice and not other devices, leading to the game appearing to be
+     * unresponsive and the player not understanding why. This overwrite preserves the user's mouse preferences,
+     * while not interfering with eye tracking, and the two modes can be switched between during a play session.
+     */
+    @Overwrite
+    public static boolean isRawMouseMotionSupported(){
+        if(MidnightControlsConfig.eyeTrackerAsMouse){
+             return false;
+        } else { //Paste original implementation from InputUtil below.
+            try {
+                return GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE != null &&
+                        (boolean) GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE.invokeExact();
+            } catch (Throwable var1) {
+                throw new RuntimeException(var1);
+            }
+        }
+    }
+}

src/main/java/eu/midnightdust/midnightcontrols/client/mixin/MouseMixin.java

@@ -15,7 +15,9 @@ import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
 import eu.midnightdust.midnightcontrols.client.util.MouseAccessor;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.client.Mouse;
-import net.minecraft.client.gui.screen.Screen;
+import net.minecraft.client.option.KeyBinding;
+import net.minecraft.client.util.GlfwUtil;
+import net.minecraft.client.util.SmoothUtil;
 import org.lwjgl.glfw.GLFW;
 import org.spongepowered.asm.mixin.Final;
 import org.spongepowered.asm.mixin.Mixin;
@@ -26,6 +28,10 @@ import org.spongepowered.asm.mixin.injection.At;
 import org.spongepowered.asm.mixin.injection.Inject;
 import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
 import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
+import eu.midnightdust.midnightcontrols.client.mouse.EyeTrackerHandler;
+
+import static org.lwjgl.glfw.GLFW.GLFW_CURSOR;
+import static org.lwjgl.glfw.GLFW.GLFW_CURSOR_HIDDEN;
 
 /**
  * Adds extra access to the mouse.
@@ -36,6 +42,37 @@ public abstract class MouseMixin implements MouseAccessor {
     @Final
     private MinecraftClient client;
 
+    @Shadow
+    private double y;
+
+    @Shadow
+    private double cursorDeltaX;
+
+    @Shadow
+    private double cursorDeltaY;
+
+    @Shadow
+    private double x;
+
+    @Shadow
+    private boolean cursorLocked;
+
+    @Shadow
+    private boolean hasResolutionChanged;
+
+    @Shadow
+    private double lastMouseUpdateTime;
+
+    @Shadow
+    @Final
+    private SmoothUtil cursorXSmoother;
+
+    @Shadow
+    @Final
+    private SmoothUtil cursorYSmoother;
+
+    @Shadow private boolean leftButtonClicked;
+
     @Accessor
     public abstract void setLeftButtonClicked(boolean value);
 
@@ -67,4 +104,37 @@ public abstract class MouseMixin implements MouseAccessor {
                 ||*/ (MidnightControlsConfig.controlsMode == ControlsMode.CONTROLLER && MidnightControlsConfig.virtualMouse))
             ci.cancel();
     }
+
+    @Inject(method = "updateMouse", at = @At("HEAD"), cancellable = true)
+    private void updateMouse(CallbackInfo ci) {
+        if (MidnightControlsConfig.eyeTrackerAsMouse && cursorLocked && client.isWindowFocused()) {
+            //Eye Tracking is only for the camera controlling cursor, we need the normal cursor everywhere else.
+            if (!client.options.smoothCameraEnabled) {
+                cursorXSmoother.clear();
+                cursorYSmoother.clear();
+            }
+            EyeTrackerHandler.updateMouseWithEyeTracking(x + cursorDeltaX, y + cursorDeltaY, client,
+                    lastMouseUpdateTime, leftButtonClicked, cursorXSmoother, cursorYSmoother);
+            lastMouseUpdateTime = GlfwUtil.getTime();
+            cursorDeltaX = 0.0;
+            cursorDeltaY = 0.0;
+            ci.cancel();
+        }
+    }
+
+    @Inject(method = "lockCursor", at = @At("HEAD"), cancellable = true)
+    private void lockCursor(CallbackInfo ci) {
+        if (MidnightControlsConfig.eyeTrackerAsMouse && client.isWindowFocused() && !this.cursorLocked) {
+            if (!MinecraftClient.IS_SYSTEM_MAC) {
+                KeyBinding.updatePressedStates();
+            }
+            //In eye tracking mode, we cannot have the cursor locked to the center.
+            GLFW.glfwSetInputMode(client.getWindow().getHandle(), GLFW_CURSOR, GLFW_CURSOR_HIDDEN);
+            cursorLocked = true; //The game uses this flag for other gameplay checks
+            client.setScreen(null);
+            hasResolutionChanged = true;
+            ci.cancel();
+        }
+    }
+
 }

src/main/java/eu/midnightdust/midnightcontrols/client/mouse/EyeTrackerHandler.java

@@ -0,0 +1,76 @@
+package eu.midnightdust.midnightcontrols.client.mouse;
+
+import net.minecraft.client.MinecraftClient;
+import net.minecraft.client.util.GlfwUtil;
+import net.minecraft.client.util.SmoothUtil;
+import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
+
+public class EyeTrackerHandler {
+
+    /**
+     * Based on the updateMouse method in the Mouse.class, this changes the mouse algorithm to suit eye tracking.
+     * This requires the cursor to not be locked, and the raw input setting to be turned off.
+     */
+    public static void updateMouseWithEyeTracking(double mouseX, double mouseY,
+                                                  MinecraftClient client,
+                                                  double lastMouseUpdateTime,
+                                                  boolean holdingLeftMouseButton,
+                                                  SmoothUtil smoothX,
+                                                  SmoothUtil smoothY
+                                             ) {
+        // The player wants objects of interest to be moved under the crosshair that is always center of screen.
+        // Normal mouse controls operate with the delta values from the direction of mouse movement,
+        // but in eye tracking we want to use the cursor's actual x,y values (their point of gaze), relative to
+        // the screen center (where the crosshair is). This new eye tracking delta creates a vector that points
+        // from the crosshair to the gaze point. As the player keeps their eyes on the object of interest, we pull
+        // that object into the center until the object is underneath the crosshair.
+        double timestamp = GlfwUtil.getTime();
+        double deltaTime = timestamp - lastMouseUpdateTime;
+
+        // The center of screen is the new (0,0)
+        double centerX = client.getWindow().getWidth() / 2.0;
+        double centerY = client.getWindow().getHeight() / 2.0;
+        double gazeRawX = mouseX - centerX;
+        double gazeRawY = mouseY - centerY;
+
+        //This part follows the original mouse.java somewhat closely, with different constants
+        double feeling = 2.5;
+        double sensitivity = client.options.getMouseSensitivity().getValue() * feeling;
+        double spyglass = sensitivity * sensitivity * sensitivity;
+        double moveScalar = spyglass * 8.0;
+
+        double frameScalar;
+        if(client.options.getPerspective().isFirstPerson() && client.player.isUsingSpyglass()) {
+            frameScalar = spyglass;
+        } else {
+            frameScalar = moveScalar;
+        }
+        if(holdingLeftMouseButton){
+            frameScalar *= 0.5; //Don't move the camera so much while mining. It's annoying.
+        }
+
+        // The longest vector connects the center to the corner of the screen, so that is our maximum magnitude for
+        // normalization. We use normalized screen size vector for resolution independent control
+        double magnitudeMax = Math.sqrt(centerX*centerX + centerY*centerY);
+        double normalizedX = gazeRawX / magnitudeMax;
+        double normalizedY = gazeRawY / magnitudeMax;
+
+        double moveX = normalizedX * frameScalar;
+        double moveY = normalizedY * frameScalar;
+        if (client.options.smoothCameraEnabled) {
+            moveX = smoothX.smooth(moveX, moveScalar*deltaTime);
+            moveY = smoothY.smooth(moveY, moveScalar*deltaTime);
+        }
+
+        // The player entity's needs their facing rotated.
+        double invertY = 1.0;
+        double moveMagnitude = Math.sqrt(normalizedX*normalizedX + normalizedY*normalizedY);
+        if (client.options.getInvertYMouse().getValue()) {
+            invertY = -1.0;
+        }
+        if (client.player != null && moveMagnitude > MidnightControlsConfig.eyeTrackerDeadzone) {
+            client.player.changeLookDirection(moveX, moveY * invertY);
+            client.getTutorialManager().onUpdateMouse(moveX, moveY);
+        }
+    }
+}

src/main/resources/assets/midnightcontrols/lang/en_us.json

@@ -153,6 +153,10 @@
   "midnightcontrols.menu.invert_right_y_axis": "Invert Right Y",
   "midnightcontrols.menu.joystick_as_mouse": "Always use left stick as mouse",
   "midnightcontrols.menu.joystick_as_mouse.tooltip": "Make the joystick behave like a mouse in every menu.",
+  "midnightcontrols.menu.eye_tracker_as_mouse": "Use Eye Tracker as Mouse",
+  "midnightcontrols.menu.eye_tracker_as_mouse.tooltip": "Replace the mouse with an eye tracking device, (for example) the Tobii 5.",
+  "midnightcontrols.menu.eye_tracker_deadzone": "Eye Tracker Deadzone Size",
+  "midnightcontrols.menu.eye_tracker_deadzone.tooltip": "Stops camera movement when looking near the cross hair",
   "midnightcontrols.menu.keyboard_controls": "Keyboard Controls...",
   "midnightcontrols.menu.left_dead_zone": "Left Stick Dead Zone",
   "midnightcontrols.menu.left_dead_zone.tooltip": "The dead zone for the controller's left analog stick.",

src/main/resources/midnightcontrols.mixins.json

@@ -13,6 +13,7 @@
     "GameRendererMixin",
     "HandledScreenMixin",
     "KeyBindingMixin",
+    "InputUtilMixin",
     "MinecraftClientMixin",
     "MouseMixin",
     "ChatScreenMixin",