From 60bd283c4c9261cd7ceed9d6ac03ba2303bcfea5 Mon Sep 17 00:00:00 2001 From: Kabliz <414924+kabliz@users.noreply.github.com> Date: Mon, 8 May 2023 20:07:01 -0700 Subject: [PATCH] Eye Tracking Support, Minecraft Without Hands Add support for eye tracking hardware, eg the Tobii 5. Add settings to toggle eye tracking in the settings menu. Add settings to adjust the deadzone while looking at the crosshair. Disable mouse raw input while eye tracking is in use, as these two modes are not compatible. --- .../client/MidnightControlsConfig.java | 2 + .../gui/MidnightControlsSettingsScreen.java | 9 +++ .../client/mixin/InputUtilMixin.java | 39 ++++++++++ .../client/mixin/MouseMixin.java | 72 +++++++++++++++++- .../client/mouse/EyeTrackerHandler.java | 76 +++++++++++++++++++ .../assets/midnightcontrols/lang/en_us.json | 4 + .../resources/midnightcontrols.mixins.json | 1 + 7 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 src/main/java/eu/midnightdust/midnightcontrols/client/mixin/InputUtilMixin.java create mode 100644 src/main/java/eu/midnightdust/midnightcontrols/client/mouse/EyeTrackerHandler.java diff --git a/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsConfig.java b/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsConfig.java index 8503995..2476180 100644 --- a/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsConfig.java +++ b/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsConfig.java @@ -64,6 +64,8 @@ public class MidnightControlsConfig extends MidnightConfig { @Entry(category = "controller", name = "midnightcontrols.menu.y_axis_rotation_speed", isSlider = true, min = 0, max = 100, precision = 10) public static double yAxisRotationSpeed = rotationSpeed; @Entry(category = "screens", name = "midnightcontrols.menu.mouse_speed", isSlider = true, min = 0, max = 150, precision = 10) public static double mouseSpeed = 25.0; @Entry(category = "screens", name = "midnightcontrols.menu.joystick_as_mouse") public static boolean joystickAsMouse = false; + @Entry(category = "screens", name = "midnightcontrols.menu.eye_tracker_as_mouse") public static boolean eyeTrackerAsMouse = false; + @Entry(category = "screens", name = "midnightcontrols.menu.eye_tracker_deadzone", isSlider = true, min = 0, max = 0.4) public static double eyeTrackerDeadzone = 0.05; @Entry(category = "controller", name = "midnightcontrols.menu.unfocused_input") public static boolean unfocusedInput = false; @Entry(category = "screens", name = "midnightcontrols.menu.virtual_mouse") public static boolean virtualMouse = false; @Entry(category = "screens", name = "midnightcontrols.menu.virtual_mouse.skin") public static VirtualMouseSkin virtualMouseSkin = VirtualMouseSkin.DEFAULT_LIGHT; diff --git a/src/main/java/eu/midnightdust/midnightcontrols/client/gui/MidnightControlsSettingsScreen.java b/src/main/java/eu/midnightdust/midnightcontrols/client/gui/MidnightControlsSettingsScreen.java index 4b81295..70fa1a4 100644 --- a/src/main/java/eu/midnightdust/midnightcontrols/client/gui/MidnightControlsSettingsScreen.java +++ b/src/main/java/eu/midnightdust/midnightcontrols/client/gui/MidnightControlsSettingsScreen.java @@ -59,6 +59,8 @@ public class MidnightControlsSettingsScreen extends SpruceScreen { private final SpruceOption yAxisRotationSpeedOption; private final SpruceOption mouseSpeedOption; private final SpruceOption joystickAsMouseOption; + private final SpruceOption eyeTrackingAsMouseOption; + private final SpruceOption eyeTrackingDeadzone; private final SpruceOption virtualMouseOption; private final SpruceOption hideCursorOption; private final SpruceOption resetOption; @@ -193,6 +195,12 @@ public class MidnightControlsSettingsScreen extends SpruceScreen { this.joystickAsMouseOption = new SpruceToggleBooleanOption("midnightcontrols.menu.joystick_as_mouse", () -> MidnightControlsConfig.joystickAsMouse, value -> MidnightControlsConfig.joystickAsMouse = value, Text.translatable("midnightcontrols.menu.joystick_as_mouse.tooltip")); + this.eyeTrackingAsMouseOption = new SpruceToggleBooleanOption("midnightcontrols.menu.eye_tracker_as_mouse", + () -> MidnightControlsConfig.eyeTrackerAsMouse, value -> MidnightControlsConfig.eyeTrackerAsMouse = value, + Text.translatable("midnightcontrols.menu.eye_tracker_as_mouse.tooltip")); + this.eyeTrackingDeadzone = new SpruceDoubleInputOption("midnightcontrols.menu.eye_tracker_deadzone", + () -> MidnightControlsConfig.eyeTrackerDeadzone, value -> MidnightControlsConfig.eyeTrackerDeadzone = value, + Text.translatable("midnightcontrols.menu.eye_tracker_deadzone.tooltip")); this.resetOption = SpruceSimpleActionOption.reset(btn -> { MidnightControlsConfig.reset(); var client = MinecraftClient.getInstance(); @@ -344,6 +352,7 @@ public class MidnightControlsSettingsScreen extends SpruceScreen { list.addSingleOptionEntry(this.virtualMouseOption); list.addSingleOptionEntry(this.hideCursorOption); list.addSingleOptionEntry(this.joystickAsMouseOption); + list.addSingleOptionEntry(this.eyeTrackingAsMouseOption); list.addSingleOptionEntry(this.advancedConfigOption); return list; } diff --git a/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/InputUtilMixin.java b/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/InputUtilMixin.java new file mode 100644 index 0000000..37505f3 --- /dev/null +++ b/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/InputUtilMixin.java @@ -0,0 +1,39 @@ +package eu.midnightdust.midnightcontrols.client.mixin; + +import net.minecraft.client.util.InputUtil; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.Shadow; +import org.spongepowered.asm.mixin.Overwrite; +import org.spongepowered.asm.mixin.Final; +import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig; +import java.lang.invoke.MethodHandle; + + +@Mixin(InputUtil.class) +public abstract class InputUtilMixin { + + @Final + @Shadow + private static MethodHandle GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE; + + /** + * @author kabliz + * @reason This method is static, and there is a terrible UX issue if raw input is turned on at the same time as + * eye tracking. Raw input only tracks literal mice and not other devices, leading to the game appearing to be + * unresponsive and the player not understanding why. This overwrite preserves the user's mouse preferences, + * while not interfering with eye tracking, and the two modes can be switched between during a play session. + */ + @Overwrite + public static boolean isRawMouseMotionSupported(){ + if(MidnightControlsConfig.eyeTrackerAsMouse){ + return false; + } else { //Paste original implementation from InputUtil below. + try { + return GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE != null && + (boolean) GLFW_RAW_MOUSE_MOTION_SUPPORTED_HANDLE.invokeExact(); + } catch (Throwable var1) { + throw new RuntimeException(var1); + } + } + } +} diff --git a/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/MouseMixin.java b/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/MouseMixin.java index 211f002..ab2b112 100644 --- a/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/MouseMixin.java +++ b/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/MouseMixin.java @@ -15,7 +15,9 @@ import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig; import eu.midnightdust.midnightcontrols.client.util.MouseAccessor; import net.minecraft.client.MinecraftClient; import net.minecraft.client.Mouse; -import net.minecraft.client.gui.screen.Screen; +import net.minecraft.client.option.KeyBinding; +import net.minecraft.client.util.GlfwUtil; +import net.minecraft.client.util.SmoothUtil; import org.lwjgl.glfw.GLFW; import org.spongepowered.asm.mixin.Final; import org.spongepowered.asm.mixin.Mixin; @@ -26,6 +28,10 @@ import org.spongepowered.asm.mixin.injection.At; import org.spongepowered.asm.mixin.injection.Inject; import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; +import eu.midnightdust.midnightcontrols.client.mouse.EyeTrackerHandler; + +import static org.lwjgl.glfw.GLFW.GLFW_CURSOR; +import static org.lwjgl.glfw.GLFW.GLFW_CURSOR_HIDDEN; /** * Adds extra access to the mouse. @@ -36,6 +42,37 @@ public abstract class MouseMixin implements MouseAccessor { @Final private MinecraftClient client; + @Shadow + private double y; + + @Shadow + private double cursorDeltaX; + + @Shadow + private double cursorDeltaY; + + @Shadow + private double x; + + @Shadow + private boolean cursorLocked; + + @Shadow + private boolean hasResolutionChanged; + + @Shadow + private double lastMouseUpdateTime; + + @Shadow + @Final + private SmoothUtil cursorXSmoother; + + @Shadow + @Final + private SmoothUtil cursorYSmoother; + + @Shadow private boolean leftButtonClicked; + @Accessor public abstract void setLeftButtonClicked(boolean value); @@ -67,4 +104,37 @@ public abstract class MouseMixin implements MouseAccessor { ||*/ (MidnightControlsConfig.controlsMode == ControlsMode.CONTROLLER && MidnightControlsConfig.virtualMouse)) ci.cancel(); } + + @Inject(method = "updateMouse", at = @At("HEAD"), cancellable = true) + private void updateMouse(CallbackInfo ci) { + if (MidnightControlsConfig.eyeTrackerAsMouse && cursorLocked && client.isWindowFocused()) { + //Eye Tracking is only for the camera controlling cursor, we need the normal cursor everywhere else. + if (!client.options.smoothCameraEnabled) { + cursorXSmoother.clear(); + cursorYSmoother.clear(); + } + EyeTrackerHandler.updateMouseWithEyeTracking(x + cursorDeltaX, y + cursorDeltaY, client, + lastMouseUpdateTime, leftButtonClicked, cursorXSmoother, cursorYSmoother); + lastMouseUpdateTime = GlfwUtil.getTime(); + cursorDeltaX = 0.0; + cursorDeltaY = 0.0; + ci.cancel(); + } + } + + @Inject(method = "lockCursor", at = @At("HEAD"), cancellable = true) + private void lockCursor(CallbackInfo ci) { + if (MidnightControlsConfig.eyeTrackerAsMouse && client.isWindowFocused() && !this.cursorLocked) { + if (!MinecraftClient.IS_SYSTEM_MAC) { + KeyBinding.updatePressedStates(); + } + //In eye tracking mode, we cannot have the cursor locked to the center. + GLFW.glfwSetInputMode(client.getWindow().getHandle(), GLFW_CURSOR, GLFW_CURSOR_HIDDEN); + cursorLocked = true; //The game uses this flag for other gameplay checks + client.setScreen(null); + hasResolutionChanged = true; + ci.cancel(); + } + } + } diff --git a/src/main/java/eu/midnightdust/midnightcontrols/client/mouse/EyeTrackerHandler.java b/src/main/java/eu/midnightdust/midnightcontrols/client/mouse/EyeTrackerHandler.java new file mode 100644 index 0000000..1cef5b6 --- /dev/null +++ b/src/main/java/eu/midnightdust/midnightcontrols/client/mouse/EyeTrackerHandler.java @@ -0,0 +1,76 @@ +package eu.midnightdust.midnightcontrols.client.mouse; + +import net.minecraft.client.MinecraftClient; +import net.minecraft.client.util.GlfwUtil; +import net.minecraft.client.util.SmoothUtil; +import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig; + +public class EyeTrackerHandler { + + /** + * Based on the updateMouse method in the Mouse.class, this changes the mouse algorithm to suit eye tracking. + * This requires the cursor to not be locked, and the raw input setting to be turned off. + */ + public static void updateMouseWithEyeTracking(double mouseX, double mouseY, + MinecraftClient client, + double lastMouseUpdateTime, + boolean holdingLeftMouseButton, + SmoothUtil smoothX, + SmoothUtil smoothY + ) { + // The player wants objects of interest to be moved under the crosshair that is always center of screen. + // Normal mouse controls operate with the delta values from the direction of mouse movement, + // but in eye tracking we want to use the cursor's actual x,y values (their point of gaze), relative to + // the screen center (where the crosshair is). This new eye tracking delta creates a vector that points + // from the crosshair to the gaze point. As the player keeps their eyes on the object of interest, we pull + // that object into the center until the object is underneath the crosshair. + double timestamp = GlfwUtil.getTime(); + double deltaTime = timestamp - lastMouseUpdateTime; + + // The center of screen is the new (0,0) + double centerX = client.getWindow().getWidth() / 2.0; + double centerY = client.getWindow().getHeight() / 2.0; + double gazeRawX = mouseX - centerX; + double gazeRawY = mouseY - centerY; + + //This part follows the original mouse.java somewhat closely, with different constants + double feeling = 2.5; + double sensitivity = client.options.getMouseSensitivity().getValue() * feeling; + double spyglass = sensitivity * sensitivity * sensitivity; + double moveScalar = spyglass * 8.0; + + double frameScalar; + if(client.options.getPerspective().isFirstPerson() && client.player.isUsingSpyglass()) { + frameScalar = spyglass; + } else { + frameScalar = moveScalar; + } + if(holdingLeftMouseButton){ + frameScalar *= 0.5; //Don't move the camera so much while mining. It's annoying. + } + + // The longest vector connects the center to the corner of the screen, so that is our maximum magnitude for + // normalization. We use normalized screen size vector for resolution independent control + double magnitudeMax = Math.sqrt(centerX*centerX + centerY*centerY); + double normalizedX = gazeRawX / magnitudeMax; + double normalizedY = gazeRawY / magnitudeMax; + + double moveX = normalizedX * frameScalar; + double moveY = normalizedY * frameScalar; + if (client.options.smoothCameraEnabled) { + moveX = smoothX.smooth(moveX, moveScalar*deltaTime); + moveY = smoothY.smooth(moveY, moveScalar*deltaTime); + } + + // The player entity's needs their facing rotated. + double invertY = 1.0; + double moveMagnitude = Math.sqrt(normalizedX*normalizedX + normalizedY*normalizedY); + if (client.options.getInvertYMouse().getValue()) { + invertY = -1.0; + } + if (client.player != null && moveMagnitude > MidnightControlsConfig.eyeTrackerDeadzone) { + client.player.changeLookDirection(moveX, moveY * invertY); + client.getTutorialManager().onUpdateMouse(moveX, moveY); + } + } +} diff --git a/src/main/resources/assets/midnightcontrols/lang/en_us.json b/src/main/resources/assets/midnightcontrols/lang/en_us.json index 5b529c5..11b5107 100644 --- a/src/main/resources/assets/midnightcontrols/lang/en_us.json +++ b/src/main/resources/assets/midnightcontrols/lang/en_us.json @@ -153,6 +153,10 @@ "midnightcontrols.menu.invert_right_y_axis": "Invert Right Y", "midnightcontrols.menu.joystick_as_mouse": "Always use left stick as mouse", "midnightcontrols.menu.joystick_as_mouse.tooltip": "Make the joystick behave like a mouse in every menu.", + "midnightcontrols.menu.eye_tracker_as_mouse": "Use Eye Tracker as Mouse", + "midnightcontrols.menu.eye_tracker_as_mouse.tooltip": "Replace the mouse with an eye tracking device, (for example) the Tobii 5.", + "midnightcontrols.menu.eye_tracker_deadzone": "Eye Tracker Deadzone Size", + "midnightcontrols.menu.eye_tracker_deadzone.tooltip": "Stops camera movement when looking near the cross hair", "midnightcontrols.menu.keyboard_controls": "Keyboard Controls...", "midnightcontrols.menu.left_dead_zone": "Left Stick Dead Zone", "midnightcontrols.menu.left_dead_zone.tooltip": "The dead zone for the controller's left analog stick.", diff --git a/src/main/resources/midnightcontrols.mixins.json b/src/main/resources/midnightcontrols.mixins.json index 33fdd50..57f2c20 100644 --- a/src/main/resources/midnightcontrols.mixins.json +++ b/src/main/resources/midnightcontrols.mixins.json @@ -13,6 +13,7 @@ "GameRendererMixin", "HandledScreenMixin", "KeyBindingMixin", + "InputUtilMixin", "MinecraftClientMixin", "MouseMixin", "ChatScreenMixin",