Implement basic virtual keyboard support

- Listener for clicks inside of text fields and other text-based screens
- Virtual keyboard screen in a QWERTY layout

common/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsClient.java

@@ -204,6 +204,7 @@ public class MidnightControlsClient extends MidnightControls {
         RainbowColor.tick();
         TouchInput.tick();
     }
+
     /**
      * Called when opening a screen.
      */

common/src/main/java/eu/midnightdust/midnightcontrols/client/gui/VirtualKeyboardScreen.java

@@ -0,0 +1,108 @@
+package eu.midnightdust.midnightcontrols.client.gui;
+
+import net.minecraft.client.gui.DrawContext;
+import net.minecraft.client.gui.widget.TextFieldWidget;
+import net.minecraft.text.Text;
+import org.thinkingstudio.obsidianui.Position;
+import org.thinkingstudio.obsidianui.SpruceTexts;
+import org.thinkingstudio.obsidianui.screen.SpruceScreen;
+import org.thinkingstudio.obsidianui.widget.SpruceButtonWidget;
+import org.thinkingstudio.obsidianui.widget.container.SpruceContainerWidget;
+
+public class VirtualKeyboardScreen extends SpruceScreen {
+    private SpruceContainerWidget container;
+    private TextFieldWidget bufferDisplay;
+    private final StringBuilder buffer;
+    private final CloseCallback closeCallback;
+
+    @FunctionalInterface
+    public interface CloseCallback {
+        void onClose(String text);
+    }
+
+    public VirtualKeyboardScreen(String initialText, CloseCallback closeCallback) {
+        super(Text.literal("Virtual Keyboard"));
+
+        this.buffer = new StringBuilder(initialText);
+        this.closeCallback = closeCallback;
+    }
+
+    @Override
+    protected void init() {
+        super.init();
+
+        this.bufferDisplay = new TextFieldWidget(this.textRenderer, this.width / 2 - 100, this.height / 4 - 40, 200, 20, Text.literal(""));
+        this.bufferDisplay.setEditable(false);
+        this.bufferDisplay.setMaxLength(1024);
+        this.bufferDisplay.setText(buffer.toString());
+        this.addDrawableChild(this.bufferDisplay);
+
+        rebuildKeyboard();
+
+        this.addDrawableChild(container);
+        this.addDrawableChild(new SpruceButtonWidget(Position.of(this, this.width / 2 - 50, this.height - 30), 100, 20, SpruceTexts.GUI_DONE, btn -> this.close()));
+    }
+
+    @Override
+    public void render(DrawContext drawContext, int mouseX, int mouseY, float delta) {
+        super.render(drawContext, mouseX, mouseY, delta);
+        drawContext.drawCenteredTextWithShadow(this.textRenderer, this.title, this.width / 2, 10, 0xFFFFFF);
+    }
+
+
+    private void rebuildKeyboard() {
+        this.container = new SpruceContainerWidget(Position.of(0, this.height / 4 - 10), this.width, this.height / 2);
+
+
+        var row1 = new String[]{"q", "w", "e", "r", "t", "y", "u", "i", "o", "p"};
+        var row2 = new String[]{"a", "s", "d", "f", "g", "h", "j", "k", "l"};
+        var row3 = new String[]{"z", "x", "c", "v", "b", "n", "m"};
+
+        addKeyRow(0, row1);
+        addKeyRow(1, row2);
+        addKeyRow(2, row3);
+    }
+
+    private void addKeyRow(int rowOffset, String... keys) {
+        int keyWidth = 20;
+        int spacing = 2;
+        int totalWidth = (keyWidth + spacing) * keys.length - spacing;
+        int startX = (this.width - totalWidth) / 2;
+        int y = this.height / 4 + rowOffset * 24;
+
+        for (int i = 0; i < keys.length; i++) {
+            String key = keys[i];
+            this.container.addChild(new SpruceButtonWidget(Position.of(startX + i * (keyWidth + spacing), y), keyWidth, 20, Text.literal(key), btn -> handleKeyPress(key)));
+        }
+    }
+
+    private void handleKeyPress(String key) {
+        if (this.client == null) return;
+
+        // TODO
+        if (key.equals("\b")) {
+            if (!buffer.isEmpty()) {
+                buffer.deleteCharAt(buffer.length() - 1);
+            }
+        } else {
+            buffer.append(key);
+        }
+
+        if (this.bufferDisplay != null) {
+            this.bufferDisplay.setText(buffer.toString());
+        }
+    }
+
+    @Override
+    public boolean shouldPause() {
+        return false;
+    }
+
+    @Override
+    public void close() {
+        super.close();
+        if (closeCallback != null) {
+            closeCallback.onClose(buffer.toString());
+        }
+    }
+}

common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/AbstractSignEditScreenAccessor.java

@@ -0,0 +1,15 @@
+package eu.midnightdust.midnightcontrols.client.mixin;
+
+import net.minecraft.block.entity.SignText;
+import net.minecraft.client.gui.screen.ingame.AbstractSignEditScreen;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.gen.Accessor;
+
+@Mixin(AbstractSignEditScreen.class)
+public interface AbstractSignEditScreenAccessor {
+    @Accessor("text")
+    SignText midnightcontrols$getText();
+
+    @Accessor("text")
+    void midnightcontrols$setText(SignText text);
+}

common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/BookEditScreenAccessor.java

@@ -0,0 +1,24 @@
+package eu.midnightdust.midnightcontrols.client.mixin;
+
+import net.minecraft.client.gui.screen.ingame.BookEditScreen;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.gen.Accessor;
+import org.spongepowered.asm.mixin.gen.Invoker;
+
+@Mixin(BookEditScreen.class)
+public interface BookEditScreenAccessor {
+    @Accessor("signing")
+    boolean midnightcontrols$isSigning();
+
+    @Accessor("title")
+    String midnightcontrols$getTitle();
+
+    @Accessor("title")
+    void midnightcontrols$setTitle(String title);
+
+    @Invoker("getCurrentPageContent")
+    String midnightcontrols$getCurrentPageContent();
+
+    @Invoker("setPageContent")
+    void midnightcontrols$setPageContent(String newContent);
+}

common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/CreativeInventoryScreenAccessor.java

@@ -58,4 +58,10 @@ public interface CreativeInventoryScreenAccessor {
      */
     @Invoker("hasScrollbar")
     boolean midnightcontrols$hasScrollbar();
+
+    /**
+     * Triggers searching the creative inventory from the current value of the internal {@link net.minecraft.client.gui.widget.TextFieldWidget}
+     */
+    @Invoker("search")
+    void midnightcontrols$search();
 }

common/src/main/resources/midnightcontrols.mixins.json

@@ -4,6 +4,7 @@
   "compatibilityLevel": "JAVA_21",
   "client": [
     "AdvancementsScreenAccessor",
+    "BookEditScreenAccessor",
     "ChatScreenMixin",
     "ClickableWidgetAccessor",
     "ClientPlayerEntityMixin",
@@ -22,6 +23,7 @@
     "RecipeBookScreenAccessor",
     "RecipeBookWidgetAccessor",
     "ScreenMixin",
+    "AbstractSignEditScreenAccessor",
     "TabNavigationWidgetAccessor",
     "WorldRendererMixin"
   ],

fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/MidnightControlsClientFabric.java

@@ -3,6 +3,7 @@ package eu.midnightdust.midnightcontrols.fabric;
 import eu.midnightdust.midnightcontrols.MidnightControlsConstants;
 import eu.midnightdust.midnightcontrols.client.MidnightControlsClient;
 import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig;
+import eu.midnightdust.midnightcontrols.fabric.event.MouseClickListener;
 import eu.midnightdust.midnightcontrols.packet.ControlsModePayload;
 import eu.midnightdust.midnightcontrols.packet.FeaturePayload;
 import eu.midnightdust.midnightcontrols.packet.HelloPayload;
@@ -11,11 +12,12 @@ import net.fabricmc.fabric.api.client.event.lifecycle.v1.ClientTickEvents;
 import net.fabricmc.fabric.api.client.keybinding.v1.KeyBindingHelper;
 import net.fabricmc.fabric.api.client.networking.v1.ClientPlayConnectionEvents;
 import net.fabricmc.fabric.api.client.networking.v1.ClientPlayNetworking;
+import net.fabricmc.fabric.api.client.screen.v1.ScreenEvents;
+import net.fabricmc.fabric.api.client.screen.v1.ScreenMouseEvents;
 import net.fabricmc.fabric.api.resource.ResourceManagerHelper;
 import net.fabricmc.fabric.api.resource.ResourcePackActivationType;
 import net.fabricmc.loader.api.FabricLoader;
 import net.fabricmc.loader.api.ModContainer;
-import org.thinkingstudio.obsidianui.fabric.event.OpenScreenCallback;
 
 import java.util.Optional;
 
@@ -51,6 +53,9 @@ public class MidnightControlsClientFabric implements ClientModInitializer {
         ClientPlayConnectionEvents.DISCONNECT.register((handler, client) -> MidnightControlsClient.onLeave());
 
         ClientTickEvents.START_CLIENT_TICK.register(MidnightControlsClient::onTick);
+        ScreenEvents.AFTER_INIT.register((client, screen, scaledWidth, scaledHeight) -> {
+            ScreenMouseEvents.allowMouseClick(screen).register(new MouseClickListener(screen));
+        });
 
         FabricLoader.getInstance().getModContainer(MidnightControlsConstants.NAMESPACE).ifPresent(modContainer -> {
             ResourceManagerHelper.registerBuiltinResourcePack(id("bedrock"), modContainer, ResourcePackActivationType.NORMAL);

fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/event/MouseClickListener.java

@@ -0,0 +1,217 @@
+package eu.midnightdust.midnightcontrols.fabric.event;
+import eu.midnightdust.midnightcontrols.client.gui.VirtualKeyboardScreen;
+import eu.midnightdust.midnightcontrols.client.mixin.AbstractSignEditScreenAccessor;
+import eu.midnightdust.midnightcontrols.client.mixin.BookEditScreenAccessor;
+import eu.midnightdust.midnightcontrols.client.mixin.CreativeInventoryScreenAccessor;
+import net.fabricmc.fabric.api.client.screen.v1.ScreenMouseEvents;
+import net.minecraft.block.entity.SignText;
+import net.minecraft.client.gui.Element;
+import net.minecraft.client.gui.ParentElement;
+import net.minecraft.client.gui.screen.ChatScreen;
+import net.minecraft.client.gui.screen.Screen;
+import net.minecraft.client.gui.screen.ingame.BookEditScreen;
+import net.minecraft.client.gui.screen.ingame.CreativeInventoryScreen;
+import net.minecraft.client.gui.screen.ingame.SignEditScreen;
+import net.minecraft.client.gui.widget.TextFieldWidget;
+import org.lwjgl.glfw.GLFW;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static eu.midnightdust.midnightcontrols.MidnightControls.logger;
+import static eu.midnightdust.midnightcontrols.client.MidnightControlsClient.client;
+
+record ScreenLink(Screen screen, List<Integer> elementPath) {}
+
+public class MouseClickListener implements ScreenMouseEvents.AllowMouseClick {
+    private final Screen screen;
+
+    private ScreenLink link;
+
+    public MouseClickListener(Screen screen) {
+        this.screen = screen;
+    }
+
+    @Override
+    public boolean allowMouseClick(Screen screen, double mouseX, double mouseY, int button) {
+        interceptMouseClick(screen, mouseX, mouseY);
+        return true;
+    }
+
+    private void interceptMouseClick(Screen screen, double mouseX, double mouseY) {
+        logger.info("In scr: {}", screen.getClass());
+        switch(screen) {
+            case BookEditScreen bookEditScreen -> {
+                if(screen.hoveredElement(mouseX, mouseY).isPresent()) {
+                    return;
+                }
+                handleBookEditScreenClick(bookEditScreen);
+            }
+            case SignEditScreen signEditScreen -> {
+                if(screen.hoveredElement(mouseX, mouseY).isPresent()) {
+                    return;
+                }
+                handleSignEditScreenClick(signEditScreen);
+            }
+            default -> {
+                var textField = findClickedTextField(screen, mouseX, mouseY);
+                if (textField == null) {
+                    return;
+                }
+                handleTextFieldClick(textField);
+            }
+        }
+    }
+
+    // Add equals and hashCode to prevent duplicate registrations
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof MouseClickListener) {
+            return ((MouseClickListener) obj).screen == this.screen;
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return screen.hashCode();
+    }
+
+    // handlers
+
+    private void handleBookEditScreenClick(BookEditScreen bookEditScreen) {
+        var accessor = (BookEditScreenAccessor) screen;
+
+        VirtualKeyboardScreen virtualKeyboardScreen;
+        if(accessor.midnightcontrols$isSigning()) {
+            virtualKeyboardScreen = new VirtualKeyboardScreen(accessor.midnightcontrols$getTitle(), (text) -> {
+                client.setScreen(bookEditScreen);
+                accessor.midnightcontrols$setTitle(text);
+            });
+        }
+        else {
+            virtualKeyboardScreen = new VirtualKeyboardScreen(accessor.midnightcontrols$getCurrentPageContent(), (text) -> {
+                client.setScreen(bookEditScreen);
+                accessor.midnightcontrols$setPageContent(text);
+            });
+        }
+
+        client.setScreen(virtualKeyboardScreen);
+    }
+
+    private void handleSignEditScreenClick(SignEditScreen signEditScreen) {
+        var accessor = (AbstractSignEditScreenAccessor) signEditScreen;
+        // TODO
+    }
+
+    private void handleTextFieldClick(TextFieldWidget textField) {
+        this.link = new ScreenLink(screen, calculatePathToElement(screen, textField));
+        var virtualKeyboardScreen = new VirtualKeyboardScreen(textField.getText(), this::handleKeyboardClose);
+        client.setScreen(virtualKeyboardScreen);
+    }
+
+    private void handleKeyboardClose(String newText) {
+        if(this.link == null) {
+            return;
+        }
+
+        client.setScreen(this.link.screen());
+        var txtField = findTextFieldByPath(screen, this.link.elementPath());
+        if (txtField == null) {
+            return;
+        }
+
+        txtField.setText(newText);
+
+
+        switch (this.link.screen()) {
+            case CreativeInventoryScreen creativeInventoryScreen -> {
+                var accessor = (CreativeInventoryScreenAccessor) creativeInventoryScreen;
+                accessor.midnightcontrols$search();
+            }
+            case ChatScreen chatScreen -> {
+                // send the chat message
+                chatScreen.keyPressed(GLFW.GLFW_KEY_ENTER, 0, 0);
+            }
+            default -> {}
+        }
+    }
+
+    // utility
+
+    private TextFieldWidget findClickedTextField(Screen screen, double mouseX, double mouseY) {
+        for (Element element : screen.children()) {
+            if (element instanceof TextFieldWidget textField) {
+                if (textField.isMouseOver(mouseX, mouseY) && textField.isFocused()) {
+                    return textField;
+                }
+            }
+        }
+
+        // not hovering over a text field
+        return null;
+    }
+
+    /**
+     * Calculates the path between a parent and a target in the UI hierarchy
+     */
+    private List<Integer> calculatePathToElement(Element parent, Element target) {
+        if (parent instanceof ParentElement parentElement) {
+            List<? extends Element> children = parentElement.children();
+
+            // check direct children first
+            for (int i = 0; i < children.size(); i++) {
+                if (children.get(i) == target) {
+                    // found it, return the path to this element
+                    return Collections.singletonList(i);
+                }
+            }
+
+            // check each child's children
+            for (int i = 0; i < children.size(); i++) {
+                if (children.get(i) instanceof ParentElement childParent) {
+                    List<Integer> subPath = calculatePathToElement(childParent, target);
+                    if (subPath != null) {
+                        // found in this subtree, prepend current index
+                        List<Integer> fullPath = new ArrayList<>();
+                        fullPath.add(i);
+                        fullPath.addAll(subPath);
+                        return fullPath;
+                    }
+                }
+            }
+        }
+
+        // Not found
+        return null;
+    }
+
+    private TextFieldWidget findTextFieldByPath(Element parent, List<Integer> path) {
+        if (path == null || path.isEmpty()) {
+            return null;
+        }
+
+        if (parent instanceof ParentElement parentElement) {
+            List<? extends Element> children = parentElement.children();
+            int index = path.getFirst();
+
+            if (index >= 0 && index < children.size()) {
+                Element child = children.get(index);
+
+                if (path.size() == 1) {
+                    // This should be our target
+                    return (child instanceof TextFieldWidget) ? (TextFieldWidget) child : null;
+                } else {
+                    // Continue traversing
+                    if (child instanceof ParentElement) {
+                        return findTextFieldByPath(child, path.subList(1, path.size()));
+                    }
+                }
+            }
+        }
+
+        return null;
+    }
+}