From 9e12381471680f1456d1ed396f582d9f3f9ed8a6 Mon Sep 17 00:00:00 2001 From: cryy <19242445+cryy@users.noreply.github.com> Date: Fri, 18 Apr 2025 19:43:01 +0200 Subject: [PATCH] Implement basic virtual keyboard support - Listener for clicks inside of text fields and other text-based screens - Virtual keyboard screen in a QWERTY layout --- .../client/MidnightControlsClient.java | 1 + .../client/gui/VirtualKeyboardScreen.java | 108 +++++++++ .../mixin/AbstractSignEditScreenAccessor.java | 15 ++ .../client/mixin/BookEditScreenAccessor.java | 24 ++ .../CreativeInventoryScreenAccessor.java | 6 + .../resources/midnightcontrols.mixins.json | 2 + .../fabric/MidnightControlsClientFabric.java | 7 +- .../fabric/event/MouseClickListener.java | 217 ++++++++++++++++++ 8 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 common/src/main/java/eu/midnightdust/midnightcontrols/client/gui/VirtualKeyboardScreen.java create mode 100644 common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/AbstractSignEditScreenAccessor.java create mode 100644 common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/BookEditScreenAccessor.java create mode 100644 fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/event/MouseClickListener.java diff --git a/common/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsClient.java b/common/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsClient.java index 21df7d7..074250f 100644 --- a/common/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsClient.java +++ b/common/src/main/java/eu/midnightdust/midnightcontrols/client/MidnightControlsClient.java @@ -204,6 +204,7 @@ public class MidnightControlsClient extends MidnightControls { RainbowColor.tick(); TouchInput.tick(); } + /** * Called when opening a screen. */ diff --git a/common/src/main/java/eu/midnightdust/midnightcontrols/client/gui/VirtualKeyboardScreen.java b/common/src/main/java/eu/midnightdust/midnightcontrols/client/gui/VirtualKeyboardScreen.java new file mode 100644 index 0000000..55ca8c4 --- /dev/null +++ b/common/src/main/java/eu/midnightdust/midnightcontrols/client/gui/VirtualKeyboardScreen.java @@ -0,0 +1,108 @@ +package eu.midnightdust.midnightcontrols.client.gui; + +import net.minecraft.client.gui.DrawContext; +import net.minecraft.client.gui.widget.TextFieldWidget; +import net.minecraft.text.Text; +import org.thinkingstudio.obsidianui.Position; +import org.thinkingstudio.obsidianui.SpruceTexts; +import org.thinkingstudio.obsidianui.screen.SpruceScreen; +import org.thinkingstudio.obsidianui.widget.SpruceButtonWidget; +import org.thinkingstudio.obsidianui.widget.container.SpruceContainerWidget; + +public class VirtualKeyboardScreen extends SpruceScreen { + private SpruceContainerWidget container; + private TextFieldWidget bufferDisplay; + private final StringBuilder buffer; + private final CloseCallback closeCallback; + + @FunctionalInterface + public interface CloseCallback { + void onClose(String text); + } + + public VirtualKeyboardScreen(String initialText, CloseCallback closeCallback) { + super(Text.literal("Virtual Keyboard")); + + this.buffer = new StringBuilder(initialText); + this.closeCallback = closeCallback; + } + + @Override + protected void init() { + super.init(); + + this.bufferDisplay = new TextFieldWidget(this.textRenderer, this.width / 2 - 100, this.height / 4 - 40, 200, 20, Text.literal("")); + this.bufferDisplay.setEditable(false); + this.bufferDisplay.setMaxLength(1024); + this.bufferDisplay.setText(buffer.toString()); + this.addDrawableChild(this.bufferDisplay); + + rebuildKeyboard(); + + this.addDrawableChild(container); + this.addDrawableChild(new SpruceButtonWidget(Position.of(this, this.width / 2 - 50, this.height - 30), 100, 20, SpruceTexts.GUI_DONE, btn -> this.close())); + } + + @Override + public void render(DrawContext drawContext, int mouseX, int mouseY, float delta) { + super.render(drawContext, mouseX, mouseY, delta); + drawContext.drawCenteredTextWithShadow(this.textRenderer, this.title, this.width / 2, 10, 0xFFFFFF); + } + + + private void rebuildKeyboard() { + this.container = new SpruceContainerWidget(Position.of(0, this.height / 4 - 10), this.width, this.height / 2); + + + var row1 = new String[]{"q", "w", "e", "r", "t", "y", "u", "i", "o", "p"}; + var row2 = new String[]{"a", "s", "d", "f", "g", "h", "j", "k", "l"}; + var row3 = new String[]{"z", "x", "c", "v", "b", "n", "m"}; + + addKeyRow(0, row1); + addKeyRow(1, row2); + addKeyRow(2, row3); + } + + private void addKeyRow(int rowOffset, String... keys) { + int keyWidth = 20; + int spacing = 2; + int totalWidth = (keyWidth + spacing) * keys.length - spacing; + int startX = (this.width - totalWidth) / 2; + int y = this.height / 4 + rowOffset * 24; + + for (int i = 0; i < keys.length; i++) { + String key = keys[i]; + this.container.addChild(new SpruceButtonWidget(Position.of(startX + i * (keyWidth + spacing), y), keyWidth, 20, Text.literal(key), btn -> handleKeyPress(key))); + } + } + + private void handleKeyPress(String key) { + if (this.client == null) return; + + // TODO + if (key.equals("\b")) { + if (!buffer.isEmpty()) { + buffer.deleteCharAt(buffer.length() - 1); + } + } else { + buffer.append(key); + } + + if (this.bufferDisplay != null) { + this.bufferDisplay.setText(buffer.toString()); + } + } + + @Override + public boolean shouldPause() { + return false; + } + + @Override + public void close() { + super.close(); + if (closeCallback != null) { + closeCallback.onClose(buffer.toString()); + } + } +} \ No newline at end of file diff --git a/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/AbstractSignEditScreenAccessor.java b/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/AbstractSignEditScreenAccessor.java new file mode 100644 index 0000000..fa5036c --- /dev/null +++ b/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/AbstractSignEditScreenAccessor.java @@ -0,0 +1,15 @@ +package eu.midnightdust.midnightcontrols.client.mixin; + +import net.minecraft.block.entity.SignText; +import net.minecraft.client.gui.screen.ingame.AbstractSignEditScreen; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.gen.Accessor; + +@Mixin(AbstractSignEditScreen.class) +public interface AbstractSignEditScreenAccessor { + @Accessor("text") + SignText midnightcontrols$getText(); + + @Accessor("text") + void midnightcontrols$setText(SignText text); +} diff --git a/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/BookEditScreenAccessor.java b/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/BookEditScreenAccessor.java new file mode 100644 index 0000000..e26f51b --- /dev/null +++ b/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/BookEditScreenAccessor.java @@ -0,0 +1,24 @@ +package eu.midnightdust.midnightcontrols.client.mixin; + +import net.minecraft.client.gui.screen.ingame.BookEditScreen; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.gen.Accessor; +import org.spongepowered.asm.mixin.gen.Invoker; + +@Mixin(BookEditScreen.class) +public interface BookEditScreenAccessor { + @Accessor("signing") + boolean midnightcontrols$isSigning(); + + @Accessor("title") + String midnightcontrols$getTitle(); + + @Accessor("title") + void midnightcontrols$setTitle(String title); + + @Invoker("getCurrentPageContent") + String midnightcontrols$getCurrentPageContent(); + + @Invoker("setPageContent") + void midnightcontrols$setPageContent(String newContent); +} diff --git a/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/CreativeInventoryScreenAccessor.java b/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/CreativeInventoryScreenAccessor.java index 94961cf..2ab6c1d 100644 --- a/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/CreativeInventoryScreenAccessor.java +++ b/common/src/main/java/eu/midnightdust/midnightcontrols/client/mixin/CreativeInventoryScreenAccessor.java @@ -58,4 +58,10 @@ public interface CreativeInventoryScreenAccessor { */ @Invoker("hasScrollbar") boolean midnightcontrols$hasScrollbar(); + + /** + * Triggers searching the creative inventory from the current value of the internal {@link net.minecraft.client.gui.widget.TextFieldWidget} + */ + @Invoker("search") + void midnightcontrols$search(); } diff --git a/common/src/main/resources/midnightcontrols.mixins.json b/common/src/main/resources/midnightcontrols.mixins.json index b00371b..4a01ff8 100644 --- a/common/src/main/resources/midnightcontrols.mixins.json +++ b/common/src/main/resources/midnightcontrols.mixins.json @@ -4,6 +4,7 @@ "compatibilityLevel": "JAVA_21", "client": [ "AdvancementsScreenAccessor", + "BookEditScreenAccessor", "ChatScreenMixin", "ClickableWidgetAccessor", "ClientPlayerEntityMixin", @@ -22,6 +23,7 @@ "RecipeBookScreenAccessor", "RecipeBookWidgetAccessor", "ScreenMixin", + "AbstractSignEditScreenAccessor", "TabNavigationWidgetAccessor", "WorldRendererMixin" ], diff --git a/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/MidnightControlsClientFabric.java b/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/MidnightControlsClientFabric.java index 60d43ac..7c9f04c 100644 --- a/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/MidnightControlsClientFabric.java +++ b/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/MidnightControlsClientFabric.java @@ -3,6 +3,7 @@ package eu.midnightdust.midnightcontrols.fabric; import eu.midnightdust.midnightcontrols.MidnightControlsConstants; import eu.midnightdust.midnightcontrols.client.MidnightControlsClient; import eu.midnightdust.midnightcontrols.client.MidnightControlsConfig; +import eu.midnightdust.midnightcontrols.fabric.event.MouseClickListener; import eu.midnightdust.midnightcontrols.packet.ControlsModePayload; import eu.midnightdust.midnightcontrols.packet.FeaturePayload; import eu.midnightdust.midnightcontrols.packet.HelloPayload; @@ -11,11 +12,12 @@ import net.fabricmc.fabric.api.client.event.lifecycle.v1.ClientTickEvents; import net.fabricmc.fabric.api.client.keybinding.v1.KeyBindingHelper; import net.fabricmc.fabric.api.client.networking.v1.ClientPlayConnectionEvents; import net.fabricmc.fabric.api.client.networking.v1.ClientPlayNetworking; +import net.fabricmc.fabric.api.client.screen.v1.ScreenEvents; +import net.fabricmc.fabric.api.client.screen.v1.ScreenMouseEvents; import net.fabricmc.fabric.api.resource.ResourceManagerHelper; import net.fabricmc.fabric.api.resource.ResourcePackActivationType; import net.fabricmc.loader.api.FabricLoader; import net.fabricmc.loader.api.ModContainer; -import org.thinkingstudio.obsidianui.fabric.event.OpenScreenCallback; import java.util.Optional; @@ -51,6 +53,9 @@ public class MidnightControlsClientFabric implements ClientModInitializer { ClientPlayConnectionEvents.DISCONNECT.register((handler, client) -> MidnightControlsClient.onLeave()); ClientTickEvents.START_CLIENT_TICK.register(MidnightControlsClient::onTick); + ScreenEvents.AFTER_INIT.register((client, screen, scaledWidth, scaledHeight) -> { + ScreenMouseEvents.allowMouseClick(screen).register(new MouseClickListener(screen)); + }); FabricLoader.getInstance().getModContainer(MidnightControlsConstants.NAMESPACE).ifPresent(modContainer -> { ResourceManagerHelper.registerBuiltinResourcePack(id("bedrock"), modContainer, ResourcePackActivationType.NORMAL); diff --git a/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/event/MouseClickListener.java b/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/event/MouseClickListener.java new file mode 100644 index 0000000..6eefac9 --- /dev/null +++ b/fabric/src/main/java/eu/midnightdust/midnightcontrols/fabric/event/MouseClickListener.java @@ -0,0 +1,217 @@ +package eu.midnightdust.midnightcontrols.fabric.event; +import eu.midnightdust.midnightcontrols.client.gui.VirtualKeyboardScreen; +import eu.midnightdust.midnightcontrols.client.mixin.AbstractSignEditScreenAccessor; +import eu.midnightdust.midnightcontrols.client.mixin.BookEditScreenAccessor; +import eu.midnightdust.midnightcontrols.client.mixin.CreativeInventoryScreenAccessor; +import net.fabricmc.fabric.api.client.screen.v1.ScreenMouseEvents; +import net.minecraft.block.entity.SignText; +import net.minecraft.client.gui.Element; +import net.minecraft.client.gui.ParentElement; +import net.minecraft.client.gui.screen.ChatScreen; +import net.minecraft.client.gui.screen.Screen; +import net.minecraft.client.gui.screen.ingame.BookEditScreen; +import net.minecraft.client.gui.screen.ingame.CreativeInventoryScreen; +import net.minecraft.client.gui.screen.ingame.SignEditScreen; +import net.minecraft.client.gui.widget.TextFieldWidget; +import org.lwjgl.glfw.GLFW; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static eu.midnightdust.midnightcontrols.MidnightControls.logger; +import static eu.midnightdust.midnightcontrols.client.MidnightControlsClient.client; + +record ScreenLink(Screen screen, List elementPath) {} + +public class MouseClickListener implements ScreenMouseEvents.AllowMouseClick { + private final Screen screen; + + private ScreenLink link; + + public MouseClickListener(Screen screen) { + this.screen = screen; + } + + @Override + public boolean allowMouseClick(Screen screen, double mouseX, double mouseY, int button) { + interceptMouseClick(screen, mouseX, mouseY); + return true; + } + + private void interceptMouseClick(Screen screen, double mouseX, double mouseY) { + logger.info("In scr: {}", screen.getClass()); + switch(screen) { + case BookEditScreen bookEditScreen -> { + if(screen.hoveredElement(mouseX, mouseY).isPresent()) { + return; + } + handleBookEditScreenClick(bookEditScreen); + } + case SignEditScreen signEditScreen -> { + if(screen.hoveredElement(mouseX, mouseY).isPresent()) { + return; + } + handleSignEditScreenClick(signEditScreen); + } + default -> { + var textField = findClickedTextField(screen, mouseX, mouseY); + if (textField == null) { + return; + } + handleTextFieldClick(textField); + } + } + } + + // Add equals and hashCode to prevent duplicate registrations + @Override + public boolean equals(Object obj) { + if (obj instanceof MouseClickListener) { + return ((MouseClickListener) obj).screen == this.screen; + } + return false; + } + + @Override + public int hashCode() { + return screen.hashCode(); + } + + // handlers + + private void handleBookEditScreenClick(BookEditScreen bookEditScreen) { + var accessor = (BookEditScreenAccessor) screen; + + VirtualKeyboardScreen virtualKeyboardScreen; + if(accessor.midnightcontrols$isSigning()) { + virtualKeyboardScreen = new VirtualKeyboardScreen(accessor.midnightcontrols$getTitle(), (text) -> { + client.setScreen(bookEditScreen); + accessor.midnightcontrols$setTitle(text); + }); + } + else { + virtualKeyboardScreen = new VirtualKeyboardScreen(accessor.midnightcontrols$getCurrentPageContent(), (text) -> { + client.setScreen(bookEditScreen); + accessor.midnightcontrols$setPageContent(text); + }); + } + + client.setScreen(virtualKeyboardScreen); + } + + private void handleSignEditScreenClick(SignEditScreen signEditScreen) { + var accessor = (AbstractSignEditScreenAccessor) signEditScreen; + // TODO + } + + private void handleTextFieldClick(TextFieldWidget textField) { + this.link = new ScreenLink(screen, calculatePathToElement(screen, textField)); + var virtualKeyboardScreen = new VirtualKeyboardScreen(textField.getText(), this::handleKeyboardClose); + client.setScreen(virtualKeyboardScreen); + } + + private void handleKeyboardClose(String newText) { + if(this.link == null) { + return; + } + + client.setScreen(this.link.screen()); + var txtField = findTextFieldByPath(screen, this.link.elementPath()); + if (txtField == null) { + return; + } + + txtField.setText(newText); + + + switch (this.link.screen()) { + case CreativeInventoryScreen creativeInventoryScreen -> { + var accessor = (CreativeInventoryScreenAccessor) creativeInventoryScreen; + accessor.midnightcontrols$search(); + } + case ChatScreen chatScreen -> { + // send the chat message + chatScreen.keyPressed(GLFW.GLFW_KEY_ENTER, 0, 0); + } + default -> {} + } + } + + // utility + + private TextFieldWidget findClickedTextField(Screen screen, double mouseX, double mouseY) { + for (Element element : screen.children()) { + if (element instanceof TextFieldWidget textField) { + if (textField.isMouseOver(mouseX, mouseY) && textField.isFocused()) { + return textField; + } + } + } + + // not hovering over a text field + return null; + } + + /** + * Calculates the path between a parent and a target in the UI hierarchy + */ + private List calculatePathToElement(Element parent, Element target) { + if (parent instanceof ParentElement parentElement) { + List children = parentElement.children(); + + // check direct children first + for (int i = 0; i < children.size(); i++) { + if (children.get(i) == target) { + // found it, return the path to this element + return Collections.singletonList(i); + } + } + + // check each child's children + for (int i = 0; i < children.size(); i++) { + if (children.get(i) instanceof ParentElement childParent) { + List subPath = calculatePathToElement(childParent, target); + if (subPath != null) { + // found in this subtree, prepend current index + List fullPath = new ArrayList<>(); + fullPath.add(i); + fullPath.addAll(subPath); + return fullPath; + } + } + } + } + + // Not found + return null; + } + + private TextFieldWidget findTextFieldByPath(Element parent, List path) { + if (path == null || path.isEmpty()) { + return null; + } + + if (parent instanceof ParentElement parentElement) { + List children = parentElement.children(); + int index = path.getFirst(); + + if (index >= 0 && index < children.size()) { + Element child = children.get(index); + + if (path.size() == 1) { + // This should be our target + return (child instanceof TextFieldWidget) ? (TextFieldWidget) child : null; + } else { + // Continue traversing + if (child instanceof ParentElement) { + return findTextFieldByPath(child, path.subList(1, path.size())); + } + } + } + } + + return null; + } +}