Skip to content

Commit

Permalink
enhancement(LorieView.java): improve input handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@twaik
twaik committed Jan 14, 2025
1 parent 989d08a commit 57c899d
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 76 deletions.
19 changes: 17 additions & 2 deletions app/src/main/cpp/lorie/activity.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ extern volatile int conn_fd; // The only variable from shared with X server code

static struct {
jclass self;
jmethodID getInstance, clientConnectedStateChanged;
jmethodID getInstance, clientConnectedStateChanged, flushComposingView;
} MainActivity = {0};

static struct {
Expand Down Expand Up @@ -115,6 +115,7 @@ static void nativeInit(JNIEnv *env, jobject thiz) {
MainActivity.self = FindClassOrDie(env, "com/termux/x11/MainActivity");
MainActivity.getInstance = FindMethodOrDie(env, MainActivity.self, "getInstance", "()Lcom/termux/x11/MainActivity;", JNI_TRUE);
MainActivity.clientConnectedStateChanged = FindMethodOrDie(env, MainActivity.self, "clientConnectedStateChanged", "()V", JNI_FALSE);
MainActivity.flushComposingView = FindMethodOrDie(env, (*env)->GetObjectClass(env, thiz), "flushComposingView", "()V", JNI_FALSE);
}

(*env)->GetJavaVM(env, &vm);
Expand All @@ -123,6 +124,13 @@ static void nativeInit(JNIEnv *env, jobject thiz) {
connect_(NULL, NULL, -1);
}

static void flushComposingView(JNIEnv* env) {
if (!env || !globalThiz)
return;

(*env)->CallVoidMethod(env, globalThiz, MainActivity.flushComposingView);
}

static int xcallback(int fd, int events, __unused void* data) {
JNIEnv *env = guienv;
jobject thiz = globalThiz;
Expand Down Expand Up @@ -260,13 +268,15 @@ static void setClipboardSyncEnabled(__unused JNIEnv* env, __unused jobject cls,

static void sendClipboardAnnounce(__unused JNIEnv *env, __unused jobject thiz) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .type = EVENT_CLIPBOARD_ANNOUNCE };
write(conn_fd, &e, sizeof(e));
}
}

static void sendClipboardEvent(JNIEnv *env, __unused jobject thiz, jbyteArray text) {
if (conn_fd != -1 && text) {
flushComposingView(env);
jsize length = (*env)->GetArrayLength(env, text);
jbyte* str = (*env)->GetByteArrayElements(env, text, NULL);
lorieEvent e = { .clipboardSend = { .t = EVENT_CLIPBOARD_SEND, .count = length } };
Expand All @@ -278,6 +288,7 @@ static void sendClipboardEvent(JNIEnv *env, __unused jobject thiz, jbyteArray te

static void sendWindowChange(__unused JNIEnv* env, __unused jobject cls, jint width, jint height, jint framerate, jstring jname) {
if (conn_fd != -1) {
flushComposingView(env);
const char *name = (!jname || width <= 0 || height <= 0) ? NULL : (*env)->GetStringUTFChars(env, jname, JNI_FALSE);
lorieEvent e = { .screenSize = { .t = EVENT_SCREEN_SIZE, .width = width, .height = height, .framerate = framerate, .name_size = (name ? strlen(name) : 0) } };
write(conn_fd, &e, sizeof(e));
Expand All @@ -290,13 +301,15 @@ static void sendWindowChange(__unused JNIEnv* env, __unused jobject cls, jint wi

static void sendMouseEvent(__unused JNIEnv* env, __unused jobject cls, jfloat x, jfloat y, jint which_button, jboolean button_down, jboolean relative) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .mouse = { .t = EVENT_MOUSE, .x = x, .y = y, .detail = which_button, .down = button_down, .relative = relative } };
write(conn_fd, &e, sizeof(e));
}
}

static void sendTouchEvent(__unused JNIEnv* env, __unused jobject cls, jint action, jint id, jint x, jint y) {
if (conn_fd != -1 && action != -1) {
flushComposingView(env);
lorieEvent e = { .touch = { .t = EVENT_TOUCH, .type = action, .id = id, .x = x, .y = y } };
write(conn_fd, &e, sizeof(e));
}
Expand All @@ -306,6 +319,7 @@ static void sendStylusEvent(__unused JNIEnv *env, __unused jobject thiz, jfloat
jint pressure, jint tilt_x, jint tilt_y,
jint orientation, jint buttons, jboolean eraser, jboolean mouse) {
if (conn_fd != -1) {
flushComposingView(env);
lorieEvent e = { .stylus = { .t = EVENT_STYLUS, .x = x, .y = y, .pressure = pressure, .tilt_x = tilt_x, .tilt_y = tilt_y, .orientation = orientation, .buttons = buttons, .eraser = eraser, .mouse = mouse } };
write(conn_fd, &e, sizeof(e));
}
Expand All @@ -320,6 +334,7 @@ static void requestStylusEnabled(__unused JNIEnv *env, __unused jclass clazz, jb

static jboolean sendKeyEvent(__unused JNIEnv* env, __unused jobject cls, jint scan_code, jint key_code, jboolean key_down) {
if (conn_fd != -1) {
flushComposingView(env);
int code = (scan_code) ?: android_to_linux_keycode[key_code];
log(DEBUG, "Sending key: %d (%d %d %d)", code + 8, scan_code, key_code, key_down);
lorieEvent e = { .key = { .t = EVENT_KEY, .key = code + 8, .state = key_down } };
Expand Down Expand Up @@ -358,7 +373,7 @@ static void sendTextEvent(JNIEnv *env, __unused jobject thiz, jbyteArray text) {
p += len;
if (p - (char*) str >= length)
break;
usleep(30000);
usleep(10000);
}

(*env)->ReleaseByteArrayElements(env, text, str, JNI_ABORT);
Expand Down
197 changes: 137 additions & 60 deletions app/src/main/java/com/termux/x11/LorieView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.drawable.ColorDrawable;
import android.os.Build;
import android.text.Editable;
import android.text.InputFilter;
import android.text.InputType;
import android.util.AttributeSet;
import android.util.Log;
Expand All @@ -24,17 +24,20 @@
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputMethodManager;
import android.view.inputmethod.InputMethodSubtype;
import android.view.inputmethod.TextSnapshot;
import android.widget.TextView;

import androidx.annotation.Keep;
import androidx.annotation.NonNull;

import com.termux.x11.input.InputStub;
import com.termux.x11.input.TouchInputHandler;

import java.nio.charset.StandardCharsets;
import java.nio.CharBuffer;
import java.util.regex.PatternSyntaxException;

import static java.nio.charset.StandardCharsets.UTF_8;

import dalvik.annotation.optimization.CriticalNative;
import dalvik.annotation.optimization.FastNative;

Expand All @@ -53,12 +56,113 @@ interface PixelFormat {
private long lastClipboardTimestamp = System.currentTimeMillis();
private static boolean clipboardSyncEnabled = false;
private static boolean hardwareKbdScancodesWorkaround = false;
private final InputMethodManager mIMM = (InputMethodManager)getContext().getSystemService(Context.INPUT_METHOD_SERVICE);
private String mImeLang;
private boolean mImeCJK;
public boolean enableGboardCJK;
private Callback mCallback;
private final Point p = new Point();
private TextView composingView;
private final InputMethodManager mImm = MainActivity.getInstance().getSystemService(InputMethodManager.class);
private final Editable mEditable = new Editable() {
// Wrap editable methods and hook and intercept changes
void onChange() {
int len = thiz.length();
char[] chars = new char[len];
thiz.getChars(0, len, chars, 0);
onComposingTextChange(chars, len);
}
private final Editable thiz = Editable.Factory.getInstance().newEditable("");
@Override public int length() { return thiz.length(); }
@Override public char charAt(int index) { return thiz.charAt(index); }
@NonNull @Override public CharSequence subSequence(int start, int end) { return thiz.subSequence(start, end); }
@Override public <T> T[] getSpans(int start, int end, Class<T> type) { return thiz.getSpans(start, end, type); }
@Override public int getSpanStart(Object tag) { return thiz.getSpanStart(tag); }
@Override public int getSpanEnd(Object tag) { return thiz.getSpanEnd(tag); }
@Override public int getSpanFlags(Object tag) { return thiz.getSpanFlags(tag); }
@Override public int nextSpanTransition(int start, int limit, Class type) { return thiz.nextSpanTransition(start, limit, type); }
@Override public void setSpan(Object what, int start, int end, int flags) { thiz.setSpan(what, start, end, flags); onChange(); }
@Override public void removeSpan(Object what) { thiz.removeSpan(what); onChange(); }
@Override public void getChars(int start, int end, char[] dest, int destoff) { thiz.getChars(start, end, dest, destoff); }
@NonNull @Override public Editable replace(int st, int en, CharSequence source, int start, int end) { Editable e = thiz.replace(st, en, source, start, end); onChange(); android.util.Log.d("EDITABLE", "replace0 "); return e; }
@NonNull @Override public Editable replace(int st, int en, CharSequence text) { Editable e = thiz.replace(st, en, text); onChange(); android.util.Log.d("EDITABLE", "replace1 " + st + " " + en + " " + text); return e; }
@NonNull @Override public Editable insert(int where, CharSequence text, int start, int end) { Editable e = thiz.insert(where, text, start, end); onChange(); android.util.Log.d("EDITABLE", "insert0"); return e; }
@NonNull @Override public Editable insert(int where, CharSequence text) { Editable e = thiz.insert(where, text); onChange(); android.util.Log.d("EDITABLE", "insert1"); return e; }
@NonNull @Override public Editable delete(int st, int en) { Editable e = thiz.delete(st, en); onChange(); android.util.Log.d("EDITABLE", "delete"); return e; }
@NonNull @Override public Editable append(CharSequence text) { Editable e = thiz.append(text); onChange(); android.util.Log.d("EDITABLE", "append0"); return e; }
@NonNull @Override public Editable append(CharSequence text, int start, int end) { Editable e = thiz.append(text, start, end); onChange(); android.util.Log.d("EDITABLE", "append1"); return e; }
@NonNull @Override public Editable append(char text) { Editable e = thiz.append(text); onChange(); android.util.Log.d("EDITABLE", "append2"); return e; }
@Override public void clear() { thiz.clear(); onChange(); }
@Override public void clearSpans() { thiz.clearSpans(); onChange(); }
@Override public void setFilters(InputFilter[] filters) { thiz.setFilters(filters); onChange(); }
@Override public InputFilter[] getFilters() { return thiz.getFilters(); }
};
private final InputConnection mConnection = new BaseInputConnection(this, false) {
private final MainActivity a = MainActivity.getInstance();
private final CharSequence seq = " ";

@Override
public Editable getEditable() {
return mEditable;
}

// Needed to send arrow keys with IME's cursor control feature
@Override
public CharSequence getTextBeforeCursor(int length, int flags) {
return seq;
}

// Needed to send arrow keys with IME's cursor control feature
@Override
public CharSequence getTextAfterCursor(int length, int flags) {
return seq;
}

void sendKey(int k) {
LorieView.this.sendKeyEvent(0, k, true);
LorieView.this.sendKeyEvent(0, k, false);
}

@Override
public boolean deleteSurroundingText(int beforeLength, int afterLength) {
for (int i=0; i<beforeLength; i++) sendKey(KeyEvent.KEYCODE_DEL);
for (int i=0; i<afterLength; i++) sendKey(KeyEvent.KEYCODE_FORWARD_DEL);
return true;
}

// Needed to send arrow keys with IME's cursor control feature
@Override
public boolean setComposingRegion(int start, int end) {
return true;
}

@Override
public boolean commitText(CharSequence text, int newCursorPosition) {
sendTextEvent(text.toString().getBytes(UTF_8));
mEditable.clear();
mImm.restartInput(LorieView.this);
// Do not steal dedicated buttons from a full external keyboard.
if (a.useTermuxEKBarBehaviour && a.mExtraKeys != null)
a.mExtraKeys.unsetSpecialKeys();
return true;
}

@Override
public boolean finishComposingText() {
synchronized (mEditable) {
if (mEditable.length() <= 0)
return true;

char[] t = new char[mEditable.length()];
mEditable.getChars(0, mEditable.length(), t, 0);
mEditable.clear();
sendTextEvent(UTF_8.encode(CharBuffer.wrap(t)).array());
}

return true;
}

@Override
public TextSnapshot takeSnapshot() {
return null;
}
};
private final SurfaceHolder.Callback mSurfaceCallback = new SurfaceHolder.Callback() {
@Override public void surfaceCreated(@NonNull SurfaceHolder holder) {
holder.setFormat(PixelFormat.BGRA_8888);
Expand Down Expand Up @@ -235,8 +339,6 @@ public void reloadPreferences(Prefs p) {
clipboardSyncEnabled = p.clipboardEnable.get();
setClipboardSyncEnabled(clipboardSyncEnabled, clipboardSyncEnabled);
TouchInputHandler.refreshInputDevices();
enableGboardCJK = p.enableGboardCJK.get();
mIMM.restartInput(this);
}

// It is used in native code
Expand All @@ -252,14 +354,14 @@ void setClipboardText(String text) {
/** @noinspection unused*/ // It is used in native code
void requestClipboard() {
if (!clipboardSyncEnabled) {
sendClipboardEvent("".getBytes(StandardCharsets.UTF_8));
sendClipboardEvent("".getBytes(UTF_8));
return;
}

CharSequence clip = clipboard.getText();
if (clip != null) {
String text = String.valueOf(clipboard.getText());
sendClipboardEvent(text.getBytes(StandardCharsets.UTF_8));
sendClipboardEvent(text.getBytes(UTF_8));
Log.d("CLIP", "sending clipboard contents: " + text);
}
}
Expand Down Expand Up @@ -298,67 +400,42 @@ public void onWindowFocusChanged(boolean hasFocus) {
TouchInputHandler.refreshInputDevices();
}

public void checkRestartInput(boolean recheck) {
if (!enableGboardCJK)
public void setComposingView(TextView v) {
composingView = v;
}

private void onComposingTextChange(char[] chars, int len) {
if (composingView == null)
return;

InputMethodSubtype methodSubtype = mIMM.getCurrentInputMethodSubtype();
String languageTag = methodSubtype == null ? null : methodSubtype.getLanguageTag();
if (languageTag != null && languageTag.length() >= 2 && !languageTag.substring(0, 2).equals(mImeLang))
mIMM.restartInput(this);
else if (recheck) { // recheck needed because sometimes requestCursorUpdates() is called too fast, before InputMethodManager detect change in IM subtype
MainActivity.handler.postDelayed(() -> checkRestartInput(false), 40);
}
composingView.setText(String.format(" %s ", new String(chars, 0, len)));
composingView.setVisibility(len == 0 ? LorieView.INVISIBLE : LorieView.VISIBLE);
}

@Override
public InputConnection onCreateInputConnection(EditorInfo outAttrs) {
outAttrs.inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS | InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD;

outAttrs.inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS | InputType.TYPE_TEXT_VARIATION_NORMAL;
outAttrs.actionLabel = "↵";
// Note that IME_ACTION_NONE cannot be used as that makes it impossible to input newlines using the on-screen
// keyboard on Android TV (see https://github.com/termux/termux-app/issues/221).
outAttrs.imeOptions = EditorInfo.IME_FLAG_NO_FULLSCREEN;
mEditable.clear();
return mConnection;
}

if (enableGboardCJK) {
InputMethodSubtype methodSubtype = mIMM.getCurrentInputMethodSubtype();
mImeLang = methodSubtype == null ? null : methodSubtype.getLanguageTag();
if (mImeLang != null && mImeLang.length() > 2)
mImeLang = mImeLang.substring(0, 2);
mImeCJK = mImeLang != null && (mImeLang.equals("zh") || mImeLang.equals("ko") || mImeLang.equals("ja"));
outAttrs.inputType = InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS |
(mImeCJK ? InputType.TYPE_TEXT_VARIATION_NORMAL : InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD);
return new BaseInputConnection(this, false) {
// workaround for Gboard
// Gboard calls requestCursorUpdates() whenever switching language
// check and then restart keyboard in different inputType when needed
@Override
public Editable getEditable() {
checkRestartInput(true);
return super.getEditable();
}
@Override
public boolean requestCursorUpdates(int cursorUpdateMode) {
checkRestartInput(true);
return super.requestCursorUpdates(cursorUpdateMode);
}
// Used in native method
/** @noinspection unused*/
@Keep
private void flushComposingView() {
synchronized (mEditable) {
if (mEditable.length() <= 0)
return;
}

@Override
public boolean commitText(CharSequence text, int newCursorPosition) {
boolean result = super.commitText(text, newCursorPosition);
if (mImeCJK)
// suppress Gboard CJK keyboard suggestion
// this workaround does not work well for non-CJK keyboards
// , when typing fast and two keypresses (commitText) are close in time
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU)
mIMM.invalidateInput(LorieView.this);
else
mIMM.restartInput(LorieView.this);
return result;
}
};
} else {
return super.onCreateInputConnection(outAttrs);
if (mEditable.length() > 0) {
mImm.restartInput(this);
}
mConnection.finishComposingText();
}

static native boolean renderingInActivity();
Expand Down
Loading

0 comments on commit 57c899d

Please sign in to comment.