From 5e978e49d7e9cc5afdbeb0756f55fda33ff38044 Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Sat, 30 May 2026 21:57:46 +1000 Subject: [PATCH 1/9] Add unattended remote access support - Add foreground service, boot receiver, and accessibility remote control - Route agent lifecycle through shared runtime controller - Add unattended setup prompts and settings status entries - Improve remote desktop refresh and session notifications - Update Gradle and scanner dependency versions --- .gitignore | 1 + .idea/git_toolbox_prj.xml | 15 + app/build.gradle | 14 +- app/src/main/AndroidManifest.xml | 35 +- .../agent/AgentForegroundService.kt | 242 ++++++ .../com/meshcentral/agent/AgentRuntime.kt | 719 ++++++++++++++++++ .../com/meshcentral/agent/BootReceiver.kt | 17 + .../meshcentral/agent/DesktopFrameEncoder.kt | 171 +++++ .../com/meshcentral/agent/MainActivity.kt | 436 ++++------- .../com/meshcentral/agent/MainFragment.kt | 4 +- .../agent/MeshAccessibilityService.kt | 332 ++++++++ .../java/com/meshcentral/agent/MeshAgent.kt | 12 +- .../agent/MeshFirebaseMessagingService.kt | 25 +- .../java/com/meshcentral/agent/MeshTunnel.kt | 65 +- .../meshcentral/agent/ScreenCaptureService.kt | 345 +++------ .../com/meshcentral/agent/SettingsFragment.kt | 69 +- app/src/main/res/menu/menu_main.xml | 5 + app/src/main/res/values/strings.xml | 36 +- .../res/xml/mesh_accessibility_service.xml | 10 + app/src/main/res/xml/root_preferences.xml | 29 +- build.gradle | 17 +- gradle.properties | 3 +- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 20 + 24 files changed, 2044 insertions(+), 580 deletions(-) create mode 100644 .idea/git_toolbox_prj.xml create mode 100644 app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt create mode 100644 app/src/main/java/com/meshcentral/agent/AgentRuntime.kt create mode 100644 app/src/main/java/com/meshcentral/agent/BootReceiver.kt create mode 100644 app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt create mode 100644 app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt create mode 100644 app/src/main/res/xml/mesh_accessibility_service.xml mode change 100644 => 100755 gradlew diff --git a/.gitignore b/.gitignore index aa724b7..92a9553 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ /.idea/workspace.xml /.idea/navEditor.xml /.idea/assetWizardSettings.xml +.vscode/ .DS_Store /build /captures diff --git a/.idea/git_toolbox_prj.xml b/.idea/git_toolbox_prj.xml new file mode 100644 index 0000000..02b915b --- /dev/null +++ b/.idea/git_toolbox_prj.xml @@ -0,0 +1,15 @@ + + + + + + + \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index 1712de1..8e69f09 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -4,20 +4,25 @@ plugins { id 'com.google.gms.google-services' } +def meshDevBuild = (project.findProperty("meshDevBuild") ?: new Date().format("yyyyMMddHHmm")).toString() + android { defaultConfig { compileSdk 34 targetSdk 34 applicationId "com.meshcentral.agent2" minSdk 23 - versionCode 28 - versionName "1.0.21" + versionCode 29 + versionName "1.0.22" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + def enterpriseEnforced = (project.findProperty("meshEnterpriseEnforced") ?: "false").toString().toBoolean() + buildConfigField "boolean", "ENTERPRISE_ENFORCED", enterpriseEnforced.toString() } buildTypes { debug { debuggable true + versionNameSuffix "-dev.$meshDevBuild" } release { // Enables code shrinking, obfuscation, and optimization for only @@ -44,6 +49,9 @@ android { kotlinOptions { jvmTarget = '1.8' } + buildFeatures { + buildConfig true + } namespace 'com.meshcentral.agent' } @@ -56,7 +64,7 @@ dependencies { implementation 'androidx.constraintlayout:constraintlayout:2.1.4' implementation 'androidx.navigation:navigation-fragment-ktx:2.7.7' implementation 'androidx.navigation:navigation-ui-ktx:2.7.7' - implementation 'com.budiyev.android:code-scanner:2.1.0' + implementation 'com.github.yuriy-budiyev:code-scanner:2.3.2' implementation 'com.karumi:dexter:6.2.2' implementation 'com.squareup.okhttp3:okhttp:4.9.0' implementation 'com.madgag.spongycastle:bcpkix-jdk15on:1.58.0.0' diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index e618687..10a925c 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -16,6 +16,9 @@ + + + + + + + + + + + + + + + + + + - \ No newline at end of file + diff --git a/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt b/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt new file mode 100644 index 0000000..52b2e15 --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt @@ -0,0 +1,242 @@ +package com.meshcentral.agent + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.graphics.Color +import android.net.Uri +import android.os.Build +import android.os.IBinder +import androidx.core.app.NotificationCompat +import androidx.core.app.NotificationManagerCompat +import androidx.core.app.ServiceCompat +import androidx.core.content.ContextCompat + +class AgentForegroundService : Service() { + override fun onCreate() { + super.onCreate() + AgentController.attachService(this) + createNotificationChannel(this) + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + AgentController.init(applicationContext) + promoteToForeground() + when (intent?.action) { + ACTION_CONNECT -> if (meshAgent == null) AgentController.toggleAgentConnection(false) + ACTION_DISCONNECT -> if (!AgentController.enterpriseEnforced && meshAgent != null) AgentController.toggleAgentConnection(true) + ACTION_STOP_SCREEN_SHARING -> AgentController.stopScreenSharingByUser() + ACTION_STOP -> { + if (!AgentController.enterpriseEnforced) { + if (meshAgent != null) AgentController.toggleAgentConnection(true) + stopSelf() + } + } + else -> { + if (AgentController.shouldAutoStart() && meshAgent == null && !g_userDisconnect) { + AgentController.toggleAgentConnection(false) + } + } + } + updateNotification() + return START_STICKY + } + + override fun onDestroy() { + try { + NotificationManagerCompat.from(this).cancel(SESSION_NOTIFICATION_ID) + } catch (_: SecurityException) { + } + AgentController.detachService(this) + super.onDestroy() + } + + override fun onBind(intent: Intent?): IBinder? = null + + fun updateNotification() { + try { + NotificationManagerCompat.from(this).notify(NOTIFICATION_ID, buildNotification(this)) + } catch (_: SecurityException) { + } + updateSessionNotification() + } + + private fun updateSessionNotification() { + val manager = NotificationManagerCompat.from(this) + val users = if (g_sessionNotification) AgentController.activeSessionUsers() else emptyList() + if (users.isEmpty()) { + try { + manager.cancel(SESSION_NOTIFICATION_ID) + } catch (_: SecurityException) { + } + return + } + createSessionNotificationChannel(this) + val text = if (users.size == 1) { + getString(R.string.session_connected_one, users[0]) + } else { + getString(R.string.session_connected_many, users.size) + } + val notification = NotificationCompat.Builder(this, SESSION_CHANNEL_ID) + .setSmallIcon(R.drawable.ic_cloud) + .setContentTitle(getString(R.string.session_active_title)) + .setContentText(text) + .setStyle(NotificationCompat.BigTextStyle().bigText(users.joinToString("\n"))) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setCategory(Notification.CATEGORY_STATUS) + .setPriority(NotificationCompat.PRIORITY_DEFAULT) + .setContentIntent(openAppPendingIntent(this, null)) + .also { builder -> + if (!AgentController.enterpriseEnforced && AgentController.hasActiveDesktopTunnel()) { + builder.addAction( + R.drawable.ic_cloud, + getString(R.string.stopsharescreen), + servicePendingIntent(this, ACTION_STOP_SCREEN_SHARING, 3) + ) + } + } + .build() + try { + manager.notify(SESSION_NOTIFICATION_ID, notification) + } catch (_: SecurityException) { + } + } + + private fun promoteToForeground() { + val type = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE + } else { + 0 + } + ServiceCompat.startForeground(this, NOTIFICATION_ID, buildNotification(this), type) + } + + companion object { + private const val CHANNEL_ID = "meshcentral_agent_foreground" + private const val CHANNEL_NAME = "MeshCentral Agent" + private const val SESSION_CHANNEL_ID = "meshcentral_agent_session" + private const val SESSION_CHANNEL_NAME = "Remote session active" + private const val NOTIFICATION_ID = 2401 + private const val RUNTIME_NOTIFICATION_ID = 2402 + private const val SESSION_NOTIFICATION_ID = 2403 + private const val ACTION_CONNECT = "com.meshcentral.agent.action.CONNECT" + private const val ACTION_DISCONNECT = "com.meshcentral.agent.action.DISCONNECT" + private const val ACTION_STOP_SCREEN_SHARING = "com.meshcentral.agent.action.STOP_SCREEN_SHARING" + private const val ACTION_STOP = "com.meshcentral.agent.action.STOP" + + fun start(context: Context) { + val intent = Intent(context, AgentForegroundService::class.java) + ContextCompat.startForegroundService(context.applicationContext, intent) + } + + fun connect(context: Context) { + val intent = Intent(context, AgentForegroundService::class.java) + intent.action = ACTION_CONNECT + ContextCompat.startForegroundService(context.applicationContext, intent) + } + + fun disconnect(context: Context) { + val intent = Intent(context, AgentForegroundService::class.java) + intent.action = ACTION_DISCONNECT + ContextCompat.startForegroundService(context.applicationContext, intent) + } + + fun showOneShotNotification(context: Context, title: String?, body: String?, url: String?) { + createNotificationChannel(context) + val notification = NotificationCompat.Builder(context, CHANNEL_ID) + .setSmallIcon(R.drawable.ic_message) + .setContentTitle(title ?: context.getString(R.string.app_name)) + .setContentText(body ?: "") + .setStyle(NotificationCompat.BigTextStyle().bigText(body ?: "")) + .setContentIntent(openAppPendingIntent(context, url)) + .setAutoCancel(true) + .setPriority(NotificationCompat.PRIORITY_DEFAULT) + .build() + try { + NotificationManagerCompat.from(context).notify(RUNTIME_NOTIFICATION_ID, notification) + } catch (_: SecurityException) { + } + } + + private fun buildNotification(context: Context): Notification { + val state = when (meshAgent?.state ?: 0) { + 1 -> context.getString(R.string.connecting) + 2 -> context.getString(R.string.authenticating) + 3 -> context.getString(R.string.connected) + else -> context.getString(R.string.disconnected) + } + val builder = NotificationCompat.Builder(context, CHANNEL_ID) + .setSmallIcon(R.drawable.ic_cloud) + .setContentTitle(context.getString(R.string.app_name)) + .setContentText(state) + .setOngoing(true) + .setCategory(Notification.CATEGORY_SERVICE) + .setPriority(NotificationCompat.PRIORITY_LOW) + .setShowWhen(false) + .setContentIntent(openAppPendingIntent(context, null)) + + if (!AgentController.enterpriseEnforced) { + if (meshAgent == null) { + builder.addAction(R.drawable.ic_cloud, context.getString(R.string.connect), servicePendingIntent(context, ACTION_CONNECT, 1)) + } else { + builder.addAction(R.drawable.ic_cloud, context.getString(R.string.disconnect), servicePendingIntent(context, ACTION_DISCONNECT, 2)) + } + } + return builder.build() + } + + private fun servicePendingIntent(context: Context, action: String, requestCode: Int): PendingIntent { + val intent = Intent(context, AgentForegroundService::class.java) + intent.action = action + val flags = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT + } else { + PendingIntent.FLAG_UPDATE_CURRENT + } + return PendingIntent.getService(context, requestCode, intent, flags) + } + + private fun openAppPendingIntent(context: Context, url: String?): PendingIntent { + val intent = Intent(context, MainActivity::class.java) + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP) + if (url != null) { + if (url.startsWith("http://") || url.startsWith("https://")) { + intent.data = Uri.parse(url) + } + intent.putExtra("url", url) + } + val flags = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + PendingIntent.FLAG_MUTABLE or PendingIntent.FLAG_UPDATE_CURRENT + } else { + PendingIntent.FLAG_UPDATE_CURRENT + } + return PendingIntent.getActivity(context, 0, intent, flags) + } + + private fun createNotificationChannel(context: Context) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val channel = NotificationChannel(CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_LOW) + channel.lightColor = Color.BLUE + channel.lockscreenVisibility = Notification.VISIBILITY_PRIVATE + val manager = context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.createNotificationChannel(channel) + } + } + + private fun createSessionNotificationChannel(context: Context) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val channel = NotificationChannel(SESSION_CHANNEL_ID, SESSION_CHANNEL_NAME, NotificationManager.IMPORTANCE_DEFAULT) + channel.lightColor = Color.BLUE + channel.lockscreenVisibility = Notification.VISIBILITY_PRIVATE + val manager = context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.createNotificationChannel(channel) + } + } + } +} diff --git a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt new file mode 100644 index 0000000..23fae87 --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt @@ -0,0 +1,719 @@ +package com.meshcentral.agent + +import android.Manifest +import android.content.BroadcastReceiver +import android.content.ComponentName +import android.content.ContentResolver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.content.IntentSender +import android.content.SharedPreferences +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.PowerManager +import android.provider.Settings +import android.util.Base64 +import android.view.Gravity +import android.widget.Toast +import androidx.core.app.NotificationManagerCompat +import androidx.core.content.ContextCompat +import androidx.preference.PreferenceManager +import com.google.firebase.messaging.FirebaseMessaging +import okio.ByteString +import okio.ByteString.Companion.toByteString +import org.spongycastle.asn1.x500.X500Name +import org.spongycastle.cert.X509v3CertificateBuilder +import org.spongycastle.cert.jcajce.JcaX509CertificateConverter +import org.spongycastle.cert.jcajce.JcaX509v3CertificateBuilder +import org.spongycastle.jce.provider.BouncyCastleProvider +import org.spongycastle.operator.jcajce.JcaContentSignerBuilder +import java.io.ByteArrayInputStream +import java.math.BigInteger +import java.security.KeyFactory +import java.security.KeyPairGenerator +import java.security.SecureRandom +import java.security.Security +import java.security.cert.CertificateFactory +import java.security.cert.X509Certificate +import java.security.spec.PKCS8EncodedKeySpec +import java.util.Date +import java.util.Random +import kotlin.math.absoluteValue + +interface AgentHost { + val contentResolver: ContentResolver + fun getApplicationContext(): Context + fun runOnHostThread(action: () -> Unit) + fun agentStateChanged() + fun refreshInfo() + fun startProjection() + fun stopProjection() + fun showAlertMessage(title: String, msg: String) + fun showToastMessage(msg: String) + fun openUrl(xpageUrl: String): Boolean + fun returnToMainScreen() + fun startActivity(intent: Intent) + fun launchIntentSenderForResult( + intentSender: IntentSender, + requestCode: Int, + fillInIntent: Intent?, + flagsMask: Int, + flagsValues: Int, + extraFlags: Int, + options: Bundle? + ): Boolean +} + +interface RemoteDesktopProvider { + val isRunning: Boolean + val width: Int + val height: Int + fun requestFullFrame() + fun handleMouseCommand(msg: ByteString): Boolean = false + fun handleTouchCommand(msg: ByteString): Boolean = false + fun handleKeyCommand(cmd: Int, msg: ByteString): Boolean = false +} + +object AgentController : AgentHost { + private lateinit var appContext: Context + private val mainHandler = Handler(Looper.getMainLooper()) + private var initialized = false + private var activity: MainActivity? = null + private var service: AgentForegroundService? = null + private var retryRunnable: Runnable? = null + private var batteryReceiver: BroadcastReceiver? = null + private var projectionRetryRunnable: Runnable? = null + private var projectionRetryCount = 0 + private val MAX_PROJECTION_RETRIES = 12 + + val enterpriseEnforced: Boolean + get() = BuildConfig.ENTERPRISE_ENFORCED + + override val contentResolver: ContentResolver + get() = appContext.contentResolver + + override fun getApplicationContext(): Context = appContext.applicationContext + + fun init(context: Context) { + ensureCryptoProvider() + appContext = context.applicationContext + loadServerLink() + loadSettings() + loadFirebaseToken() + if (!initialized) { + initialized = true + registerBatteryReceiver() + } + } + + fun attachActivity(mainActivity: MainActivity) { + init(mainActivity.applicationContext) + activity = mainActivity + g_mainActivity = mainActivity + refreshInfo() + } + + fun detachActivity(mainActivity: MainActivity) { + if (activity === mainActivity) { + activity = null + g_mainActivity = null + } + } + + fun attachService(agentService: AgentForegroundService) { + init(agentService.applicationContext) + service = agentService + } + + fun detachService(agentService: AgentForegroundService) { + if (service === agentService) { + service = null + } + } + + fun hasServerLink(): Boolean { + loadServerLink() + return serverLink != null + } + + fun shouldAutoStart(): Boolean { + loadServerLink() + loadSettings() + return serverLink != null && (enterpriseEnforced || g_autoConnect) + } + + fun setMeshServerLink(x: String?) { + if ((serverLink == x) || (hardCodedServerLink != null)) return + if (meshAgent != null) { + meshAgent?.Stop() + meshAgent = null + } + serverLink = x + val sharedPreferences = appContext.getSharedPreferences("meshagent", Context.MODE_PRIVATE) + sharedPreferences.edit().putString("qrmsh", x).apply() + + if (x != null) { + PreferenceManager.getDefaultSharedPreferences(appContext) + .edit() + .putBoolean("pref_autoconnect", true) + .apply() + g_autoConnect = true + AgentForegroundService.start(appContext) + requestBatteryOptimizationExemption() + } else { + stopProjection() + service?.stopSelf() + } + + g_userDisconnect = false + refreshInfo() + if (g_autoConnect || enterpriseEnforced) { + toggleAgentConnection(false) + } + } + + fun settingsChanged() { + loadSettings() + if (!enterpriseEnforced && !g_autoConnect) { + stopRetryTimer() + refreshInfo() + service?.updateNotification() + return + } + if ((meshAgent == null) && !g_userDisconnect && hasServerLink()) { + AgentForegroundService.start(appContext) + toggleAgentConnection(false) + } + refreshInfo() + service?.updateNotification() + } + + fun toggleAgentConnection(userInitiated: Boolean) { + loadServerLink() + loadSettings() + if ((meshAgent == null) && (serverLink != null)) { + ensureIdentity() + if (!userInitiated) { + g_userDisconnect = false + startAgent() + } else { + if (g_autoConnect || enterpriseEnforced) { + if (g_userDisconnect) { + g_userDisconnect = false + startAgent() + } else { + g_userDisconnect = true + stopRetryTimer() + } + } else { + g_userDisconnect = true + startAgent() + } + } + } else if (meshAgent != null) { + if (userInitiated && !enterpriseEnforced) { + g_userDisconnect = true + } + stopProjection() + meshAgent?.Stop() + meshAgent = null + stopRetryTimer() + } + refreshInfo() + service?.updateNotification() + } + + private fun startAgent() { + val host = getServerHost() ?: return + val hash = getServerHash() ?: return + val group = getDevGroup() ?: return + meshAgent = MeshAgent(this, host, hash, group) + meshAgent?.Start() + } + + override fun agentStateChanged() { + runOnHostThread { + if ((meshAgent != null) && (meshAgent?.state == 0)) { + meshAgent = null + } + if (((meshAgent != null) && (meshAgent?.state != 0)) || g_userDisconnect || (!g_autoConnect && !enterpriseEnforced)) { + stopRetryTimer() + } else if ((meshAgent == null) && !g_userDisconnect && (g_autoConnect || enterpriseEnforced) && retryRunnable == null) { + startRetryTimer() + } + refreshInfo() + service?.updateNotification() + } + } + + override fun refreshInfo() { + runOnHostThread { + mainFragment?.refreshInfo() + activity?.invalidateOptionsMenu() + service?.updateNotification() + } + } + + override fun runOnHostThread(action: () -> Unit) { + if (Looper.myLooper() == Looper.getMainLooper()) { + action() + } else { + mainHandler.post { action() } + } + } + + override fun startProjection() { + if (meshAgent == null || meshAgent?.state != 3) return + if (isRemoteDesktopRunning()) return + val accessibility = MeshAccessibilityService.instance + if (accessibility != null) { + cancelProjectionRetry() + if (accessibility.startDesktop()) return + } else if (isAccessibilityServiceEnabled() && waitForAccessibilityProjection()) { + // Unattended access is granted but the accessibility service has not rebound yet (common + // right after an app update). Wait for it to connect instead of reporting setup as missing. + return + } + cancelProjectionRetry() + + val mainActivity = activity + if (mainActivity != null) { + if (isAccessibilityServiceEnabled()) { + // Accessibility is granted but not currently usable (e.g. pre-Android 11, or it + // failed to bind); legacy screen capture is the only remaining option. + mainActivity.startMediaProjectionPrompt() + } else { + // Don't auto-pop Android's screen-capture consent. Offer Accessibility setup first + // and make legacy capture an explicit opt-in choice. + mainActivity.promptScreenShareChoice() + } + return + } + + sendDesktopMessage("Remote desktop requires Accessibility unattended access or an open app screen for Android capture consent.") + showToastMessage("Enable unattended access in settings to share the screen in the background.") + showRuntimeNotification( + appContext.getString(R.string.unattended_access_required), + appContext.getString(R.string.open_app_to_finish_unattended_setup), + null + ) + } + + override fun stopProjection() { + val provider = g_remoteDesktopProvider + if (provider is MeshAccessibilityService) { + provider.stopDesktop() + } + if (g_ScreenCaptureService != null) { + appContext.startService(ScreenCaptureService.getStopIntent(appContext)) + } + } + + fun stopScreenSharingByUser() { + val agent = meshAgent + if (agent != null) { + // Snapshot with filter() so closing tunnels (which mutates the list) is safe to iterate. + val desktopTunnels = agent.tunnels.filter { (it.state == 2) && (it.usage == 2) } + for (t in desktopTunnels) t.Stop() + } + if (isRemoteDesktopRunning()) stopProjection() + refreshInfo() + } + + fun isRemoteDesktopRunning(): Boolean { + return g_remoteDesktopProvider?.isRunning == true || g_ScreenCaptureService != null + } + + fun hasActiveDesktopTunnel(): Boolean { + val agent = meshAgent ?: return false + return agent.tunnels.any { (it.state == 2) && (it.usage == 2) } + } + + // Display names of the remote users with an active session (desktop or files), de-duplicated. + // File-transfer sub-tunnels (usage 10) are ignored so downloads don't flicker the notification. + fun activeSessionUsers(): List { + val agent = meshAgent ?: return emptyList() + val names = LinkedHashSet() + for (t in agent.tunnels.toList()) { + if (t.state != 2 || t.usage == 10) continue + val sessionUser = t.sessionUserName2 + if (sessionUser.isNullOrEmpty()) continue + names.add(friendlySessionName(agent, sessionUser)) + } + return names.toList() + } + + private fun friendlySessionName(agent: MeshAgent, sessionUserName2: String): String { + return try { + val parts = sessionUserName2.split("/") + if (parts.size >= 3) { + val userid = parts[0] + "/" + parts[1] + "/" + parts[2] + val guest = if (parts.size >= 4) " - " + parts[3] else "" + (agent.userinfo[userid]?.realname ?: parts[2]) + guest + } else { + sessionUserName2 + } + } catch (ex: Exception) { + sessionUserName2 + } + } + + // True if our accessibility service is enabled in Android settings. This is the durable signal + // for "unattended access is granted"; MeshAccessibilityService.instance is only set while the + // service is actively bound, which is transiently null after an app update or process restart. + fun isAccessibilityServiceEnabled(): Boolean { + if (!::appContext.isInitialized) return false + val enabledServices = try { + Settings.Secure.getString(appContext.contentResolver, Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES) + } catch (ex: Exception) { + null + } ?: return false + val component = ComponentName(appContext, MeshAccessibilityService::class.java) + val flat = component.flattenToString() + val flatShort = component.flattenToShortString() + return enabledServices.split(':').any { it.equals(flat, ignoreCase = true) || it.equals(flatShort, ignoreCase = true) } + } + + // Schedule another startProjection() attempt while we wait for the accessibility service to + // rebind. Returns false once the bounded number of attempts is exhausted so the caller can fall + // back to its normal handling. Bounded to roughly six seconds. + private fun waitForAccessibilityProjection(): Boolean { + if (projectionRetryCount >= MAX_PROJECTION_RETRIES) return false + projectionRetryCount++ + projectionRetryRunnable?.let { mainHandler.removeCallbacks(it) } + val runnable = Runnable { + projectionRetryRunnable = null + startProjection() + } + projectionRetryRunnable = runnable + mainHandler.postDelayed(runnable, 500) + return true + } + + private fun cancelProjectionRetry() { + projectionRetryRunnable?.let { mainHandler.removeCallbacks(it) } + projectionRetryRunnable = null + projectionRetryCount = 0 + } + + fun activeRemoteDesktopProvider(): RemoteDesktopProvider? { + return g_remoteDesktopProvider ?: g_ScreenCaptureService + } + + fun requestDesktopRefresh() { + activeRemoteDesktopProvider()?.requestFullFrame() + } + + fun isRetrying(): Boolean { + return retryRunnable != null + } + + fun checkNoMoreDesktopTunnels() { + val agent = meshAgent ?: return + val activeDesktopTunnels = agent.tunnels.count { (it.state == 2) && (it.usage == 2) } + if (activeDesktopTunnels == 0) { + stopProjection() + refreshInfo() + } + } + + fun sendDesktopTunnelData(data: ByteString) { + val agent = meshAgent ?: return + for (t in agent.tunnels) { + if ((t.state == 2) && (t.usage == 2) && (t._webSocket != null)) { + t._webSocket!!.send(data) + } + } + } + + fun sendDesktopMessage(message: String) { + val bytes = message.toByteArray(Charsets.UTF_8) + val data = ByteArray(4 + bytes.size) + data[1] = 17 + data[2] = ((data.size shr 8) and 0xFF).toByte() + data[3] = (data.size and 0xFF).toByte() + bytes.copyInto(data, 4) + sendDesktopTunnelData(data.toByteString()) + } + + fun handleDesktopMouseCommand(msg: ByteString): Boolean { + val provider = activeInputProvider() + if (provider != null && provider.handleMouseCommand(msg)) return true + sendDesktopMessage("Remote input requires Accessibility unattended access.") + return false + } + + fun handleDesktopTouchCommand(msg: ByteString): Boolean { + val provider = activeInputProvider() + if (provider != null && provider.handleTouchCommand(msg)) return true + sendDesktopMessage("Remote touch input requires Accessibility unattended access.") + return false + } + + fun handleDesktopKeyCommand(cmd: Int, msg: ByteString): Boolean { + val provider = activeInputProvider() + if (provider != null && provider.handleKeyCommand(cmd, msg)) return true + sendDesktopMessage("Remote keyboard input is limited on Android and requires Accessibility unattended access.") + return false + } + + private fun activeInputProvider(): RemoteDesktopProvider? { + val activeProvider = activeRemoteDesktopProvider() + if (activeProvider is MeshAccessibilityService) return activeProvider + return MeshAccessibilityService.instance ?: activeProvider + } + + override fun showAlertMessage(title: String, msg: String) { + val mainActivity = activity + if (mainActivity != null) { + mainActivity.showAlertMessage(title, msg) + } else { + showRuntimeNotification(title, msg, null) + } + } + + override fun showToastMessage(msg: String) { + runOnHostThread { + val toast = Toast.makeText(appContext, msg, Toast.LENGTH_LONG) + toast.setGravity(Gravity.CENTER, 0, 300) + toast.show() + } + } + + override fun openUrl(xpageUrl: String): Boolean { + val mainActivity = activity + if (mainActivity != null) { + return mainActivity.openUrlInApp(xpageUrl) + } + return try { + val intent = Intent(Intent.ACTION_VIEW, Uri.parse(xpageUrl)) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + appContext.startActivity(intent) + true + } catch (ex: Exception) { + false + } + } + + override fun returnToMainScreen() { + activity?.returnToMainScreen() + } + + override fun startActivity(intent: Intent) { + val mainActivity = activity + if (mainActivity != null) { + mainActivity.startActivity(intent) + } else { + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + appContext.startActivity(intent) + } + } + + override fun launchIntentSenderForResult( + intentSender: IntentSender, + requestCode: Int, + fillInIntent: Intent?, + flagsMask: Int, + flagsValues: Int, + extraFlags: Int, + options: Bundle? + ): Boolean { + val mainActivity = activity ?: return false + return try { + mainActivity.startIntentSenderForResult( + intentSender, + requestCode, + fillInIntent, + flagsMask, + flagsValues, + extraFlags, + options + ) + true + } catch (ex: Exception) { + false + } + } + + fun requestBatteryOptimizationExemption() { + if (!::appContext.isInitialized) return + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return + val powerManager = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager + if (powerManager.isIgnoringBatteryOptimizations(appContext.packageName)) return + val mainActivity = activity ?: return + if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) != PackageManager.PERMISSION_GRANTED) { + return + } + try { + val intent = Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) + intent.data = Uri.parse("package:${appContext.packageName}") + mainActivity.startActivity(intent) + } catch (ex: Exception) { + try { + mainActivity.startActivity(Intent(Settings.ACTION_IGNORE_BATTERY_OPTIMIZATION_SETTINGS)) + } catch (_: Exception) { + } + } + } + + fun isIgnoringBatteryOptimizations(): Boolean { + if (!::appContext.isInitialized) return false + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return true + val powerManager = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager + return powerManager.isIgnoringBatteryOptimizations(appContext.packageName) + } + + fun areNotificationsEnabled(): Boolean { + if (!::appContext.isInitialized) return false + if (!NotificationManagerCompat.from(appContext).areNotificationsEnabled()) return false + return Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU || + ContextCompat.checkSelfPermission(appContext, Manifest.permission.POST_NOTIFICATIONS) == PackageManager.PERMISSION_GRANTED + } + + fun showRuntimeNotification(title: String?, body: String?, url: String?) { + val mainActivity = activity + if (mainActivity != null) { + mainActivity.showNotification(title, body, url) + } else { + AgentForegroundService.showOneShotNotification(appContext, title, body, url) + } + } + + private fun loadServerLink() { + serverLink = if (hardCodedServerLink != null) { + hardCodedServerLink + } else { + appContext.getSharedPreferences("meshagent", Context.MODE_PRIVATE).getString("qrmsh", null) + } + } + + private fun loadSettings() { + val pm: SharedPreferences = PreferenceManager.getDefaultSharedPreferences(appContext) + g_autoConnect = enterpriseEnforced || pm.getBoolean("pref_autoconnect", false) + g_autoConsent = enterpriseEnforced || pm.getBoolean("pref_autoconsent", false) + g_sessionNotification = pm.getBoolean("pref_session_notification", false) + if (enterpriseEnforced) { + pm.edit() + .putBoolean("pref_autoconnect", true) + .putBoolean("pref_autoconsent", true) + .apply() + } + } + + private fun loadFirebaseToken() { + try { + FirebaseMessaging.getInstance().token.addOnSuccessListener { tokenString -> + pushMessagingToken = tokenString + meshAgent?.sendCoreInfo() + } + } catch (_: Exception) { + } + } + + private fun registerBatteryReceiver() { + if (batteryReceiver != null) return + batteryReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + meshAgent?.batteryStateChanged(intent) + } + } + val intentFilter = IntentFilter() + intentFilter.addAction(Intent.ACTION_POWER_CONNECTED) + intentFilter.addAction(Intent.ACTION_POWER_DISCONNECTED) + intentFilter.addAction(Intent.ACTION_BATTERY_CHANGED) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + appContext.registerReceiver(batteryReceiver, intentFilter, Context.RECEIVER_NOT_EXPORTED) + } else { + appContext.registerReceiver(batteryReceiver, intentFilter) + } + } + + private fun startRetryTimer() { + if (retryRunnable != null) return + retryRunnable = object : Runnable { + override fun run() { + if ((meshAgent == null) && !g_userDisconnect && (g_autoConnect || enterpriseEnforced)) { + toggleAgentConnection(false) + } + mainHandler.postDelayed(this, 10000) + } + } + mainHandler.postDelayed(retryRunnable!!, 10000) + } + + private fun stopRetryTimer() { + retryRunnable?.let { mainHandler.removeCallbacks(it) } + retryRunnable = null + } + + private fun ensureIdentity() { + if (agentCertificate != null && agentCertificateKey != null) return + val sharedPreferences = appContext.getSharedPreferences("meshagent", Context.MODE_PRIVATE) + val certb64: String? = sharedPreferences.getString("agentCert", null) + val keyb64: String? = sharedPreferences.getString("agentKey", null) + if ((certb64 == null) || (keyb64 == null)) { + val keyGen = KeyPairGenerator.getInstance("RSA") + keyGen.initialize(2048, SecureRandom()) + val keypair = keyGen.generateKeyPair() + + var serial = BigInteger("12345678") + try { + serial = BigInteger.valueOf(Random().nextInt().toLong().absoluteValue) + } catch (_: Exception) { + } + + val builder: X509v3CertificateBuilder = JcaX509v3CertificateBuilder( + X500Name("CN=android.agent.meshcentral.com"), + serial, + Date(System.currentTimeMillis() - 86400000L * 365), + Date(253402300799000L), + X500Name("CN=android.agent.meshcentral.com"), + keypair.public + ) + agentCertificate = JcaX509CertificateConverter().setProvider("SC").getCertificate( + builder.build(JcaContentSignerBuilder("SHA256withRSA").build(keypair.private)) + ) + agentCertificateKey = keypair.private + sharedPreferences.edit() + .putString("agentCert", Base64.encodeToString(agentCertificate?.encoded, Base64.DEFAULT)) + .putString("agentKey", Base64.encodeToString(agentCertificateKey?.encoded, Base64.DEFAULT)) + .apply() + } else { + agentCertificate = CertificateFactory.getInstance("X509").generateCertificate( + ByteArrayInputStream(Base64.decode(certb64, Base64.DEFAULT)) + ) as X509Certificate + val keySpec = PKCS8EncodedKeySpec(Base64.decode(keyb64, Base64.DEFAULT)) + agentCertificateKey = KeyFactory.getInstance("RSA").generatePrivate(keySpec) + } + } + + private fun ensureCryptoProvider() { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.insertProviderAt(BouncyCastleProvider(), 1) + } + } + + fun getServerHost(): String? { + val link = serverLink ?: return null + val x: List = link.split(',') + val serverHost = x[0] + return serverHost.substring(5) + } + + fun getServerHash(): String? { + val link = serverLink ?: return null + val x: List = link.split(',') + return x.getOrNull(1) + } + + fun getDevGroup(): String? { + val link = serverLink ?: return null + val x: List = link.split(',') + return x.getOrNull(2) + } +} diff --git a/app/src/main/java/com/meshcentral/agent/BootReceiver.kt b/app/src/main/java/com/meshcentral/agent/BootReceiver.kt new file mode 100644 index 0000000..0ffe37e --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/BootReceiver.kt @@ -0,0 +1,17 @@ +package com.meshcentral.agent + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent + +class BootReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + val action = intent.action ?: return + if (action == Intent.ACTION_BOOT_COMPLETED || action == Intent.ACTION_MY_PACKAGE_REPLACED) { + AgentController.init(context.applicationContext) + if (AgentController.shouldAutoStart()) { + AgentForegroundService.start(context) + } + } + } +} diff --git a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt new file mode 100644 index 0000000..eca9245 --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt @@ -0,0 +1,171 @@ +package com.meshcentral.agent + +import android.graphics.Bitmap +import okio.ByteString +import okio.ByteString.Companion.toByteString +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream + +class DesktopFrameEncoder { + private var tilesWide: Int = 0 + private var tilesHigh: Int = 0 + private var frameWidth: Int = 0 + private var frameHeight: Int = 0 + private var tilesCount: Int = 0 + private var oldcrcs: IntArray? = null + private var newcrcs: IntArray? = null + private var forceFullFrame = true + + fun requestFullFrame() { + forceFullFrame = true + } + + fun encode(bitmap: Bitmap, sink: (ByteString) -> Unit) { + if (frameWidth != bitmap.width || frameHeight != bitmap.height || oldcrcs == null || newcrcs == null) { + frameWidth = bitmap.width + frameHeight = bitmap.height + tilesWide = (bitmap.width + 63) / 64 + tilesHigh = (bitmap.height + 63) / 64 + tilesCount = tilesWide * tilesHigh + oldcrcs = IntArray(tilesCount) + newcrcs = IntArray(tilesCount) + forceFullFrame = true + } + + computeAllCRCs(bitmap) + var changedTiles = 0 + for (i in 0 until tilesCount) { + if (forceFullFrame || oldcrcs!![i] != newcrcs!![i]) changedTiles++ + } + if (changedTiles == 0) return + + if (forceFullFrame || ((changedTiles * 100) >= (tilesCount * 85))) { + sink(buildImageCommand(bitmap, 0, 0, bitmap.width, bitmap.height)) + for (i in 0 until tilesCount) oldcrcs!![i] = newcrcs!![i] + forceFullFrame = false + return + } + + var sendx = -1 + var sendy = 0 + var sendw = 0 + for (i in 0 until tilesHigh) { + for (j in 0 until tilesWide) { + val tileNumber = (i * tilesWide) + j + if (oldcrcs!![tileNumber] != newcrcs!![tileNumber]) { + oldcrcs!![tileNumber] = newcrcs!![tileNumber] + if (sendx == -1) { + sendx = j + sendy = i + sendw = 1 + } else { + sendw += 1 + } + } else if (sendx != -1) { + sendSubBitmapRow(bitmap, sendx, sendy, sendw, sink) + sendx = -1 + } + } + if (sendx != -1) { + sendSubBitmapRow(bitmap, sendx, sendy, sendw, sink) + sendx = -1 + } + } + if (sendx != -1) { + sendSubBitmapRow(bitmap, sendx, sendy, sendw, sink) + } + forceFullFrame = false + } + + private fun sendSubBitmapRow(bitmap: Bitmap, x: Int, y: Int, w: Int, sink: (ByteString) -> Unit) { + var h = y + 1 + var exit = false + while (h < tilesHigh) { + for (xx in x until (x + w)) { + val tileNumber = (h * tilesWide) + xx + if (oldcrcs!![tileNumber] == newcrcs!![tileNumber]) { + exit = true + break + } + } + if (!exit) { + for (xx in x until (x + w)) { + val tileNumber = (h * tilesWide) + xx + oldcrcs!![tileNumber] = newcrcs!![tileNumber] + } + } else { + break + } + h++ + } + h -= y + sink(buildImageCommand(bitmap, x * 64, y * 64, w * 64, h * 64)) + } + + private fun computeAllCRCs(bitmap: Bitmap) { + for (i in 0 until tilesCount) newcrcs!![i] = 1 + for (y in 0 until tilesHigh) { + var h = 64 + if (((y * 64) + 64) > bitmap.height) h = bitmap.height - (y * 64) + for (x in 0 until tilesWide) { + var w = 64 + if (((x * 64) + 64) > bitmap.width) w = bitmap.width - (x * 64) + val t = (y * tilesWide) + x + val pixels = IntArray(w * h) + bitmap.getPixels(pixels, 0, w, x * 64, y * 64, w, h) + for (pixel in pixels) newcrcs!![t] = adler32(pixel, newcrcs!![t]) + } + } + } + + private fun buildImageCommand(bitmap: Bitmap, x: Int, y: Int, w: Int, h: Int): ByteString { + var ww = w + var hh = h + if (x + w > bitmap.width) ww = bitmap.width - x + if (y + h > bitmap.height) hh = bitmap.height - y + val croppedBitmap = if (x == 0 && y == 0 && ww == bitmap.width && hh == bitmap.height) { + bitmap + } else { + Bitmap.createBitmap(bitmap, x, y, ww, hh) + } + + val bytesOut = ByteArrayOutputStream() + val dos = DataOutputStream(bytesOut) + dos.writeShort(27) + dos.writeShort(8) + dos.writeInt(0) + dos.writeShort(3) + dos.writeShort(0) + dos.writeShort(x) + dos.writeShort(y) + when (g_desktop_imageType) { + 4 -> { + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { + croppedBitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, g_desktop_compressionLevel, dos) + } else { + @Suppress("DEPRECATION") + croppedBitmap.compress(Bitmap.CompressFormat.WEBP, g_desktop_compressionLevel, dos) + } + } + 2 -> croppedBitmap.compress(Bitmap.CompressFormat.PNG, g_desktop_compressionLevel, dos) + else -> croppedBitmap.compress(Bitmap.CompressFormat.JPEG, g_desktop_compressionLevel, dos) + } + if (croppedBitmap !== bitmap) croppedBitmap.recycle() + + val data = bytesOut.toByteArray() + val cmdSize = data.size - 8 + data[4] = (cmdSize shr 24).toByte() + data[5] = (cmdSize shr 16).toByte() + data[6] = (cmdSize shr 8).toByte() + data[7] = cmdSize.toByte() + return data.toByteString() + } + + private fun adler32(n: Int, state: Int): Int { + var a = state shr 16 + var b = state and 0xFFFF + a = (a + n) % 65521 + b = (b + a) % 65521 + return (b shl 16) + a + } +} diff --git a/app/src/main/java/com/meshcentral/agent/MainActivity.kt b/app/src/main/java/com/meshcentral/agent/MainActivity.kt index aca1448..bac1c1e 100644 --- a/app/src/main/java/com/meshcentral/agent/MainActivity.kt +++ b/app/src/main/java/com/meshcentral/agent/MainActivity.kt @@ -8,21 +8,16 @@ import android.app.Notification import android.app.NotificationChannel import android.app.NotificationManager import android.app.PendingIntent -import android.content.BroadcastReceiver import android.content.Context import android.content.Intent -import android.content.IntentFilter -import android.content.SharedPreferences import android.content.pm.PackageManager import android.graphics.Color import android.media.projection.MediaProjectionManager import android.net.Uri import android.os.Build import android.os.Bundle -import android.os.CountDownTimer import android.provider.Settings import android.text.InputType -import android.util.Base64 import android.view.Gravity import android.view.Menu import android.view.MenuItem @@ -31,28 +26,12 @@ import android.widget.Toast import androidx.appcompat.app.AppCompatActivity import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat -import androidx.preference.PreferenceManager import com.google.firebase.messaging.FirebaseMessaging import org.json.JSONObject -import org.spongycastle.asn1.x500.X500Name -import org.spongycastle.cert.X509v3CertificateBuilder -import org.spongycastle.cert.jcajce.JcaX509CertificateConverter -import org.spongycastle.cert.jcajce.JcaX509v3CertificateBuilder import org.spongycastle.jce.provider.BouncyCastleProvider -import org.spongycastle.operator.jcajce.JcaContentSignerBuilder -import java.io.ByteArrayInputStream -import java.math.BigInteger -import java.security.KeyFactory -import java.security.KeyPairGenerator import java.security.PrivateKey -import java.security.SecureRandom import java.security.Security -import java.security.cert.CertificateFactory import java.security.cert.X509Certificate -import java.security.spec.PKCS8EncodedKeySpec -import java.util.Date -import java.util.Random -import kotlin.math.absoluteValue // You can hardcode a server connection string into this application by setting this string. @@ -81,11 +60,12 @@ var pendingActivities : ArrayList = ArrayList(R.id.toolbar) @@ -125,13 +99,6 @@ class MainActivity : AppCompatActivity() { // Setup notification manager notificationManager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - // Register to get battery events - val intentFilter = IntentFilter() - intentFilter.addAction(Intent.ACTION_POWER_CONNECTED) - intentFilter.addAction(Intent.ACTION_POWER_DISCONNECTED) - intentFilter.addAction(Intent.ACTION_BATTERY_CHANGED) - registerReceiver(batteryInfoReceiver, intentFilter) - // Check if this device has a camera cameraPresent = applicationContext.packageManager.hasSystemFeature(PackageManager.FEATURE_CAMERA) @@ -149,48 +116,47 @@ class MainActivity : AppCompatActivity() { } */ - // See if we there open by a notification with a URL - var intentUrl : String? = intent.getStringExtra("url") - //println("Main Activity Create URL: $intentUrl") - if (intentUrl != null) { - intent.removeExtra("url") - if (intentUrl.lowercase().startsWith("2fa://")) { - // if there is no server link, ignore this - if (serverLink != null) { - // This activity was created by a 2FA message - g_auth_url = Uri.parse(intentUrl) - // If not connected, connect to the server now. - if (meshAgent == null) { - toggleAgentConnection(false); - } else { - // Switch to 2FA auth screen - if (mainFragment != null) { - mainFragment?.moveToAuthPage() - } - } - - } - } else if (intentUrl.lowercase().startsWith("http://") || intentUrl.lowercase().startsWith("https://")) { - // Open an HTTP or HTTPS URL. - var getintent: Intent = Intent(Intent.ACTION_VIEW, Uri.parse(intentUrl)); - startActivity(getintent); - } - } + handleIntentUrl(intent) // Activate the settings settingsChanged() - if (g_autoConnect && !g_userDisconnect && (meshAgent == null)) { - toggleAgentConnection(false) + if (serverLink != null) { + requestAllPermissions() + AgentController.requestBatteryOptimizationExemption() } } - private fun sendConsoleMessage(msg: String) { - if (meshAgent != null) { meshAgent?.sendConsoleResponse(msg, null) } + override fun onResume() { + super.onResume() + if (serverLink != null) { + window.decorView.post { showUnattendedSetupPromptIfNeeded(false) } + } + invalidateOptionsMenu() } - private val batteryInfoReceiver: BroadcastReceiver = object : BroadcastReceiver() { - override fun onReceive(context: Context, intent: Intent) { - if (meshAgent != null) { meshAgent?.batteryStateChanged(intent) } + override fun onNewIntent(intent: Intent) { + super.onNewIntent(intent) + setIntent(intent) + handleIntentUrl(intent) + } + + private fun handleIntentUrl(intent: Intent?) { + val intentUrl: String = intent?.getStringExtra("url") ?: return + intent.removeExtra("url") + if (intentUrl.lowercase().startsWith("2fa://")) { + if (serverLink != null) { + g_auth_url = Uri.parse(intentUrl) + if (meshAgent == null) { + toggleAgentConnection(false) + } else { + if (mainFragment != null) { + mainFragment?.moveToAuthPage() + } + } + } + } else if (intentUrl.lowercase().startsWith("http://") || intentUrl.lowercase().startsWith("https://")) { + val getintent = Intent(Intent.ACTION_VIEW, Uri.parse(intentUrl)) + startActivity(getintent) } } @@ -209,15 +175,17 @@ class MainActivity : AppCompatActivity() { var item3 = menu.findItem(R.id.action_close); item3.isVisible = (visibleScreen != 1); var item4 = menu.findItem(R.id.action_sharescreen); - item4.isVisible = false // (g_ScreenCaptureService == null) && (meshAgent != null) && (meshAgent!!.state == 3) + item4.isVisible = false var item5 = menu.findItem(R.id.action_stopscreensharing); - item5.isVisible = (g_ScreenCaptureService != null) + item5.isVisible = AgentController.isRemoteDesktopRunning() var item6 = menu.findItem(R.id.action_manual_setup_server); item6.isVisible = (visibleScreen == 1) && (serverLink == null) && (hardCodedServerLink == null) var item7 = menu.findItem(R.id.action_testAuth); item7.isVisible = false //(visibleScreen == 1) && (serverLink != null); var item8 = menu.findItem(R.id.action_settings); item8.isVisible = (visibleScreen == 1); + var itemCheckSetup = menu.findItem(R.id.action_check_setup); + itemCheckSetup.isVisible = (visibleScreen == 1) && (serverLink != null); var item9 = menu.findItem(R.id.action_enablepushauthentication); if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.N) { item9.isVisible = (notificationManager.areNotificationsEnabled() == false) @@ -253,8 +221,11 @@ class MainActivity : AppCompatActivity() { } if (item.itemId == R.id.action_stopscreensharing) { - // Stop projection - stopProjection() + AgentController.stopScreenSharingByUser() + } + + if (item.itemId == R.id.action_check_setup) { + showUnattendedSetupPromptIfNeeded(true) } if ((item.itemId == R.id.action_manual_setup_server) && (hardCodedServerLink == null)) { @@ -274,8 +245,13 @@ class MainActivity : AppCompatActivity() { if (item.itemId == R.id.action_enablepushauthentication) { // Ask to Enable Push Notifications for Push Authentication - val intent = Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) - intent.putExtra(Settings.EXTRA_APP_PACKAGE, packageName) + val intent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) + .putExtra(Settings.EXTRA_APP_PACKAGE, packageName) + } else { + Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS) + .setData(Uri.parse("package:$packageName")) + } startActivity(intent) } @@ -286,7 +262,7 @@ class MainActivity : AppCompatActivity() { } override fun onDestroy() { - g_mainActivity = null + AgentController.detachActivity(this) if (alert != null) { alert?.dismiss() alert = null @@ -300,7 +276,7 @@ class MainActivity : AppCompatActivity() { if (requestCode == MainActivity.Companion.REQUEST_CODE) { if (resultCode == RESULT_OK) { - startService(com.meshcentral.agent.ScreenCaptureService.getStartIntent(this, resultCode, data)) + ContextCompat.startForegroundService(this, com.meshcentral.agent.ScreenCaptureService.getStartIntent(this, resultCode, data)) if (meshAgent?.tunnels?.getOrNull(0) != null) { val json = JSONObject() json.put("type", "console") @@ -310,14 +286,7 @@ class MainActivity : AppCompatActivity() { } return } else { - if (meshAgent?.tunnels?.getOrNull(0) != null) { - val json = JSONObject() - json.put("type", "console") - json.put("msg", "denied") - json.put("msgid", 2) - meshAgent!!.tunnels[0].sendCtrlResponse(json) - meshAgent!!.tunnels[0].Stop() - } + sendDesktopConsentDenied() return } } @@ -338,21 +307,15 @@ class MainActivity : AppCompatActivity() { } fun setMeshServerLink(x: String?) { - if ((serverLink == x) || (hardCodedServerLink != null)) return - if (meshAgent != null) { // Stop the agent - meshAgent?.Stop() - meshAgent = null + AgentController.setMeshServerLink(x) + if (x != null) { + requestAllPermissions() + window.decorView.post { showUnattendedSetupPromptIfNeeded(true) } } - serverLink = x - val sharedPreferences = getSharedPreferences("meshagent", Context.MODE_PRIVATE) - sharedPreferences.edit().putString("qrmsh", x).apply() - mainFragment?.refreshInfo() - g_userDisconnect = false - if (g_autoConnect) { toggleAgentConnection(false) } } // Open a URL in the web view fragment - fun openUrl(xpageUrl: String) : Boolean { + fun openUrlInApp(xpageUrl: String) : Boolean { if (visibleScreen == 2) return false pageUrl = xpageUrl; if (visibleScreen == 1) { @@ -379,23 +342,6 @@ class MainActivity : AppCompatActivity() { } } - fun agentStateChanged() { - this.runOnUiThread { - if ((meshAgent != null) && (meshAgent?.state == 0)) { - meshAgent = null - } - if (((meshAgent != null) && (meshAgent?.state == 2)) || (g_userDisconnect) || (!g_autoConnect)) stopRetryTimer() - else if ((meshAgent == null) && (!g_userDisconnect) && (g_autoConnect) && (g_retryTimer == null)) startRetryTimer() - mainFragment?.refreshInfo() - } - } - - fun refreshInfo() { - this.runOnUiThread { - mainFragment?.refreshInfo() - } - } - fun confirmServerClear() { if (hardCodedServerLink != null) return if (alert != null) { @@ -434,25 +380,6 @@ class MainActivity : AppCompatActivity() { } } - fun getServerHost() : String? { - if (serverLink == null) return null - var x : List = serverLink!!.split(',') - var serverHost = x[0] - return serverHost.substring(5) - } - - fun getServerHash() : String? { - if (serverLink == null) return null - var x : List = serverLink!!.split(',') - return x[1] - } - - fun getDevGroup() : String? { - if (serverLink == null) return null - var x : List = serverLink!!.split(',') - return x[2] - } - fun isAgentDisconnected() : Boolean { return (meshAgent == null) } @@ -493,96 +420,66 @@ class MainActivity : AppCompatActivity() { } } + private fun showUnattendedSetupPromptIfNeeded(force: Boolean) { + if (serverLink == null || isFinishing || isDestroyed) return + val missingItems = ArrayList() + if (!AgentController.isAccessibilityServiceEnabled()) { + missingItems.add(getString(R.string.missing_accessibility)) + } + if (!AgentController.isIgnoringBatteryOptimizations()) { + missingItems.add(getString(R.string.missing_battery)) + } + if (!AgentController.areNotificationsEnabled()) { + missingItems.add(getString(R.string.missing_notifications)) + } + if (missingItems.isEmpty()) { + if (force) showToastMessage(getString(R.string.unattended_setup_complete)) + return + } + + // A manual re-open always shows the prompt and clears any earlier dismissal; an automatic + // check stays hidden if the user already dismissed it this session. + if (force) { + unattendedPromptDismissed = false + } else if (unattendedPromptDismissed) { + return + } + + if (alert != null) { + alert?.dismiss() + alert = null + } + val missingText = missingItems.joinToString(separator = "\n") { "- $it" } + val builder = AlertDialog.Builder(this) + .setTitle(getString(R.string.unattended_setup_title)) + .setMessage(getString(R.string.unattended_setup_message, BuildConfig.VERSION_NAME, missingText)) + .setPositiveButton(R.string.open_accessibility_settings) { _, _ -> + startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)) + } + .setNeutralButton(R.string.open_app_settings) { _, _ -> + mainFragment?.moveToSettingsPage() + } + .setNegativeButton(R.string.later) { dialog, _ -> + unattendedPromptDismissed = true + dialog.dismiss() + } + alert = builder.show() + } + override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { super.onRequestPermissionsResult(requestCode, permissions, grantResults) if (requestCode == REQUEST_ALL_PERMISSIONS) { - permissions.forEachIndexed { index, permission -> + permissions.forEachIndexed { index, _ -> if (grantResults[index] == PackageManager.PERMISSION_DENIED) { - // Handle each denied permission if necessary } } } } fun toggleAgentConnection(userInitiated : Boolean) { - //println("toggleAgentConnection") - if ((meshAgent == null) && (serverLink != null)) { - // Create and connect the agent - requestAllPermissions(); - if (agentCertificate == null) { - val sharedPreferences = getSharedPreferences("meshagent", Context.MODE_PRIVATE) - var certb64 : String? = sharedPreferences?.getString("agentCert", null) - var keyb64 : String? = sharedPreferences?.getString("agentKey", null) - if ((certb64 == null) || (keyb64 == null)) { - //println("Generating new certificates...") - - // Generate an RSA key pair - val keyGen = KeyPairGenerator.getInstance("RSA") - keyGen.initialize(2048, SecureRandom()) - val keypair = keyGen.generateKeyPair() - - // Generate Serial Number - var serial : BigInteger = BigInteger("12345678"); - try { serial = BigInteger.valueOf(Random().nextInt().toLong().absoluteValue) } catch (ex: Exception) {} - - // Create self signed certificate - val builder: X509v3CertificateBuilder = JcaX509v3CertificateBuilder( - X500Name("CN=android.agent.meshcentral.com"), // issuer authority - serial, // serial number of certificate - Date(System.currentTimeMillis() - 86400000L * 365), // start of validity - Date(253402300799000L), // end of certificate validity - X500Name("CN=android.agent.meshcentral.com"), // subject name of certificate - keypair.public) // public key of certificate - agentCertificate = JcaX509CertificateConverter().setProvider("SC").getCertificate(builder - .build(JcaContentSignerBuilder("SHA256withRSA").build(keypair.private))) // Private key of signing authority , here it is self signed - agentCertificateKey = keypair.private - - // Save the certificate and key - sharedPreferences?.edit()?.putString("agentCert", Base64.encodeToString(agentCertificate?.encoded, Base64.DEFAULT))?.apply() - sharedPreferences?.edit()?.putString("agentKey", Base64.encodeToString(agentCertificateKey?.encoded, Base64.DEFAULT))?.apply() - } else { - //println("Loading certificates...") - agentCertificate = CertificateFactory.getInstance("X509").generateCertificate( - ByteArrayInputStream(Base64.decode(certb64, Base64.DEFAULT)) - ) as X509Certificate - val keySpec = PKCS8EncodedKeySpec(Base64.decode(keyb64, Base64.DEFAULT)) - agentCertificateKey = KeyFactory.getInstance("RSA").generatePrivate(keySpec) - } - //println("Cert: ${agentCertificate.toString()}") - //println("XKey: ${agentCertificateKey.toString()}") - } - - if (!userInitiated) { - meshAgent = MeshAgent(this, getServerHost()!!, getServerHash()!!, getDevGroup()!!) - meshAgent?.Start() - } else { - if (g_autoConnect) { - if (g_userDisconnect) { - // We are not trying to connect, switch to connecting - g_userDisconnect = false - meshAgent = - MeshAgent(this, getServerHost()!!, getServerHash()!!, getDevGroup()!!) - meshAgent?.Start() - } else { - // We are trying to connect, switch to not trying - g_userDisconnect = true - } - } else { - // We are not in auto connect mode, try to connect - g_userDisconnect = true - meshAgent = - MeshAgent(this, getServerHost()!!, getServerHash()!!, getDevGroup()!!) - meshAgent?.Start() - } - } - } else if (meshAgent != null) { - // Stop the agent - if (userInitiated) { g_userDisconnect = true } - stopProjection() - meshAgent?.Stop() - meshAgent = null - } - mainFragment?.refreshInfo() + requestAllPermissions() + AgentForegroundService.start(this) + AgentController.toggleAgentConnection(userInitiated) } fun showNotification(title: String?, body: String?, url: String?) { @@ -612,10 +509,20 @@ class MainActivity : AppCompatActivity() { .setAutoCancel(true) //.setLargeIcon(BitmapFactory.decodeResource(resources, R.mipmap.ic_launcher)) .setContentIntent(pendingIntent) + } else { + builder = Notification.Builder(this) + .setSmallIcon(R.drawable.ic_message) + .setContentTitle(title) + .setContentText(body) + .setAutoCancel(true) + .setContentIntent(pendingIntent) } // Add notification - notificationManager.notify(0, builder.build()) + try { + notificationManager.notify(0, builder.build()) + } catch (_: SecurityException) { + } } fun isMshStringValid(x: String):Boolean { @@ -664,78 +571,59 @@ class MainActivity : AppCompatActivity() { builder.show() } - // Start screen sharing fun startProjection() { - if ((g_ScreenCaptureService != null) || (meshAgent == null) || (meshAgent!!.state != 3)) return - val mProjectionManager = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager - startActivityForResult(mProjectionManager.createScreenCaptureIntent(), MainActivity.Companion.REQUEST_CODE) + AgentController.startProjection() } - // Stop screen sharing - fun stopProjection() { - if (g_ScreenCaptureService == null) return - startService(com.meshcentral.agent.ScreenCaptureService.getStopIntent(this)) + fun startMediaProjectionPrompt() { + if (AgentController.isRemoteDesktopRunning() || (meshAgent == null) || (meshAgent!!.state != 3)) return + val mProjectionManager = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + startActivityForResult(mProjectionManager.createScreenCaptureIntent(), MainActivity.Companion.REQUEST_CODE) } - fun settingsChanged() { - this.runOnUiThread { - val pm: SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) - g_autoConnect = pm.getBoolean("pref_autoconnect", false) - g_autoConsent = pm.getBoolean("pref_autoconsent", false) - g_userDisconnect = false - if (g_autoConnect == false) { - if (g_retryTimer != null) { - stopRetryTimer() - mainFragment?.refreshInfo() - } - } else { - if ((meshAgent == null) && (!g_userDisconnect) && (g_retryTimer == null)) { - toggleAgentConnection(false) - } + fun promptScreenShareChoice() { + if (AgentController.isRemoteDesktopRunning() || (meshAgent == null) || (meshAgent!!.state != 3)) return + if (isFinishing || isDestroyed) return + if (alert != null) { + alert?.dismiss() + alert = null + } + alert = AlertDialog.Builder(this) + .setTitle(R.string.share_screen_choice_title) + .setMessage(R.string.share_screen_choice_message) + .setPositiveButton(R.string.open_accessibility_settings) { _, _ -> + startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)) } - if (g_autoConsent) { - startProjection() - } else if (!g_autoConsent && g_ScreenCaptureService != null) { - stopProjection() + .setNeutralButton(R.string.share_screen_once) { _, _ -> + startMediaProjectionPrompt() } - } + .setNegativeButton(android.R.string.cancel) { dialog, _ -> + sendDesktopConsentDenied() + dialog.dismiss() + } + .show() } - // Start the connection retry timer, try to connect the agent every 10 seconds - private fun startRetryTimer() { - this.runOnUiThread { - if (g_retryTimer == null) { - g_retryTimer = object : CountDownTimer(120000000, 10000) { - override fun onTick(millisUntilFinished: Long) { - println("onTick!!!") - if ((meshAgent == null) && (!g_userDisconnect)) { - toggleAgentConnection(false) - } - } + private fun sendDesktopConsentDenied() { + val tunnel = meshAgent?.tunnels?.getOrNull(0) ?: return + val json = JSONObject() + json.put("type", "console") + json.put("msg", "denied") + json.put("msgid", 2) + tunnel.sendCtrlResponse(json) + tunnel.Stop() + } - override fun onFinish() { - println("onFinish!!!") - stopRetryTimer() - startRetryTimer() - } - } - g_retryTimer?.start() - } - } + fun stopProjection() { + AgentController.stopProjection() } - // Stop the connection retry timer - private fun stopRetryTimer() { - this.runOnUiThread { - if (g_retryTimer != null) { - g_retryTimer?.cancel() - g_retryTimer = null - } - } + fun settingsChanged() { + AgentController.settingsChanged() } companion object { private const val REQUEST_CODE = 100 const val REQUEST_ALL_PERMISSIONS = 1 } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/meshcentral/agent/MainFragment.kt b/app/src/main/java/com/meshcentral/agent/MainFragment.kt index d0c7ee5..9d7a176 100644 --- a/app/src/main/java/com/meshcentral/agent/MainFragment.kt +++ b/app/src/main/java/com/meshcentral/agent/MainFragment.kt @@ -135,7 +135,7 @@ class MainFragment : Fragment(), MultiplePermissionsListener { } view?.findViewById(R.id.agentActionButton)?.isEnabled = true if (state == 0) { - if (g_retryTimer != null) { + if (AgentController.isRetrying()) { // Trying to connect view?.findViewById(R.id.mainImageView)?.alpha = 0.5F view?.findViewById(R.id.agentStatusTextview)?.text = @@ -355,4 +355,4 @@ class MainFragment : Fragment(), MultiplePermissionsListener { } super.onDestroy() } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt new file mode 100644 index 0000000..67ec238 --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -0,0 +1,332 @@ +package com.meshcentral.agent + +import android.accessibilityservice.AccessibilityService +import android.accessibilityservice.GestureDescription +import android.graphics.Bitmap +import android.graphics.Path +import android.os.Build +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.view.Display +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import okio.ByteString +import kotlin.math.absoluteValue +import kotlin.math.max + +class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { + private val mainHandler = Handler(Looper.getMainLooper()) + private val encoder = DesktopFrameEncoder() + private val captureRunnable = Runnable { captureFrame() } + private var active = false + private var capturing = false + private var lastWidth = 0 + private var lastHeight = 0 + private var pointerDownX: Int? = null + private var pointerDownY: Int? = null + private var unsupportedKeyboardNotified = false + + override val isRunning: Boolean + get() = active + + override val width: Int + get() = if (lastWidth > 0) lastWidth else resources.displayMetrics.widthPixels + + override val height: Int + get() = if (lastHeight > 0) lastHeight else resources.displayMetrics.heightPixels + + override fun onServiceConnected() { + super.onServiceConnected() + AgentController.init(applicationContext) + instance = this + AgentController.refreshInfo() + if (AgentController.hasActiveDesktopTunnel()) { + AgentController.startProjection() + } + } + + override fun onDestroy() { + if (instance === this) instance = null + stopDesktop() + super.onDestroy() + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + } + + override fun onInterrupt() { + } + + fun startDesktop(): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + AgentController.sendDesktopMessage("Unattended screenshots require Android 11 or later.") + return false + } + if (active) return true + active = true + g_remoteDesktopProvider = this + unsupportedKeyboardNotified = false + encoder.requestFullFrame() + updateTunnelDisplaySize() + captureFrame() + meshAgent?.sendConsoleResponse("Started unattended display sharing", null) + return true + } + + fun stopDesktop() { + val wasActive = active + active = false + mainHandler.removeCallbacks(captureRunnable) + if (g_remoteDesktopProvider === this) { + g_remoteDesktopProvider = null + } + if (wasActive) { + meshAgent?.sendConsoleResponse("Stopped unattended display sharing", null) + } + } + + override fun requestFullFrame() { + encoder.requestFullFrame() + } + + override fun handleMouseCommand(msg: ByteString): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || msg.size < 10) return false + val flags = u(msg[5]) + var x = readShort(msg, 6) + var y = readShort(msg, 8) + if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { + x = (x * 1024) / g_desktop_scalingLevel + y = (y * 1024) / g_desktop_scalingLevel + } + + if (msg.size >= 12) { + val delta = readSignedShort(msg, 10) + if (delta != 0) { + val distance = if (delta > 0) -350 else 350 + dispatchSwipe(x, y, x, y + distance, 250) + return true + } + } + + return when { + flags == 0x88 -> { + dispatchTap(x, y) + mainHandler.postDelayed({ dispatchTap(x, y) }, 120) + true + } + flags == 0x02 || flags == 0x08 || flags == 0x20 -> { + pointerDownX = x + pointerDownY = y + true + } + flags == 0x04 || flags == 0x10 || flags == 0x40 -> { + val startX = pointerDownX ?: x + val startY = pointerDownY ?: y + pointerDownX = null + pointerDownY = null + if ((startX - x).absoluteValue < 8 && (startY - y).absoluteValue < 8) { + dispatchTap(x, y) + } else { + dispatchSwipe(startX, startY, x, y, 350) + } + true + } + else -> true + } + } + + override fun handleTouchCommand(msg: ByteString): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || msg.size < 14 || u(msg[4]) != 1) return false + val flags = readInt(msg, 6) + var x = readShort(msg, 10) + var y = readShort(msg, 12) + if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { + x = (x * 1024) / g_desktop_scalingLevel + y = (y * 1024) / g_desktop_scalingLevel + } + return when { + (flags and 0x00010000) != 0 -> { + pointerDownX = x + pointerDownY = y + true + } + (flags and 0x00040000) != 0 -> { + val startX = pointerDownX ?: x + val startY = pointerDownY ?: y + pointerDownX = null + pointerDownY = null + if ((startX - x).absoluteValue < 8 && (startY - y).absoluteValue < 8) { + dispatchTap(x, y) + } else { + dispatchSwipe(startX, startY, x, y, 350) + } + true + } + else -> true + } + } + + override fun handleKeyCommand(cmd: Int, msg: ByteString): Boolean { + return when (cmd) { + 1 -> handleLegacyKey(msg) + 85 -> handleUnicodeKey(msg) + else -> false + } + } + + private fun captureFrame() { + if (!active || capturing || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return + capturing = true + takeScreenshot(Display.DEFAULT_DISPLAY, mainExecutor, object : AccessibilityService.TakeScreenshotCallback { + override fun onSuccess(screenshot: AccessibilityService.ScreenshotResult) { + try { + val wrapped = Bitmap.wrapHardwareBuffer(screenshot.hardwareBuffer, screenshot.colorSpace) + if (wrapped != null) { + val bitmap = wrapped.copy(Bitmap.Config.ARGB_8888, false) + val dimensionsChanged = lastWidth != bitmap.width || lastHeight != bitmap.height + lastWidth = bitmap.width + lastHeight = bitmap.height + if (dimensionsChanged) updateTunnelDisplaySize() + val encodedBitmap = if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { + Bitmap.createScaledBitmap( + bitmap, + max(1, (bitmap.width * g_desktop_scalingLevel) / 1024), + max(1, (bitmap.height * g_desktop_scalingLevel) / 1024), + false + ) + } else { + bitmap + } + encoder.encode(encodedBitmap) { AgentController.sendDesktopTunnelData(it) } + if (encodedBitmap !== bitmap) encodedBitmap.recycle() + bitmap.recycle() + } + } catch (ex: Exception) { + AgentController.sendDesktopMessage("Unable to capture unattended screenshot: ${ex.message}") + } finally { + screenshot.hardwareBuffer.close() + capturing = false + scheduleNextCapture() + } + } + + override fun onFailure(errorCode: Int) { + capturing = false + AgentController.sendDesktopMessage("Unable to capture unattended screenshot, error $errorCode.") + scheduleNextCapture() + } + }) + } + + private fun scheduleNextCapture() { + if (!active) return + mainHandler.postDelayed(captureRunnable, max(100L, g_desktop_frameRateLimiter.toLong())) + } + + private fun handleLegacyKey(msg: ByteString): Boolean { + if (msg.size < 6) return false + val action = u(msg[4]) + val keyCode = u(msg[5]) + if (action != 0) return true + when (keyCode) { + 8 -> return editFocusedText { if (it.isNotEmpty()) it.dropLast(1) else it } + 13 -> return editFocusedText { "$it\n" } + 27 -> { + performGlobalAction(GLOBAL_ACTION_BACK) + return true + } + 36 -> { + performGlobalAction(GLOBAL_ACTION_HOME) + return true + } + 93 -> { + performGlobalAction(GLOBAL_ACTION_RECENTS) + return true + } + } + notifyUnsupportedKeyboard() + return false + } + + private fun handleUnicodeKey(msg: ByteString): Boolean { + if (msg.size < 7) return false + val action = u(msg[4]) + if (action != 0) return true + val charCode = readShort(msg, 5) + val char = charCode.toChar().toString() + return editFocusedText { it + char }.also { + if (!it) notifyUnsupportedKeyboard() + } + } + + private fun editFocusedText(transform: (String) -> String): Boolean { + val node = rootInActiveWindow?.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: return false + if (!node.isEditable) return false + val args = Bundle() + args.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + transform(node.text?.toString() ?: "") + ) + return node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) + } + + private fun notifyUnsupportedKeyboard() { + if (unsupportedKeyboardNotified) return + unsupportedKeyboardNotified = true + AgentController.sendDesktopMessage("Android unattended keyboard input is limited to focused editable text and basic navigation keys.") + } + + private fun dispatchTap(x: Int, y: Int) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return + val path = Path() + path.moveTo(x.toFloat(), y.toFloat()) + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, 80)) + .build() + dispatchGesture(gesture, null, null) + } + + private fun dispatchSwipe(startX: Int, startY: Int, endX: Int, endY: Int, duration: Long) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return + val path = Path() + path.moveTo(startX.toFloat(), startY.toFloat()) + path.lineTo(endX.toFloat(), endY.toFloat()) + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, duration)) + .build() + dispatchGesture(gesture, null, null) + } + + private fun updateTunnelDisplaySize() { + val agent = meshAgent ?: return + for (t in agent.tunnels) { + if ((t.state == 2) && (t.usage == 2)) { + t.updateDesktopDisplaySize() + } + } + } + + private fun readShort(msg: ByteString, offset: Int): Int { + return (u(msg[offset]) shl 8) + u(msg[offset + 1]) + } + + private fun readSignedShort(msg: ByteString, offset: Int): Int { + val value = readShort(msg, offset) + return if ((value and 0x8000) != 0) value - 0x10000 else value + } + + private fun readInt(msg: ByteString, offset: Int): Int { + return (u(msg[offset]) shl 24) + + (u(msg[offset + 1]) shl 16) + + (u(msg[offset + 2]) shl 8) + + u(msg[offset + 3]) + } + + private fun u(byte: Byte): Int = byte.toInt() and 0xFF + + companion object { + var instance: MeshAccessibilityService? = null + private set + } +} diff --git a/app/src/main/java/com/meshcentral/agent/MeshAgent.kt b/app/src/main/java/com/meshcentral/agent/MeshAgent.kt index 33158be..b9057cb 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAgent.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAgent.kt @@ -43,8 +43,8 @@ class MeshUserInfo(userid: String, realname: String?, image: Bitmap?) { } } -class MeshAgent(parent: MainActivity, host: String, certHash: String, devGroupId: String) : WebSocketListener() { - val parent : MainActivity = parent +class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: String) : WebSocketListener() { + val parent : AgentHost = parent val host : String = host val serverCertHash: String = certHash val devGroupId: String = devGroupId @@ -318,7 +318,7 @@ class MeshAgent(parent: MainActivity, host: String, certHash: String, devGroupId // Cause some data to be sent over the websocket control channel every 2 minutes to keep it open private fun startConnectionTimer() { - parent.runOnUiThread { + parent.runOnHostThread { connectionTimer = object: CountDownTimer(120000000, 120000) { override fun onTick(millisUntilFinished: Long) { if (sendNetworkUpdate(false) == false) { // See if we need to update network information @@ -673,7 +673,7 @@ class MeshAgent(parent: MainActivity, host: String, certHash: String, devGroupId private fun getSysBatteryInfo() : JSONObject? { try { val batteryStatus: Intent? = IntentFilter(Intent.ACTION_BATTERY_CHANGED).let { ifilter -> - parent.applicationContext.registerReceiver(null, ifilter) + parent.getApplicationContext().registerReceiver(null, ifilter) } val status: Int = batteryStatus?.getIntExtra(BatteryManager.EXTRA_STATUS, -1) ?: -1 val isCharging: Boolean = status == BatteryManager.BATTERY_STATUS_CHARGING @@ -884,7 +884,7 @@ class MeshAgent(parent: MainActivity, host: String, certHash: String, devGroupId } "kvmstart" -> { // Start remote desktop - if (g_ScreenCaptureService == null) { + if (!AgentController.isRemoteDesktopRunning()) { parent.startProjection() r = "ok" } else { @@ -893,7 +893,7 @@ class MeshAgent(parent: MainActivity, host: String, certHash: String, devGroupId } "kvmstop" -> { // Stop remote desktop - if (g_ScreenCaptureService != null) { + if (AgentController.isRemoteDesktopRunning()) { parent.stopProjection() r = "ok" } else { diff --git a/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt b/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt index 89957a8..9702f0b 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt @@ -31,6 +31,7 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { } override fun onMessageReceived(remoteMessage: RemoteMessage) { + AgentController.init(applicationContext) println("onMessageReceived-from: ${remoteMessage.from}") println("onMessageReceived-data: ${remoteMessage.data}") println("serverLink: $serverLink") @@ -58,12 +59,14 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { if ((url != null) && (url.startsWith("2fa://"))) { // Move to user authentication + g_auth_url = Uri.parse(url) + AgentForegroundService.start(this) + if (meshAgent == null) { + AgentController.toggleAgentConnection(false) + } if (g_mainActivity != null) { g_mainActivity?.runOnUiThread { - g_auth_url = Uri.parse(url) - if (meshAgent == null) { - g_mainActivity?.toggleAgentConnection(false); - } else { + if (meshAgent != null) { // Switch to 2FA auth screen if (mainFragment != null) { mainFragment?.moveToAuthPage() @@ -75,6 +78,8 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { if (g_mainActivity != null) { println("Showing notification with URL: $url"); g_mainActivity?.showNotification(remoteMessage.notification?.title, remoteMessage.notification?.body, url) + } else { + AgentController.showRuntimeNotification(remoteMessage.notification?.title, remoteMessage.notification?.body, url) } } else if (remoteMessage.data != null) { var cmd : String? = remoteMessage.data["con"] @@ -138,13 +143,11 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { // Vibrate the device if (splitCmd.size < 2) { r = "Usage:\r\n vibrate [milliseconds]"; - } else if (g_mainActivity == null) { - r = "No main activity"; } else if (splitCmd.size >= 2) { var t : Long = 0 try { t = splitCmd[1].toLong() } catch (e : Exception) {} if ((t > 0) && (t <= 10000)) { - val v = g_mainActivity!!.getApplicationContext() + val v = applicationContext .getSystemService(Context.VIBRATOR_SERVICE) as Vibrator if (v == null) { r = "Not supported" @@ -170,10 +173,8 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { "flash" -> { if (splitCmd.size < 2) { r = "Usage:\r\n flash [milliseconds]"; - } else if (g_mainActivity == null) { - r = "No main activity"; } else if (splitCmd.size >= 2) { - var isFlashAvailable = g_mainActivity!!.getApplicationContext().getPackageManager() + var isFlashAvailable = applicationContext.getPackageManager() .hasSystemFeature(PackageManager.FEATURE_CAMERA_FRONT); if (!isFlashAvailable) { r = "Flash not available" @@ -181,7 +182,7 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { var t : Long = 0 try { t = splitCmd[1].toLong() } catch (e : Exception) {} if ((t > 0) && (t <= 10000)) { - var mCameraManager = g_mainActivity!!.getApplicationContext().getSystemService(Context.CAMERA_SERVICE) as CameraManager + var mCameraManager = applicationContext.getSystemService(Context.CAMERA_SERVICE) as CameraManager try { var mCameraId = mCameraManager.getCameraIdList()[0]; mCameraManager.setTorchMode(mCameraId, true); @@ -266,4 +267,4 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { fun ByteArray.toHex(): String { return joinToString("") { "%02x".format(it) } } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt index 793494e..c6d02bb 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt @@ -156,8 +156,8 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo parent.removeTunnel(this) // Notify the parent that this tunnel is done // Check if there are no more remote desktop tunnels - if ((usage == 2) && (g_ScreenCaptureService != null)) { - g_ScreenCaptureService!!.checkNoMoreDesktopTunnels() + if (usage == 2) { + AgentController.checkNoMoreDesktopTunnels() } } @@ -205,24 +205,30 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo usage = xusage; // 2 = Desktop, 5 = Files, 10 = File transfer state = 2 + AgentController.refreshInfo() + // Start the connection time except if this is a file transfer if (usage != 10) { //println("Connected usage $usage") startConnectionTimer() if (usage == 2) { // If this is a remote desktop usage... - if (!g_autoConsent && g_ScreenCaptureService == null) { + if (!g_autoConsent && !AgentController.isRemoteDesktopRunning()) { // asking for consent if (meshAgent?.tunnels?.getOrNull(0) != null) { val json = JSONObject() + val msg = if (!AgentController.isAccessibilityServiceEnabled() && g_mainActivity == null) { + "Open the Android app to approve screen capture, or enable Accessibility Remote Control for unattended desktop." + } else { + "Waiting for user to grant access..." + } json.put("type", "console") - json.put("msg", "Waiting for user to grant access...") + json.put("msg", msg) json.put("msgid", 1) meshAgent!!.tunnels[0].sendCtrlResponse(json) } } - if (g_ScreenCaptureService == null) { - // Request media projection + if (!AgentController.isRemoteDesktopRunning()) { parent.parent.startProjection() } else { if (meshAgent?.tunnels?.getOrNull(0) != null) { @@ -232,8 +238,10 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo json.put("msgid", 0) meshAgent!!.tunnels[0].sendCtrlResponse(json) } - // Send the display size + // Send the display size and push a full frame of the current screen so the + // reconnecting viewer sees it immediately instead of waiting for a change. updateDesktopDisplaySize() + AgentController.requestDesktopRefresh() } } } else { @@ -310,10 +318,10 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo private fun processBinaryDesktopCmd(cmd : Int, cmdsize: Int, msg: ByteString) { when (cmd) { 1 -> { // Legacy key input - // Nop + AgentController.handleDesktopKeyCommand(cmd, msg) } 2 -> { // Mouse input - // Nop + AgentController.handleDesktopMouseCommand(msg) } 5 -> { // Remote Desktop Settings if (cmdsize < 6) return @@ -325,14 +333,17 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo updateDesktopDisplaySize() } 6 -> { // Refresh - // Nop + AgentController.requestDesktopRefresh() println("Desktop Refresh") } 8 -> { // Pause // Nop } 85 -> { // Unicode key input - // Nop + AgentController.handleDesktopKeyCommand(cmd, msg) + } + 15 -> { // Touch input + AgentController.handleDesktopTouchCommand(msg) } 87 -> { // Input Lock // Nop @@ -344,12 +355,12 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo } fun updateDesktopDisplaySize() { - if ((g_ScreenCaptureService == null) || (_webSocket == null)) return - //println("updateDesktopDisplaySize: ${g_ScreenCaptureService!!.mWidth} x ${g_ScreenCaptureService!!.mHeight}") + val provider = AgentController.activeRemoteDesktopProvider() + if ((provider == null) || (_webSocket == null)) return // Get the display size - var mWidth : Int = g_ScreenCaptureService!!.mWidth - var mHeight : Int = g_ScreenCaptureService!!.mHeight + var mWidth : Int = provider.width + var mHeight : Int = provider.height // Scale the display if needed if (g_desktop_scalingLevel != 1024) { @@ -372,7 +383,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo // Cause some data to be sent over the websocket control channel every 2 minutes to keep it open private fun startConnectionTimer() { - parent.parent.runOnUiThread { + parent.parent.runOnHostThread { connectionTimer = object: CountDownTimer(120000000, 120000) { override fun onTick(millisUntilFinished: Long) { if (_webSocket != null) { @@ -453,7 +464,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo } } else { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - val resolver: ContentResolver = parent.parent.getContentResolver() + val resolver: ContentResolver = parent.parent.contentResolver val contentValues = ContentValues() var fileUri: Uri? = null contentValues.put(MediaStore.MediaColumns.DISPLAY_NAME, name) @@ -561,7 +572,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo r.put(f) } } else { - val cursor: Cursor? = parent.parent.getContentResolver().query( + val cursor: Cursor? = parent.parent.contentResolver.query( uri, projection, null, @@ -625,7 +636,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo fileDeleteResponse(req, false) // Send failure } } else { - val cursor: Cursor? = parent.parent.getContentResolver().query( + val cursor: Cursor? = parent.parent.contentResolver.query( uri, projection, null, @@ -650,7 +661,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo } for (i in 0 until filenames.length()) { try { - parent.parent.contentResolver.delete(fileUriArray[i],null,null) + parent.parent.contentResolver.delete(fileUriArray[i],null,null) fileDeleteResponse(req, true) // Send success } catch (securityException: SecurityException) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { @@ -665,7 +676,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo // Launch the activity val intentSender = recoverableSecurityException.userAction.actionIntent.intentSender - parent.parent.startIntentSenderForResult( + val launched = parent.parent.launchIntentSenderForResult( intentSender, activityCode, null, @@ -674,6 +685,10 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo 0, null ) + if (!launched) { + pendingActivities.remove(pad) + fileDeleteResponse(req, false) + } } else { fileDeleteResponse(req, false) // Send fail } @@ -721,7 +736,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo val contentUrl = Uri.fromFile(file) try { // Serve the file - parent.parent.getContentResolver().openInputStream(contentUrl).use { stream -> + parent.parent.contentResolver.openInputStream(contentUrl).use { stream -> // Perform operation on stream var buf = ByteArray(65535) var len : Int @@ -741,7 +756,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo // file does not exist } } else { - val cursor: Cursor? = parent.parent.getContentResolver().query( + val cursor: Cursor? = parent.parent.contentResolver.query( uri, projection, null, @@ -765,7 +780,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo parent.logServerEventEx(106, eventArgs, "Download: ${filename}, Size: $fileSize", serverData); // Serve the file - parent.parent.getContentResolver().openInputStream(contentUrl).use { stream -> + parent.parent.contentResolver.openInputStream(contentUrl).use { stream -> // Perform operation on stream var buf = ByteArray(65535) var len : Int @@ -845,4 +860,4 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo } */ -} \ No newline at end of file +} diff --git a/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt b/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt index e27cc29..05e558c 100644 --- a/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt +++ b/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt @@ -26,11 +26,10 @@ import android.view.OrientationEventListener import android.view.WindowManager import androidx.core.util.Pair import okio.ByteString -import okio.ByteString.Companion.toByteString -import java.io.* +import kotlin.math.max -class ScreenCaptureService : Service() { +class ScreenCaptureService : Service(), RemoteDesktopProvider { private var mMediaProjection: MediaProjection? = null private var mImageReader: ImageReader? = null private var mHandler: Handler? = null @@ -41,247 +40,69 @@ class ScreenCaptureService : Service() { private var mOrientationChangeCallback: ScreenCaptureService.OrientationChangeCallback? = null var mWidth = 0 var mHeight = 0 + private var forceFullFrame = false + private val encoder = DesktopFrameEncoder() + private var lastFrameBitmap: Bitmap? = null - // Tile data - private var tilesWide : Int = 0 - private var tilesHigh : Int = 0 - private var tilesFullWide : Int = 0 - private var tilesFullHigh : Int = 0 - private var tilesRemainingWidth : Int = 0 - private var tilesRemainingHeight : Int = 0 - private var tilesCount : Int = 0 - private var oldcrcs : IntArray? = null - private var newcrcs : IntArray? = null + override val isRunning: Boolean + get() = mMediaProjection != null + + override val width: Int + get() = mWidth + + override val height: Int + get() = mHeight private inner class ImageAvailableListener : OnImageAvailableListener { override fun onImageAvailable(reader: ImageReader) { - if ((meshAgent == null) && (g_mainActivity != null)) { - g_mainActivity!!.stopProjection() + if (meshAgent == null) { + AgentController.stopProjection() return } - - var bitmap: Bitmap? = null + val imageReader = mImageReader ?: return var image: android.media.Image? = null - if (mImageReader == null) return - try { - image = mImageReader!!.acquireLatestImage() - // Skip this image if null or websocket push-back is high - if ((image != null) && (checkDesktopTunnelPushback() < 65535) && (meshAgent?.tunnels?.getOrNull(0) != null)) { - val planes: Array = image.getPlanes() - val buffer = planes[0].buffer - val pixelStride = planes[0].pixelStride - val rowStride = planes[0].rowStride - val rowPadding = rowStride - pixelStride * mWidth - - // Create the bitmap - bitmap = Bitmap.createBitmap(mWidth + rowPadding / pixelStride, mHeight, Bitmap.Config.ARGB_8888) - bitmap!!.copyPixelsFromBuffer(buffer) - - // Resize the bitmap if needed - if (g_desktop_scalingLevel != 1024) { - val newWidth = (mWidth * g_desktop_scalingLevel) / 1024 - val newHeight = (mHeight * g_desktop_scalingLevel) / 1024 - bitmap = getResizedBitmap(bitmap, newWidth, newHeight) - } - - // Setup or update the CRC buffer and tile information. - val wt = (bitmap!!.width / 64) - val ht = (bitmap.height / 64) - if ((tilesFullWide != wt) || (tilesFullHigh != ht)) { - tilesWide = wt; - tilesHigh = ht; - tilesFullWide = tilesWide - tilesFullHigh = tilesHigh - tilesRemainingWidth = (bitmap.width % 64); - tilesRemainingHeight = (bitmap.height % 64); - if (tilesRemainingWidth != 0) { tilesWide++; } - if (tilesRemainingHeight != 0) { tilesHigh++; } - tilesCount = (tilesWide * tilesHigh); - oldcrcs = IntArray(tilesCount); // 64 x 64 tiles - newcrcs = IntArray(tilesCount); // 64 x 64 tiles - //println("New tile count: $tilesCount") - } - - // Compute all tile CRC's - computeAllCRCs(bitmap); - - // Compute how many tiles have changed - var changedTiles : Int = 0; - for (i in 0 until tilesCount) { if (oldcrcs!![i] != newcrcs!![i]) { changedTiles++; } } - if (changedTiles > 0) { - // If 85% of the all tiles have changed, send the entire screen - if ((changedTiles * 100) >= (tilesCount * 85)) - { - sendEntireImage(bitmap) - for (i in 0 until tilesCount) { oldcrcs!![i] = newcrcs!![i]; } - } - else - { - // Send all changed tiles - // This version has horizontal & vertical optimization, JPEG as wide as possible then as high as possible - var sendx : Int = -1; - var sendy : Int = 0; - var sendw : Int = 0; - for (i in 0 until tilesHigh) - { - for (j in 0 until tilesWide) - { - val tileNumber : Int = (i * tilesWide) + j; - if (oldcrcs!![tileNumber] != newcrcs!![tileNumber]) - { - oldcrcs!![tileNumber] = newcrcs!![tileNumber]; - if (sendx == -1) { sendx = j; sendy = i; sendw = 1; } else { sendw += 1; } - } - else - { - if (sendx != -1) { sendSubBitmapRow(bitmap, sendx, sendy, sendw); sendx = -1; } - } - } - if (sendx != -1) { sendSubBitmapRow(bitmap, sendx, sendy, sendw); sendx = -1; } - } - if (sendx != -1) { sendSubBitmapRow(bitmap, sendx, sendy, sendw); sendx = -1; } - } - } - } + image = imageReader.acquireLatestImage() + if (image != null) processImage(image) } catch (e: Exception) { e.printStackTrace() + } finally { + image?.close() } - if (bitmap != null) { bitmap.recycle() } - if (image != null) { image.close() } } } - private fun sendSubBitmapRow(bm: Bitmap, x : Int, y : Int, w : Int) { - var h : Int = (y + 1) - var exit : Boolean = false - while (h < tilesHigh) { - // Check if the row is all different - for (xx in x until (x + w)) { - val tileNumber = (h * tilesWide) + xx; - if (oldcrcs!![tileNumber] == newcrcs!![tileNumber]) { exit = true; break; } - } - // If all different set the CRC's to the same, otherwise exit. - if (!exit) { - for (xx in x until (x + w)) { - val tileNumber : Int = (h * tilesWide) + xx; - oldcrcs!![tileNumber] = newcrcs!![tileNumber]; - } - } else break; - h++ - } - h -= y - sendSubImage(bm, x * 64, y * 64, w * 64, h * 64); - } + private fun processImage(image: android.media.Image) { + if ((checkDesktopTunnelPushback() >= 65535) || (meshAgent?.tunnels?.getOrNull(0) == null)) return - private fun Adler32(n : Int, state: Int) : Int { - var a = state shr 16; - var b = state and 0xFFFF; - a = (a + n) % 65521 - b = (b + a) % 65521 - return (b shl 16) + a - } + val planes: Array = image.getPlanes() + val buffer = planes[0].buffer + val pixelStride = planes[0].pixelStride + val rowStride = planes[0].rowStride + val rowPadding = rowStride - pixelStride * mWidth - // Compute all CRC's - private fun computeAllCRCs(bm: Bitmap) { - // Clear all CRC's - for (i in 0 until tilesCount) { newcrcs!![i] = 1 } - - // Compute all of the CRC's - for (y in 0 until tilesHigh) { - var h : Int = 64; - if (((y * 64) + 64) > bm.height) { h = (bm.height - (y * 64)) } - for (x in 0 until tilesWide) { - var w : Int = 64; - if (((x * 64) + 64) > bm.width) { w = (bm.width - (x * 64)) } - val t = (y * tilesWide) + x - val pixels = IntArray(w * h) - bm.getPixels(pixels, 0, w, x * 64, y * 64, w, h) - for (i in 0 until pixels.size) { newcrcs!![t] = Adler32(pixels[i], newcrcs!![t]) } - } + var bitmap = Bitmap.createBitmap(mWidth + rowPadding / pixelStride, mHeight, Bitmap.Config.ARGB_8888) + bitmap.copyPixelsFromBuffer(buffer) + + if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { + val newWidth = max(1, (mWidth * g_desktop_scalingLevel) / 1024) + val newHeight = max(1, (mHeight * g_desktop_scalingLevel) / 1024) + bitmap = getResizedBitmap(bitmap, newWidth, newHeight) ?: bitmap } - } - // Send a sub bitmap - private fun sendSubImage(bm: Bitmap, x: Int, y: Int, w: Int, h :Int) { - var ww = w; - var hh = h; - if (x + w > bm.width) { ww = (bm.width - x) } - if (y + h > bm.height) { hh = (bm.height - y) } - // Extract the sub bitmap if needed - val cropedBitmap: Bitmap = Bitmap.createBitmap(bm, x, y, ww, hh) - // Write bitmap to a memory and build a jumbo command - var bytesOut = ByteArrayOutputStream() - var dos = DataOutputStream(bytesOut) - dos.writeShort(27) // Jumbo command - dos.writeShort(8) // Jumbo command size - dos.writeInt(0) // Next command size (0 for now) - dos.writeShort(3) // Image command - dos.writeShort(0) // Image command size, 0 since jumbo is used - dos.writeShort(x) // X - dos.writeShort(y) // Y - if (g_desktop_imageType == 4) { // WebP - if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { - cropedBitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, g_desktop_compressionLevel, dos) - } else { - if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { - cropedBitmap.compress(Bitmap.CompressFormat.WEBP_LOSSLESS, g_desktop_compressionLevel, dos) - } else { - cropedBitmap.compress(Bitmap.CompressFormat.WEBP, g_desktop_compressionLevel, dos) - } - } - } else if (g_desktop_imageType == 2) { // PNG - cropedBitmap.compress(Bitmap.CompressFormat.PNG, g_desktop_compressionLevel, dos) - } else { // JPEG (Default) - cropedBitmap.compress(Bitmap.CompressFormat.JPEG, g_desktop_compressionLevel, dos) + if (forceFullFrame) { + encoder.requestFullFrame() + forceFullFrame = false } - cropedBitmap.recycle() - var data = bytesOut.toByteArray() - var cmdSize : Int = (data.size - 8) - data[4] = (cmdSize shr 24).toByte() - data[5] = (cmdSize shr 16).toByte() - data[6] = (cmdSize shr 8).toByte() - data[7] = (cmdSize).toByte() - sendDesktopTunnelData(data.toByteString()) // Send the data to all remote desktop tunnels - } + encoder.encode(bitmap) { AgentController.sendDesktopTunnelData(it) } - private fun sendEntireImage(bm: Bitmap) { - // Write bitmap to a memory and build a jumbo command - var bytesOut = ByteArrayOutputStream() - var dos = DataOutputStream(bytesOut) - dos.writeShort(27) // Jumbo command - dos.writeShort(8) // Jumbo command size - dos.writeInt(0) // Next command size (0 for now) - dos.writeShort(3) // Image command - dos.writeShort(0) // Image command size, 0 since jumbo is used - dos.writeShort(0) // X - dos.writeShort(0) // Y - if (g_desktop_imageType == 4) { // WebP - if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { - bm.compress(Bitmap.CompressFormat.WEBP_LOSSY, g_desktop_compressionLevel, dos) - } else { - if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { - bm.compress(Bitmap.CompressFormat.WEBP_LOSSLESS, g_desktop_compressionLevel, dos) - } else { - bm.compress(Bitmap.CompressFormat.WEBP, g_desktop_compressionLevel, dos) - } - } - } else if (g_desktop_imageType == 2) { // PNG - bm.compress(Bitmap.CompressFormat.PNG, g_desktop_compressionLevel, dos) - } else { // JPEG (Default) - bm.compress(Bitmap.CompressFormat.JPEG, g_desktop_compressionLevel, dos) + if (lastFrameBitmap !== bitmap) { + lastFrameBitmap?.recycle() + lastFrameBitmap = bitmap } - var data = bytesOut.toByteArray() - var cmdSize : Int = (data.size - 8) - data[4] = (cmdSize shr 24).toByte() - data[5] = (cmdSize shr 16).toByte() - data[6] = (cmdSize shr 8).toByte() - data[7] = (cmdSize).toByte() - sendDesktopTunnelData(data.toByteString()) // Send the data to all remote desktop tunnels } - // Resize a bitmap private fun getResizedBitmap(bm: Bitmap, newWidth: Int, newHeight: Int): Bitmap? { val width = bm.width val height = bm.height @@ -403,6 +224,7 @@ class ScreenCaptureService : Service() { } g_ScreenCaptureService = this + g_remoteDesktopProvider = this updateTunnelDisplaySize() sendAgentConsole("Started display sharing") } @@ -421,7 +243,15 @@ class ScreenCaptureService : Service() { if (mMediaProjection != null) { mMediaProjection!!.stop() g_ScreenCaptureService = null + if (g_remoteDesktopProvider === this) { + g_remoteDesktopProvider = null + } + lastFrameBitmap?.recycle() + lastFrameBitmap = null sendAgentConsole("Stopped display sharing") + // The globals are cleared asynchronously on this handler thread, so refresh the + // UI now that capture has actually stopped (hides the "Stop Screen Sharing" item). + AgentController.refreshInfo() } } } @@ -478,7 +308,7 @@ class ScreenCaptureService : Service() { } private val virtualDisplayFlags: Int - private get() = DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY or DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC + get() = DisplayManager.VIRTUAL_DISPLAY_FLAG_OWN_CONTENT_ONLY or DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC } fun updateTunnelDisplaySize() { @@ -491,21 +321,7 @@ class ScreenCaptureService : Service() { } fun checkNoMoreDesktopTunnels() { - if (meshAgent == null) return; - var desktopTunnelCloud = 0 - for (t in meshAgent!!.tunnels) { - // If this is a connected desktop tunnel, count it - if ((t.state == 2) && (t.usage == 2)) { desktopTunnelCloud++ } - } - if (desktopTunnelCloud == 0) { - // If there are no more desktop tunnels, stop projection - if (!g_autoConsent) { - g_mainActivity!!.stopProjection() - } else { // reset the tilesFullWide and tilesFullHigh so on next connect it will send the whole image rather than changed tiles - tilesFullWide = 0 - tilesFullHigh = 0 - } - } + AgentController.checkNoMoreDesktopTunnels() } // Get the maximum outbound queue size of all remote desktop sockets @@ -524,12 +340,55 @@ class ScreenCaptureService : Service() { // Send data to all remote desktop sockets fun sendDesktopTunnelData(data: ByteString) { - if (meshAgent == null) return; - for (t in meshAgent!!.tunnels) { - // If this is a connected desktop tunnel, send the data - if ((t.state == 2) && (t.usage == 2) && (t._webSocket != null)) { - t._webSocket!!.send(data) - } + AgentController.sendDesktopTunnelData(data) + } + + override fun requestFullFrame() { + val handler = mHandler + if (handler == null) { + forceFullFrame = true + return + } + handler.post { + forceFullFrame = true + encoder.requestFullFrame() + pushCurrentFrame() + } + } + + // Push a full frame of the current screen immediately, instead of waiting for the next on-screen + // change to trigger onImageAvailable. This is what makes the first image appear right away on a + // (re)connect / refresh while the app is in the background on a static screen. + private fun pushCurrentFrame() { + val imageReader = mImageReader ?: return + if (meshAgent?.tunnels?.getOrNull(0) == null) return + + val image = try { imageReader.acquireLatestImage() } catch (e: Exception) { null } + if (image != null) { + try { processImage(image) } finally { image.close() } + return + } + + val cached = lastFrameBitmap + if (cached != null && !cached.isRecycled) { + encoder.requestFullFrame() + forceFullFrame = false + encoder.encode(cached) { AgentController.sendDesktopTunnelData(it) } + return + } + + recreateVirtualDisplay() + } + + private fun recreateVirtualDisplay() { + try { + mVirtualDisplay?.release() + mVirtualDisplay = null + mImageReader?.setOnImageAvailableListener(null, null) + mImageReader = null + if (mMediaProjection != null) createVirtualDisplay() + } catch (e: Exception) { + e.printStackTrace() } } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt b/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt index e88166c..80641c7 100644 --- a/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt +++ b/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt @@ -1,20 +1,60 @@ package com.meshcentral.agent import android.os.Bundle +import android.content.Intent +import android.net.Uri +import android.os.Build +import android.provider.Settings import android.view.View import androidx.navigation.fragment.findNavController +import androidx.preference.Preference import androidx.preference.PreferenceFragmentCompat +import androidx.preference.SwitchPreferenceCompat class SettingsFragment : PreferenceFragmentCompat() { override fun onCreatePreferences(savedInstanceState: Bundle?, rootKey: String?) { setPreferencesFromResource(R.xml.root_preferences, rootKey) + if (AgentController.enterpriseEnforced) { + findPreference("pref_autoconnect")?.isEnabled = false + findPreference("pref_autoconsent")?.isEnabled = false + findPreference("pref_autoconnect")?.summary = getString(R.string.enterprise_enforced) + findPreference("pref_autoconsent")?.summary = getString(R.string.enterprise_enforced) + } + findPreference("pref_unattended_accessibility")?.setOnPreferenceClickListener { + startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)) + true + } + findPreference("pref_battery_optimization")?.setOnPreferenceClickListener { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && !AgentController.isIgnoringBatteryOptimizations()) { + try { + val intent = Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) + intent.data = Uri.parse("package:${requireContext().packageName}") + startActivity(intent) + } catch (ex: Exception) { + startActivity(Intent(Settings.ACTION_IGNORE_BATTERY_OPTIMIZATION_SETTINGS)) + } + } + true + } + findPreference("pref_notification_permission")?.setOnPreferenceClickListener { + val intent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) + .putExtra(Settings.EXTRA_APP_PACKAGE, requireContext().packageName) + } else { + Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS) + .setData(Uri.parse("package:${requireContext().packageName}")) + } + startActivity(intent) + true + } } override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) settingsFragment = this; visibleScreen = 5; + refreshStatus() } override fun onDestroy() { @@ -26,4 +66,31 @@ class SettingsFragment : PreferenceFragmentCompat() { g_mainActivity?.settingsChanged() findNavController().navigate(R.id.action_settingsFragment_to_FirstFragment) } -} \ No newline at end of file + + private fun refreshStatus() { + findPreference("pref_unattended_accessibility")?.summary = + if (AgentController.isAccessibilityServiceEnabled()) { + getString(R.string.ready) + } else { + getString(R.string.unattended_accessibility_summary) + } + findPreference("pref_battery_optimization")?.summary = + if (AgentController.isIgnoringBatteryOptimizations()) { + getString(R.string.ready) + } else { + getString(R.string.battery_optimization_summary) + } + findPreference("pref_notification_permission")?.summary = + if (AgentController.areNotificationsEnabled()) { + getString(R.string.ready) + } else { + getString(R.string.notification_permission_summary) + } + findPreference("pref_boot_start")?.summary = + if (AgentController.shouldAutoStart()) { + getString(R.string.ready) + } else { + getString(R.string.needs_setup) + } + } +} diff --git a/app/src/main/res/menu/menu_main.xml b/app/src/main/res/menu/menu_main.xml index cd98730..a5bf840 100644 --- a/app/src/main/res/menu/menu_main.xml +++ b/app/src/main/res/menu/menu_main.xml @@ -27,6 +27,11 @@ android:orderInCategory="104" android:title="@string/stopsharescreen" app:showAsAction="never" /> + Automatic Consent Automatically give consent to remote agent Always ask for consent when remote agent connects + Unattended Access + Accessibility Remote Control + Enable once in Android settings for unattended screen sharing and remote input + Battery Optimization + Allow background operation for reliable startup and reconnect + Notifications + Allow the foreground service status notification + Startup + Starts automatically after reboot when paired + Ready + Needs setup + Unattended access required + Open MeshCentral Agent to finish Android unattended setup + Allows MeshCentral Agent to share the screen and perform remote input after setup on managed devices. + Enterprise enforced + Finish unattended access setup + This build is %1$s.\n\nMissing setup:\n%2$s\n\nRemote desktop will show a black screen until Accessibility Remote Control is enabled, or until the app is open so Android can ask for screen capture consent. + Open Accessibility + Open App Settings + Later + Share this screen? + A remote user is requesting to view this screen.\n\nFor unattended remote control, enable Accessibility Remote Control (recommended). Otherwise you can allow Android screen capture for this session. + Share screen + Accessibility Remote Control + battery unrestricted mode + notification permission + Check Unattended Setup + Unattended access setup is complete. + Connection notification + Show a notification while someone is connected + No notification while someone is connected + Remote session active + Connected %1$s + %1$d users connected Setup to: %1$s? Clear server setup? Server Pairing Link Invalid Server Pairing Linbk - \ No newline at end of file + diff --git a/app/src/main/res/xml/mesh_accessibility_service.xml b/app/src/main/res/xml/mesh_accessibility_service.xml new file mode 100644 index 0000000..221ed86 --- /dev/null +++ b/app/src/main/res/xml/mesh_accessibility_service.xml @@ -0,0 +1,10 @@ + + diff --git a/app/src/main/res/xml/root_preferences.xml b/app/src/main/res/xml/root_preferences.xml index 50f85ef..bf85594 100644 --- a/app/src/main/res/xml/root_preferences.xml +++ b/app/src/main/res/xml/root_preferences.xml @@ -17,6 +17,33 @@ app:summaryOff="@string/always_ask_for_consent_when_remote_agent_connects" app:defaultValue="false" app:useSimpleSummaryProvider="true" /> + - \ No newline at end of file + + + + + + + diff --git a/build.gradle b/build.gradle index 27b203c..22a0392 100644 --- a/build.gradle +++ b/build.gradle @@ -1,14 +1,14 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. buildscript { - ext.kotlin_version = '1.9.10' + ext.kotlin_version = '1.9.24' repositories { google() - jcenter() + mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:8.1.2' + classpath 'com.android.tools.build:gradle:8.7.3' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" - classpath 'com.google.gms:google-services:4.4.0' + classpath 'com.google.gms:google-services:4.4.3' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files @@ -18,10 +18,11 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() + maven { url 'https://jitpack.io' } } } -task clean(type: Delete) { - delete rootProject.buildDir -} \ No newline at end of file +tasks.register('clean', Delete) { + delete layout.buildDirectory +} diff --git a/gradle.properties b/gradle.properties index 9bba2f4..913533d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,10 +19,9 @@ android.useAndroidX=true android.enableJetifier=true # Kotlin code style for this project: "official" or "obsolete": kotlin.code.style=official -android.defaults.buildfeatures.buildconfig=true android.nonTransitiveRClass=false android.nonFinalResIds=false # Enable more aggressive optimizations -# android.enableR8.fullMode=true \ No newline at end of file +# android.enableR8.fullMode=true diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index da1db5f..19cfad9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 index 1b6c787..74c5bbe --- a/gradlew +++ b/gradlew @@ -139,6 +139,26 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi +java_major=$("$JAVACMD" -version 2>&1 | sed -n 's/.* version "\([0-9][0-9]*\).*/\1/p' | head -n 1) +if [ "$java_major" -ge 22 ] 2>/dev/null; then + for java_home_candidate in "$JAVA17_HOME" "$JDK17_HOME" "$JAVA_HOME_17_X64" \ + /usr/lib/jvm/java-17-openjdk \ + /usr/lib/jvm/java-17-openjdk-amd64 \ + /Library/Java/JavaVirtualMachines/temurin-17.jdk/Contents/Home \ + /Library/Java/JavaVirtualMachines/zulu-17.jdk/Contents/Home + do + if [ -n "$java_home_candidate" ] && [ -x "$java_home_candidate/bin/java" ]; then + JAVA_HOME=$java_home_candidate + JAVACMD=$JAVA_HOME/bin/java + java_major=$("$JAVACMD" -version 2>&1 | sed -n 's/.* version "\([0-9][0-9]*\).*/\1/p' | head -n 1) + break + fi + done +fi +if [ "$java_major" -ge 22 ] 2>/dev/null; then + DEFAULT_JVM_OPTS="$DEFAULT_JVM_OPTS \"--enable-native-access=ALL-UNNAMED\"" +fi + # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( From 4751c1e0746d658c8e67a3338949b4930081da66 Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Sun, 31 May 2026 08:01:58 +1000 Subject: [PATCH 2/9] Improve agent lifecycle management and remote desktop handling for better battery usage - Add retry backoff logic with delay for agent reconnection attempts - Introduce `shouldKeepForegroundServiceRunning` for better service control - Optimise frame encoding with idle frame delay adjustments - Enhance user disconnect handling via shared preferences - Adjust remote desktop frame capturing to dynamically handle delays --- .../agent/AgentForegroundService.kt | 6 +- .../com/meshcentral/agent/AgentRuntime.kt | 71 ++++++++++++++++--- .../meshcentral/agent/DesktopFrameEncoder.kt | 7 +- .../agent/MeshAccessibilityService.kt | 18 ++++- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt b/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt index 52b2e15..da9865e 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt @@ -44,7 +44,11 @@ class AgentForegroundService : Service() { } } updateNotification() - return START_STICKY + val keepRunning = AgentController.shouldKeepForegroundServiceRunning() + if (!keepRunning) { + stopSelf() + } + return if (keepRunning) START_STICKY else START_NOT_STICKY } override fun onDestroy() { diff --git a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt index 23fae87..e0d15e3 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt @@ -80,12 +80,16 @@ interface RemoteDesktopProvider { } object AgentController : AgentHost { + private const val INITIAL_RETRY_DELAY_MS = 10_000L + private const val MAX_RETRY_DELAY_MS = 300_000L private lateinit var appContext: Context private val mainHandler = Handler(Looper.getMainLooper()) private var initialized = false private var activity: MainActivity? = null private var service: AgentForegroundService? = null private var retryRunnable: Runnable? = null + private var retryDelayMs = INITIAL_RETRY_DELAY_MS + private var retryAttemptInProgress = false private var batteryReceiver: BroadcastReceiver? = null private var projectionRetryRunnable: Runnable? = null private var projectionRetryCount = 0 @@ -144,7 +148,13 @@ object AgentController : AgentHost { fun shouldAutoStart(): Boolean { loadServerLink() loadSettings() - return serverLink != null && (enterpriseEnforced || g_autoConnect) + return serverLink != null && (enterpriseEnforced || (g_autoConnect && !g_userDisconnect)) + } + + fun shouldKeepForegroundServiceRunning(): Boolean { + loadServerLink() + loadSettings() + return meshAgent != null || shouldAutoStart() || hasActiveDesktopTunnel() } fun setMeshServerLink(x: String?) { @@ -170,7 +180,7 @@ object AgentController : AgentHost { service?.stopSelf() } - g_userDisconnect = false + setUserDisconnected(false) refreshInfo() if (g_autoConnect || enterpriseEnforced) { toggleAgentConnection(false) @@ -180,9 +190,11 @@ object AgentController : AgentHost { fun settingsChanged() { loadSettings() if (!enterpriseEnforced && !g_autoConnect) { + setUserDisconnected(false) stopRetryTimer() refreshInfo() service?.updateNotification() + stopServiceIfIdle() return } if ((meshAgent == null) && !g_userDisconnect && hasServerLink()) { @@ -191,6 +203,7 @@ object AgentController : AgentHost { } refreshInfo() service?.updateNotification() + stopServiceIfIdle() } fun toggleAgentConnection(userInitiated: Boolean) { @@ -199,25 +212,28 @@ object AgentController : AgentHost { if ((meshAgent == null) && (serverLink != null)) { ensureIdentity() if (!userInitiated) { - g_userDisconnect = false + setUserDisconnected(false) + if (!retryAttemptInProgress) resetRetryBackoff() startAgent() } else { if (g_autoConnect || enterpriseEnforced) { if (g_userDisconnect) { - g_userDisconnect = false + setUserDisconnected(false) + resetRetryBackoff() startAgent() } else { - g_userDisconnect = true + setUserDisconnected(true) stopRetryTimer() } } else { - g_userDisconnect = true + setUserDisconnected(true) + resetRetryBackoff() startAgent() } } } else if (meshAgent != null) { if (userInitiated && !enterpriseEnforced) { - g_userDisconnect = true + setUserDisconnected(true) } stopProjection() meshAgent?.Stop() @@ -226,6 +242,7 @@ object AgentController : AgentHost { } refreshInfo() service?.updateNotification() + stopServiceIfIdle() } private fun startAgent() { @@ -241,6 +258,9 @@ object AgentController : AgentHost { if ((meshAgent != null) && (meshAgent?.state == 0)) { meshAgent = null } + if (meshAgent?.state == 3) { + resetRetryBackoff() + } if (((meshAgent != null) && (meshAgent?.state != 0)) || g_userDisconnect || (!g_autoConnect && !enterpriseEnforced)) { stopRetryTimer() } else if ((meshAgent == null) && !g_userDisconnect && (g_autoConnect || enterpriseEnforced) && retryRunnable == null) { @@ -248,6 +268,7 @@ object AgentController : AgentHost { } refreshInfo() service?.updateNotification() + stopServiceIfIdle() } } @@ -597,15 +618,18 @@ object AgentController : AgentHost { g_autoConnect = enterpriseEnforced || pm.getBoolean("pref_autoconnect", false) g_autoConsent = enterpriseEnforced || pm.getBoolean("pref_autoconsent", false) g_sessionNotification = pm.getBoolean("pref_session_notification", false) + g_userDisconnect = !enterpriseEnforced && pm.getBoolean("pref_user_disconnect", false) if (enterpriseEnforced) { pm.edit() .putBoolean("pref_autoconnect", true) .putBoolean("pref_autoconsent", true) + .putBoolean("pref_user_disconnect", false) .apply() } } private fun loadFirebaseToken() { + if (pushMessagingToken != null) return try { FirebaseMessaging.getInstance().token.addOnSuccessListener { tokenString -> pushMessagingToken = tokenString @@ -637,13 +661,22 @@ object AgentController : AgentHost { if (retryRunnable != null) return retryRunnable = object : Runnable { override fun run() { + retryRunnable = null + if ((meshAgent == null) && !g_userDisconnect && (g_autoConnect || enterpriseEnforced)) { + retryAttemptInProgress = true + try { + toggleAgentConnection(false) + } finally { + retryAttemptInProgress = false + } + } + retryDelayMs = (retryDelayMs * 2).coerceAtMost(MAX_RETRY_DELAY_MS) if ((meshAgent == null) && !g_userDisconnect && (g_autoConnect || enterpriseEnforced)) { - toggleAgentConnection(false) + startRetryTimer() } - mainHandler.postDelayed(this, 10000) } } - mainHandler.postDelayed(retryRunnable!!, 10000) + mainHandler.postDelayed(retryRunnable!!, retryDelayMs) } private fun stopRetryTimer() { @@ -651,6 +684,24 @@ object AgentController : AgentHost { retryRunnable = null } + private fun resetRetryBackoff() { + retryDelayMs = INITIAL_RETRY_DELAY_MS + } + + private fun setUserDisconnected(disconnected: Boolean) { + g_userDisconnect = disconnected && !enterpriseEnforced + PreferenceManager.getDefaultSharedPreferences(appContext) + .edit() + .putBoolean("pref_user_disconnect", g_userDisconnect) + .apply() + } + + private fun stopServiceIfIdle() { + if (!shouldKeepForegroundServiceRunning()) { + service?.stopSelf() + } + } + private fun ensureIdentity() { if (agentCertificate != null && agentCertificateKey != null) return val sharedPreferences = appContext.getSharedPreferences("meshagent", Context.MODE_PRIVATE) diff --git a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt index eca9245..ef92453 100644 --- a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt +++ b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt @@ -20,7 +20,7 @@ class DesktopFrameEncoder { forceFullFrame = true } - fun encode(bitmap: Bitmap, sink: (ByteString) -> Unit) { + fun encode(bitmap: Bitmap, sink: (ByteString) -> Unit): Boolean { if (frameWidth != bitmap.width || frameHeight != bitmap.height || oldcrcs == null || newcrcs == null) { frameWidth = bitmap.width frameHeight = bitmap.height @@ -37,13 +37,13 @@ class DesktopFrameEncoder { for (i in 0 until tilesCount) { if (forceFullFrame || oldcrcs!![i] != newcrcs!![i]) changedTiles++ } - if (changedTiles == 0) return + if (changedTiles == 0) return false if (forceFullFrame || ((changedTiles * 100) >= (tilesCount * 85))) { sink(buildImageCommand(bitmap, 0, 0, bitmap.width, bitmap.height)) for (i in 0 until tilesCount) oldcrcs!![i] = newcrcs!![i] forceFullFrame = false - return + return true } var sendx = -1 @@ -75,6 +75,7 @@ class DesktopFrameEncoder { sendSubBitmapRow(bitmap, sendx, sendy, sendw, sink) } forceFullFrame = false + return true } private fun sendSubBitmapRow(bitmap: Bitmap, x: Int, y: Int, w: Int, sink: (ByteString) -> Unit) { diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt index 67ec238..b144a8a 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -14,6 +14,7 @@ import android.view.accessibility.AccessibilityNodeInfo import okio.ByteString import kotlin.math.absoluteValue import kotlin.math.max +import kotlin.math.min class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private val mainHandler = Handler(Looper.getMainLooper()) @@ -26,6 +27,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private var pointerDownX: Int? = null private var pointerDownY: Int? = null private var unsupportedKeyboardNotified = false + private var nextFrameDelayMs = MIN_FRAME_DELAY_MS override val isRunning: Boolean get() = active @@ -53,6 +55,8 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { } override fun onAccessibilityEvent(event: AccessibilityEvent?) { + if (!active) return + nextFrameDelayMs = MIN_FRAME_DELAY_MS } override fun onInterrupt() { @@ -67,6 +71,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { active = true g_remoteDesktopProvider = this unsupportedKeyboardNotified = false + nextFrameDelayMs = MIN_FRAME_DELAY_MS encoder.requestFullFrame() updateTunnelDisplaySize() captureFrame() @@ -87,6 +92,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { } override fun requestFullFrame() { + nextFrameDelayMs = MIN_FRAME_DELAY_MS encoder.requestFullFrame() } @@ -198,7 +204,12 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { } else { bitmap } - encoder.encode(encodedBitmap) { AgentController.sendDesktopTunnelData(it) } + val sentFrame = encoder.encode(encodedBitmap) { AgentController.sendDesktopTunnelData(it) } + nextFrameDelayMs = if (sentFrame) { + MIN_FRAME_DELAY_MS + } else { + min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) + } if (encodedBitmap !== bitmap) encodedBitmap.recycle() bitmap.recycle() } @@ -221,7 +232,8 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun scheduleNextCapture() { if (!active) return - mainHandler.postDelayed(captureRunnable, max(100L, g_desktop_frameRateLimiter.toLong())) + val requestedDelay = max(MIN_FRAME_DELAY_MS, g_desktop_frameRateLimiter.toLong()) + mainHandler.postDelayed(captureRunnable, max(requestedDelay, nextFrameDelayMs)) } private fun handleLegacyKey(msg: ByteString): Boolean { @@ -328,5 +340,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { companion object { var instance: MeshAccessibilityService? = null private set + private const val MIN_FRAME_DELAY_MS = 100L + private const val MAX_IDLE_FRAME_DELAY_MS = 10_000L } } From b948b1f2c7fdc01656b4351a4ecf7c94847df0e3 Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Sun, 31 May 2026 19:26:00 +1000 Subject: [PATCH 3/9] Avoid unattended screen capture before remote desktop session - MeshAgent: stop starting projection immediately after the agent control connection authenticates. - AgentRuntime: require an active desktop tunnel before starting screen projection. --- app/src/main/java/com/meshcentral/agent/AgentRuntime.kt | 1 + app/src/main/java/com/meshcentral/agent/MeshAgent.kt | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt index e0d15e3..d8c6935 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt @@ -290,6 +290,7 @@ object AgentController : AgentHost { override fun startProjection() { if (meshAgent == null || meshAgent?.state != 3) return + if (!hasActiveDesktopTunnel()) return if (isRemoteDesktopRunning()) return val accessibility = MeshAccessibilityService.instance if (accessibility != null) { diff --git a/app/src/main/java/com/meshcentral/agent/MeshAgent.kt b/app/src/main/java/com/meshcentral/agent/MeshAgent.kt index b9057cb..ad7a21c 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAgent.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAgent.kt @@ -308,10 +308,6 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S sendNetworkUpdate(false) sendServerImageRequest() - if (g_autoConsent) { - parent.startProjection() - } - // Send battery state if (_webSocket != null) { _webSocket?.send(getSysBatteryInfo().toString().toByteArray().toByteString()) } } From 4f1f0aa720b63dfcfd55860435de4c4002ac03a3 Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Sun, 14 Jun 2026 16:37:15 +1000 Subject: [PATCH 4/9] Add explicit user consent for screen sharing when automatic consent is disabled - Implement user consent prompt via dialogs and notifications for remote desktop access. - Add new notification channel for "Approve/Deny" screen sharing actions. - Ensure thread-safe handling and lifecycle management for capture processes. - Remove redundant battery optimization request logic. - Optimize screenshot capture flow with throttling and backoff mechanisms. --- .../agent/AgentForegroundService.kt | 52 ++++++++++ .../com/meshcentral/agent/AgentRuntime.kt | 94 ++++++++++++++----- .../meshcentral/agent/DesktopFrameEncoder.kt | 3 +- .../com/meshcentral/agent/MainActivity.kt | 26 ++++- .../agent/MeshAccessibilityService.kt | 38 ++++++-- app/src/main/res/values/strings.xml | 5 + 6 files changed, 183 insertions(+), 35 deletions(-) diff --git a/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt b/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt index da9865e..a7cf940 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentForegroundService.kt @@ -31,6 +31,14 @@ class AgentForegroundService : Service() { ACTION_CONNECT -> if (meshAgent == null) AgentController.toggleAgentConnection(false) ACTION_DISCONNECT -> if (!AgentController.enterpriseEnforced && meshAgent != null) AgentController.toggleAgentConnection(true) ACTION_STOP_SCREEN_SHARING -> AgentController.stopScreenSharingByUser() + ACTION_APPROVE_SCREEN_SHARING -> { + cancelConsentNotification(this) + AgentController.confirmUnattendedConsent() + } + ACTION_DENY_SCREEN_SHARING -> { + cancelConsentNotification(this) + AgentController.denyUnattendedConsent() + } ACTION_STOP -> { if (!AgentController.enterpriseEnforced) { if (meshAgent != null) AgentController.toggleAgentConnection(true) @@ -126,12 +134,17 @@ class AgentForegroundService : Service() { private const val CHANNEL_NAME = "MeshCentral Agent" private const val SESSION_CHANNEL_ID = "meshcentral_agent_session" private const val SESSION_CHANNEL_NAME = "Remote session active" + private const val CONSENT_CHANNEL_ID = "meshcentral_agent_consent" + private const val CONSENT_CHANNEL_NAME = "Screen sharing approval" private const val NOTIFICATION_ID = 2401 private const val RUNTIME_NOTIFICATION_ID = 2402 private const val SESSION_NOTIFICATION_ID = 2403 + private const val CONSENT_NOTIFICATION_ID = 2404 private const val ACTION_CONNECT = "com.meshcentral.agent.action.CONNECT" private const val ACTION_DISCONNECT = "com.meshcentral.agent.action.DISCONNECT" private const val ACTION_STOP_SCREEN_SHARING = "com.meshcentral.agent.action.STOP_SCREEN_SHARING" + private const val ACTION_APPROVE_SCREEN_SHARING = "com.meshcentral.agent.action.APPROVE_SCREEN_SHARING" + private const val ACTION_DENY_SCREEN_SHARING = "com.meshcentral.agent.action.DENY_SCREEN_SHARING" private const val ACTION_STOP = "com.meshcentral.agent.action.STOP" fun start(context: Context) { @@ -168,6 +181,36 @@ class AgentForegroundService : Service() { } } + // Consent request with Approve/Deny actions, for when the app isn't foregrounded. + fun showConsentNotification(context: Context) { + createConsentNotificationChannel(context) + val body = context.getString(R.string.approve_screen_sharing_body) + val notification = NotificationCompat.Builder(context, CONSENT_CHANNEL_ID) + .setSmallIcon(R.drawable.ic_cloud) + .setContentTitle(context.getString(R.string.approve_screen_sharing_title)) + .setContentText(body) + .setStyle(NotificationCompat.BigTextStyle().bigText(body)) + .setContentIntent(openAppPendingIntent(context, null)) + .setCategory(Notification.CATEGORY_CALL) + .setPriority(NotificationCompat.PRIORITY_HIGH) + .setOngoing(true) + .setAutoCancel(false) + .addAction(R.drawable.ic_cloud, context.getString(R.string.approve), servicePendingIntent(context, ACTION_APPROVE_SCREEN_SHARING, 4)) + .addAction(R.drawable.ic_cloud, context.getString(R.string.deny), servicePendingIntent(context, ACTION_DENY_SCREEN_SHARING, 5)) + .build() + try { + NotificationManagerCompat.from(context).notify(CONSENT_NOTIFICATION_ID, notification) + } catch (_: SecurityException) { + } + } + + fun cancelConsentNotification(context: Context) { + try { + NotificationManagerCompat.from(context).cancel(CONSENT_NOTIFICATION_ID) + } catch (_: SecurityException) { + } + } + private fun buildNotification(context: Context): Notification { val state = when (meshAgent?.state ?: 0) { 1 -> context.getString(R.string.connecting) @@ -242,5 +285,14 @@ class AgentForegroundService : Service() { manager.createNotificationChannel(channel) } } + + private fun createConsentNotificationChannel(context: Context) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val channel = NotificationChannel(CONSENT_CHANNEL_ID, CONSENT_CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH) + channel.lockscreenVisibility = Notification.VISIBILITY_PRIVATE + val manager = context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.createNotificationChannel(channel) + } + } } } diff --git a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt index d8c6935..6c9b242 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt @@ -22,10 +22,12 @@ import android.view.Gravity import android.widget.Toast import androidx.core.app.NotificationManagerCompat import androidx.core.content.ContextCompat +import androidx.lifecycle.Lifecycle import androidx.preference.PreferenceManager import com.google.firebase.messaging.FirebaseMessaging import okio.ByteString import okio.ByteString.Companion.toByteString +import org.json.JSONObject import org.spongycastle.asn1.x500.X500Name import org.spongycastle.cert.X509v3CertificateBuilder import org.spongycastle.cert.jcajce.JcaX509CertificateConverter @@ -91,6 +93,8 @@ object AgentController : AgentHost { private var retryDelayMs = INITIAL_RETRY_DELAY_MS private var retryAttemptInProgress = false private var batteryReceiver: BroadcastReceiver? = null + private var settingsListener: SharedPreferences.OnSharedPreferenceChangeListener? = null + private var handlingSettingsChange = false private var projectionRetryRunnable: Runnable? = null private var projectionRetryCount = 0 private val MAX_PROJECTION_RETRIES = 12 @@ -112,6 +116,7 @@ object AgentController : AgentHost { if (!initialized) { initialized = true registerBatteryReceiver() + registerSettingsListener() } } @@ -174,7 +179,6 @@ object AgentController : AgentHost { .apply() g_autoConnect = true AgentForegroundService.start(appContext) - requestBatteryOptimizationExemption() } else { stopProjection() service?.stopSelf() @@ -288,14 +292,34 @@ object AgentController : AgentHost { } } + // May run on the tunnel's OkHttp thread; dialogs and activities must start on the main thread, or a + // dialog built off it throws "Can't create handler ... Looper.prepare()". override fun startProjection() { + runOnHostThread { startProjectionOnHostThread() } + } + + private fun startProjectionOnHostThread() { if (meshAgent == null || meshAgent?.state != 3) return if (!hasActiveDesktopTunnel()) return if (isRemoteDesktopRunning()) return val accessibility = MeshAccessibilityService.instance if (accessibility != null) { cancelProjectionRetry() - if (accessibility.startDesktop()) return + if (g_autoConsent) { + if (accessibility.startDesktop()) return + } else { + // Automatic Consent off: require explicit approval before capturing. + val mainActivity = activity + val resumed = mainActivity?.lifecycle?.currentState?.isAtLeast(Lifecycle.State.RESUMED) == true + if (mainActivity != null && resumed) { + mainActivity.promptUnattendedConsent() + } else { + // No foreground activity to host a dialog, so ask via the notification. + sendDesktopMessage("Waiting for the device user to approve screen sharing.") + AgentForegroundService.showConsentNotification(appContext) + } + return + } } else if (isAccessibilityServiceEnabled() && waitForAccessibilityProjection()) { // Unattended access is granted but the accessibility service has not rebound yet (common // right after an app update). Wait for it to connect instead of reporting setup as missing. @@ -326,7 +350,24 @@ object AgentController : AgentHost { ) } + fun confirmUnattendedConsent() { + if (::appContext.isInitialized) AgentForegroundService.cancelConsentNotification(appContext) + if (meshAgent?.state != 3 || !hasActiveDesktopTunnel() || isRemoteDesktopRunning()) return + runOnHostThread { MeshAccessibilityService.instance?.startDesktop() } + } + + fun denyUnattendedConsent() { + val tunnel = meshAgent?.tunnels?.getOrNull(0) ?: return + val json = JSONObject() + json.put("type", "console") + json.put("msg", "denied") + json.put("msgid", 2) + tunnel.sendCtrlResponse(json) + tunnel.Stop() + } + override fun stopProjection() { + if (::appContext.isInitialized) AgentForegroundService.cancelConsentNotification(appContext) val provider = g_remoteDesktopProvider if (provider is MeshAccessibilityService) { provider.stopDesktop() @@ -562,27 +603,6 @@ object AgentController : AgentHost { } } - fun requestBatteryOptimizationExemption() { - if (!::appContext.isInitialized) return - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return - val powerManager = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager - if (powerManager.isIgnoringBatteryOptimizations(appContext.packageName)) return - val mainActivity = activity ?: return - if (ContextCompat.checkSelfPermission(appContext, Manifest.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) != PackageManager.PERMISSION_GRANTED) { - return - } - try { - val intent = Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) - intent.data = Uri.parse("package:${appContext.packageName}") - mainActivity.startActivity(intent) - } catch (ex: Exception) { - try { - mainActivity.startActivity(Intent(Settings.ACTION_IGNORE_BATTERY_OPTIMIZATION_SETTINGS)) - } catch (_: Exception) { - } - } - } - fun isIgnoringBatteryOptimizations(): Boolean { if (!::appContext.isInitialized) return false if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return true @@ -620,7 +640,11 @@ object AgentController : AgentHost { g_autoConsent = enterpriseEnforced || pm.getBoolean("pref_autoconsent", false) g_sessionNotification = pm.getBoolean("pref_session_notification", false) g_userDisconnect = !enterpriseEnforced && pm.getBoolean("pref_user_disconnect", false) - if (enterpriseEnforced) { + // Only write if a value differs, or the change listener loops on enterprise builds. + if (enterpriseEnforced && + (!pm.getBoolean("pref_autoconnect", false) || + !pm.getBoolean("pref_autoconsent", false) || + pm.getBoolean("pref_user_disconnect", false))) { pm.edit() .putBoolean("pref_autoconnect", true) .putBoolean("pref_autoconsent", true) @@ -629,6 +653,28 @@ object AgentController : AgentHost { } } + // Apply setting toggles immediately, not only when the Settings screen closes. pref_user_disconnect + // is internal state written in code, so it's deliberately not watched. + private fun registerSettingsListener() { + if (settingsListener != null) return + val pm = PreferenceManager.getDefaultSharedPreferences(appContext) + val listener = SharedPreferences.OnSharedPreferenceChangeListener { _, key -> + if (handlingSettingsChange) return@OnSharedPreferenceChangeListener + when (key) { + "pref_autoconnect", "pref_autoconsent", "pref_session_notification" -> { + handlingSettingsChange = true + try { + settingsChanged() + } finally { + handlingSettingsChange = false + } + } + } + } + settingsListener = listener + pm.registerOnSharedPreferenceChangeListener(listener) + } + private fun loadFirebaseToken() { if (pushMessagingToken != null) return try { diff --git a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt index ef92453..3603378 100644 --- a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt +++ b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt @@ -14,7 +14,8 @@ class DesktopFrameEncoder { private var tilesCount: Int = 0 private var oldcrcs: IntArray? = null private var newcrcs: IntArray? = null - private var forceFullFrame = true + // Written from the tunnel/main thread, read on the capture thread. + @Volatile private var forceFullFrame = true fun requestFullFrame() { forceFullFrame = true diff --git a/app/src/main/java/com/meshcentral/agent/MainActivity.kt b/app/src/main/java/com/meshcentral/agent/MainActivity.kt index bac1c1e..c0cd59a 100644 --- a/app/src/main/java/com/meshcentral/agent/MainActivity.kt +++ b/app/src/main/java/com/meshcentral/agent/MainActivity.kt @@ -122,7 +122,6 @@ class MainActivity : AppCompatActivity() { settingsChanged() if (serverLink != null) { requestAllPermissions() - AgentController.requestBatteryOptimizationExemption() } } @@ -130,6 +129,10 @@ class MainActivity : AppCompatActivity() { super.onResume() if (serverLink != null) { window.decorView.post { showUnattendedSetupPromptIfNeeded(false) } + // Retry a session that connected while backgrounded and is waiting to prompt for consent. + if (AgentController.hasActiveDesktopTunnel() && !AgentController.isRemoteDesktopRunning()) { + AgentController.startProjection() + } } invalidateOptionsMenu() } @@ -604,6 +607,27 @@ class MainActivity : AppCompatActivity() { .show() } + // Per-connection consent prompt shown when Automatic Consent is off. + fun promptUnattendedConsent() { + if (AgentController.isRemoteDesktopRunning() || (meshAgent == null) || (meshAgent!!.state != 3)) return + if (isFinishing || isDestroyed) return + if (alert != null) { + alert?.dismiss() + alert = null + } + alert = AlertDialog.Builder(this) + .setTitle(R.string.share_screen_choice_title) + .setMessage(R.string.unattended_consent_message) + .setPositiveButton(R.string.share_screen_once) { _, _ -> + AgentController.confirmUnattendedConsent() + } + .setNegativeButton(android.R.string.cancel) { dialog, _ -> + sendDesktopConsentDenied() + dialog.dismiss() + } + .show() + } + private fun sendDesktopConsentDenied() { val tunnel = meshAgent?.tunnels?.getOrNull(0) ?: return val json = JSONObject() diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt index b144a8a..2ca0693 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -12,6 +12,8 @@ import android.view.Display import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import okio.ByteString +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors import kotlin.math.absoluteValue import kotlin.math.max import kotlin.math.min @@ -20,14 +22,17 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private val mainHandler = Handler(Looper.getMainLooper()) private val encoder = DesktopFrameEncoder() private val captureRunnable = Runnable { captureFrame() } - private var active = false - private var capturing = false - private var lastWidth = 0 - private var lastHeight = 0 + // Encode off the main thread so it can't block accessibility input dispatch. + private val captureExecutor: ExecutorService = Executors.newSingleThreadExecutor() + @Volatile private var active = false + @Volatile private var capturing = false + @Volatile private var lastWidth = 0 + @Volatile private var lastHeight = 0 private var pointerDownX: Int? = null private var pointerDownY: Int? = null private var unsupportedKeyboardNotified = false - private var nextFrameDelayMs = MIN_FRAME_DELAY_MS + @Volatile private var nextFrameDelayMs = MIN_FRAME_DELAY_MS + @Volatile private var screenshotErrorNotified = false override val isRunning: Boolean get() = active @@ -51,6 +56,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { override fun onDestroy() { if (instance === this) instance = null stopDesktop() + captureExecutor.shutdown() super.onDestroy() } @@ -184,8 +190,10 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun captureFrame() { if (!active || capturing || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return capturing = true - takeScreenshot(Display.DEFAULT_DISPLAY, mainExecutor, object : AccessibilityService.TakeScreenshotCallback { + takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, object : AccessibilityService.TakeScreenshotCallback { override fun onSuccess(screenshot: AccessibilityService.ScreenshotResult) { + // Recovered: allow the next error to be reported again. + screenshotErrorNotified = false try { val wrapped = Bitmap.wrapHardwareBuffer(screenshot.hardwareBuffer, screenshot.colorSpace) if (wrapped != null) { @@ -214,7 +222,10 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { bitmap.recycle() } } catch (ex: Exception) { - AgentController.sendDesktopMessage("Unable to capture unattended screenshot: ${ex.message}") + if (!screenshotErrorNotified) { + screenshotErrorNotified = true + AgentController.sendDesktopMessage("Unable to capture unattended screenshot: ${ex.message}") + } } finally { screenshot.hardwareBuffer.close() capturing = false @@ -224,7 +235,12 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { override fun onFailure(errorCode: Int) { capturing = false - AgentController.sendDesktopMessage("Unable to capture unattended screenshot, error $errorCode.") + // Throttled or transient error; back off quietly instead of flooding the console. + nextFrameDelayMs = min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) + if (errorCode != AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT && !screenshotErrorNotified) { + screenshotErrorNotified = true + AgentController.sendDesktopMessage("Unable to capture unattended screenshot, error $errorCode.") + } scheduleNextCapture() } }) @@ -233,7 +249,9 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun scheduleNextCapture() { if (!active) return val requestedDelay = max(MIN_FRAME_DELAY_MS, g_desktop_frameRateLimiter.toLong()) - mainHandler.postDelayed(captureRunnable, max(requestedDelay, nextFrameDelayMs)) + // Don't request faster than the system screenshot throttle, whatever the server asks for. + val delay = maxOf(requestedDelay, nextFrameDelayMs, MIN_SCREENSHOT_INTERVAL_MS) + mainHandler.postDelayed(captureRunnable, delay) } private fun handleLegacyKey(msg: ByteString): Boolean { @@ -342,5 +360,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private set private const val MIN_FRAME_DELAY_MS = 100L private const val MAX_IDLE_FRAME_DELAY_MS = 10_000L + // System throttles takeScreenshot() faster than ~3 fps. + private const val MIN_SCREENSHOT_INTERVAL_MS = 350L } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 69a1be7..7e04339 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -93,6 +93,11 @@ Remote session active Connected %1$s %1$d users connected + A remote user wants to view and control this screen for this session. Allow screen sharing? + Approve screen sharing + A remote user is requesting to view and control this screen. + Approve + Deny Setup to: %1$s? Clear server setup? Server Pairing Link From 175a3b619a0c63c762abd946414099acba5666f8 Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Fri, 28 Aug 2026 08:53:34 +1000 Subject: [PATCH 5/9] Add GitHub Actions workflow for Android PR build and improve accessibility handling - Add `.github/workflows/android-pr-build.yml` to automate pull request builds, including debug APK upload. - Refactor `MeshAccessibilityService` screenshot handling with callback extraction. - Prevent unintended `NullPointerException` by introducing safe access and cleanup methods. - Harden navigation logic in fragments with exception handling. - Improve encoded bitmap memory handling during desktop frame capture. - Update remote desktop input documentation to reflect accessibility feature improvements. - Enhance WebSocket handling with safe null checks and connection closures. - Add enhanced test coverage for edge cases and validation logic. --- .github/workflows/android-pr-build.yml | 45 ++++++++ .gitignore | 1 + .idea/git_toolbox_prj.xml | 15 --- .../com/meshcentral/agent/AgentRuntime.kt | 13 ++- .../meshcentral/agent/DesktopFrameEncoder.kt | 10 +- .../com/meshcentral/agent/MainActivity.kt | 13 +-- .../com/meshcentral/agent/MainFragment.kt | 17 ++- .../agent/MeshAccessibilityService.kt | 107 ++++++++++-------- .../agent/MeshFirebaseMessagingService.kt | 7 +- .../java/com/meshcentral/agent/MeshTunnel.kt | 56 ++++----- .../meshcentral/agent/ProtocolValidation.kt | 7 +- .../com/meshcentral/agent/ScannerFragment.kt | 1 - .../meshcentral/agent/ScreenCaptureService.kt | 6 +- .../com/meshcentral/agent/WebViewFragment.kt | 10 +- .../agent/ProtocolValidationTest.kt | 28 ++++- docs/index.md | 5 +- docs/overview.md | 24 ++-- docs/remote-desktop.md | 38 +++---- 18 files changed, 239 insertions(+), 164 deletions(-) create mode 100644 .github/workflows/android-pr-build.yml delete mode 100644 .idea/git_toolbox_prj.xml diff --git a/.github/workflows/android-pr-build.yml b/.github/workflows/android-pr-build.yml new file mode 100644 index 0000000..046f9cb --- /dev/null +++ b/.github/workflows/android-pr-build.yml @@ -0,0 +1,45 @@ +name: Android PR Build + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: android-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + build: + name: Build and test + runs-on: ubuntu-latest + + steps: + - name: Check out source + uses: actions/checkout@v5 + + - name: Set up Java + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v5 + + # No signing secrets on pull requests (and none on fork PRs at all); the release build + # falls back to debug signing. assembleRelease still runs so R8 / resource shrinking break + # the check here rather than after merge. + - name: Build and test + run: | + chmod +x ./gradlew + ./gradlew --no-daemon testDebugUnitTest assembleDebug assembleRelease + + - name: Upload debug APK + uses: actions/upload-artifact@v4 + with: + name: meshcentral-agent-debug + path: app/build/outputs/apk/debug/app-debug.apk + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 5c8ffce..eea4d8c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ /.idea/workspace.xml /.idea/navEditor.xml /.idea/assetWizardSettings.xml +/.idea/git_toolbox_prj.xml .vscode/ .DS_Store /build diff --git a/.idea/git_toolbox_prj.xml b/.idea/git_toolbox_prj.xml deleted file mode 100644 index 02b915b..0000000 --- a/.idea/git_toolbox_prj.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt index 7590b44..bdfa0f8 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt @@ -359,7 +359,7 @@ object AgentController : AgentHost { } fun denyUnattendedConsent() { - val tunnel = meshAgent?.tunnels?.getOrNull(0) ?: return + val tunnel = activeDesktopTunnel() ?: return val json = JSONObject() json.put("type", "console") json.put("msg", "denied") @@ -399,6 +399,11 @@ object AgentController : AgentHost { return agent.tunnels.any { (it.state == 2) && (it.usage == 2) } } + // The connected desktop tunnel, used to route consent responses to the viewer that opened it. + fun activeDesktopTunnel(): MeshTunnel? { + return meshAgent?.tunnels?.toList()?.firstOrNull { (it.state == 2) && (it.usage == 2) } + } + // Display names of the remote users with an active session (desktop or files), de-duplicated. // File-transfer sub-tunnels (usage 10) are ignored so downloads don't flicker the notification. fun activeSessionUsers(): List { @@ -489,9 +494,9 @@ object AgentController : AgentHost { fun sendDesktopTunnelData(data: ByteString) { val agent = meshAgent ?: return - for (t in agent.tunnels) { - if ((t.state == 2) && (t.usage == 2) && (t._webSocket != null)) { - t._webSocket!!.send(data) + for (t in agent.tunnels.toList()) { + if ((t.state == 2) && (t.usage == 2)) { + t._webSocket?.send(data) } } } diff --git a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt index 3603378..e1d9c9e 100644 --- a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt +++ b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt @@ -14,6 +14,8 @@ class DesktopFrameEncoder { private var tilesCount: Int = 0 private var oldcrcs: IntArray? = null private var newcrcs: IntArray? = null + // Reused per-tile pixel scratch, so a full-screen CRC pass doesn't allocate one array per tile. + private val tilePixels = IntArray(64 * 64) // Written from the tunnel/main thread, read on the capture thread. @Volatile private var forceFullFrame = true @@ -113,9 +115,11 @@ class DesktopFrameEncoder { var w = 64 if (((x * 64) + 64) > bitmap.width) w = bitmap.width - (x * 64) val t = (y * tilesWide) + x - val pixels = IntArray(w * h) - bitmap.getPixels(pixels, 0, w, x * 64, y * 64, w, h) - for (pixel in pixels) newcrcs!![t] = adler32(pixel, newcrcs!![t]) + val count = w * h + bitmap.getPixels(tilePixels, 0, w, x * 64, y * 64, w, h) + var crc = newcrcs!![t] + for (i in 0 until count) crc = adler32(tilePixels[i], crc) + newcrcs!![t] = crc } } } diff --git a/app/src/main/java/com/meshcentral/agent/MainActivity.kt b/app/src/main/java/com/meshcentral/agent/MainActivity.kt index b972664..a830af7 100644 --- a/app/src/main/java/com/meshcentral/agent/MainActivity.kt +++ b/app/src/main/java/com/meshcentral/agent/MainActivity.kt @@ -91,13 +91,13 @@ class MainActivity : AppCompatActivity() { ) { result -> if (result.resultCode == RESULT_OK) { ContextCompat.startForegroundService(this, ScreenCaptureService.getStartIntent(this, result.resultCode, result.data)) - meshAgent?.tunnels?.getOrNull(0)?.sendCtrlResponse(JSONObject().apply { + AgentController.activeDesktopTunnel()?.sendCtrlResponse(JSONObject().apply { put("type", "console") put("msg", null) put("msgid", 0) }) } else { - meshAgent?.tunnels?.getOrNull(0)?.let { tunnel -> + AgentController.activeDesktopTunnel()?.let { tunnel -> tunnel.sendCtrlResponse(JSONObject().apply { put("type", "console") put("msg", "denied") @@ -371,6 +371,7 @@ class MainActivity : AppCompatActivity() { alert = null } this.runOnUiThread { + if (isFinishing || isDestroyed) return@runOnUiThread val builder = AlertDialog.Builder(this) builder.setTitle(title) builder.setMessage(msg) @@ -648,13 +649,7 @@ class MainActivity : AppCompatActivity() { } private fun sendDesktopConsentDenied() { - val tunnel = meshAgent?.tunnels?.getOrNull(0) ?: return - val json = JSONObject() - json.put("type", "console") - json.put("msg", "denied") - json.put("msgid", 2) - tunnel.sendCtrlResponse(json) - tunnel.Stop() + AgentController.denyUnattendedConsent() } fun stopProjection() { diff --git a/app/src/main/java/com/meshcentral/agent/MainFragment.kt b/app/src/main/java/com/meshcentral/agent/MainFragment.kt index bd1636d..8f339dd 100644 --- a/app/src/main/java/com/meshcentral/agent/MainFragment.kt +++ b/app/src/main/java/com/meshcentral/agent/MainFragment.kt @@ -1,6 +1,5 @@ package com.meshcentral.agent -import android.R.attr.* import android.annotation.SuppressLint import android.app.AlertDialog import android.content.res.Resources @@ -68,24 +67,24 @@ class MainFragment : Fragment() { } fun moveToScanner() { - println("moveToScanner $visibleScreen") - if (visibleScreen == 1) { findNavController().navigate(R.id.action_FirstFragment_to_SecondFragment) } + if (visibleScreen != 1) return + try { findNavController().navigate(R.id.action_FirstFragment_to_SecondFragment) } catch (ex: Exception) {} } @Suppress("UNUSED_PARAMETER") fun moveToWebPage(pageUrl: String) { - println("moveToWebPage $visibleScreen") - if (visibleScreen == 1) { findNavController().navigate(R.id.action_FirstFragment_to_webViewFragment) } + if (visibleScreen != 1) return + try { findNavController().navigate(R.id.action_FirstFragment_to_webViewFragment) } catch (ex: Exception) {} } fun moveToAuthPage() { - println("moveToAuthPage $visibleScreen") - if (visibleScreen == 1) { findNavController().navigate(R.id.action_FirstFragment_to_authFragment) } + if (visibleScreen != 1) return + try { findNavController().navigate(R.id.action_FirstFragment_to_authFragment) } catch (ex: Exception) {} } fun moveToSettingsPage() { - println("moveToSettingsPage $visibleScreen") - if (visibleScreen == 1) { findNavController().navigate(R.id.action_FirstFragment_to_settingsFragment) } + if (visibleScreen != 1) return + try { findNavController().navigate(R.id.action_FirstFragment_to_settingsFragment) } catch (ex: Exception) {} } private fun getStringEx(resId: Int) : String { diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt index 2ca0693..df8c71f 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -190,60 +190,69 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun captureFrame() { if (!active || capturing || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return capturing = true - takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, object : AccessibilityService.TakeScreenshotCallback { - override fun onSuccess(screenshot: AccessibilityService.ScreenshotResult) { - // Recovered: allow the next error to be reported again. - screenshotErrorNotified = false - try { - val wrapped = Bitmap.wrapHardwareBuffer(screenshot.hardwareBuffer, screenshot.colorSpace) - if (wrapped != null) { - val bitmap = wrapped.copy(Bitmap.Config.ARGB_8888, false) - val dimensionsChanged = lastWidth != bitmap.width || lastHeight != bitmap.height - lastWidth = bitmap.width - lastHeight = bitmap.height - if (dimensionsChanged) updateTunnelDisplaySize() - val encodedBitmap = if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { - Bitmap.createScaledBitmap( - bitmap, - max(1, (bitmap.width * g_desktop_scalingLevel) / 1024), - max(1, (bitmap.height * g_desktop_scalingLevel) / 1024), - false - ) - } else { - bitmap - } - val sentFrame = encoder.encode(encodedBitmap) { AgentController.sendDesktopTunnelData(it) } - nextFrameDelayMs = if (sentFrame) { - MIN_FRAME_DELAY_MS - } else { - min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) - } - if (encodedBitmap !== bitmap) encodedBitmap.recycle() - bitmap.recycle() - } - } catch (ex: Exception) { - if (!screenshotErrorNotified) { - screenshotErrorNotified = true - AgentController.sendDesktopMessage("Unable to capture unattended screenshot: ${ex.message}") - } - } finally { - screenshot.hardwareBuffer.close() - capturing = false - scheduleNextCapture() - } - } + try { + takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, screenshotCallback) + } catch (ex: Exception) { + capturing = false + scheduleNextCapture() + } + } - override fun onFailure(errorCode: Int) { - capturing = false - // Throttled or transient error; back off quietly instead of flooding the console. - nextFrameDelayMs = min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) - if (errorCode != AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT && !screenshotErrorNotified) { + private val screenshotCallback = object : AccessibilityService.TakeScreenshotCallback { + override fun onSuccess(screenshot: AccessibilityService.ScreenshotResult) { + // Recovered: allow the next error to be reported again. + screenshotErrorNotified = false + var bitmap: Bitmap? = null + var encodedBitmap: Bitmap? = null + try { + val wrapped = Bitmap.wrapHardwareBuffer(screenshot.hardwareBuffer, screenshot.colorSpace) + ?: return + bitmap = wrapped.copy(Bitmap.Config.ARGB_8888, false) + wrapped.recycle() + val dimensionsChanged = lastWidth != bitmap.width || lastHeight != bitmap.height + lastWidth = bitmap.width + lastHeight = bitmap.height + if (dimensionsChanged) updateTunnelDisplaySize() + encodedBitmap = if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { + Bitmap.createScaledBitmap( + bitmap, + max(1, (bitmap.width * g_desktop_scalingLevel) / 1024), + max(1, (bitmap.height * g_desktop_scalingLevel) / 1024), + false + ) + } else { + bitmap + } + val sentFrame = encoder.encode(encodedBitmap) { AgentController.sendDesktopTunnelData(it) } + nextFrameDelayMs = if (sentFrame) { + MIN_FRAME_DELAY_MS + } else { + min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) + } + } catch (ex: Throwable) { + if (!screenshotErrorNotified) { screenshotErrorNotified = true - AgentController.sendDesktopMessage("Unable to capture unattended screenshot, error $errorCode.") + AgentController.sendDesktopMessage("Unable to capture unattended screenshot: ${ex.message}") } + } finally { + if (encodedBitmap != null && encodedBitmap !== bitmap) encodedBitmap.recycle() + bitmap?.recycle() + screenshot.hardwareBuffer.close() + capturing = false scheduleNextCapture() } - }) + } + + override fun onFailure(errorCode: Int) { + capturing = false + // Throttled or transient error; back off quietly instead of flooding the console. + nextFrameDelayMs = min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) + if (errorCode != AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT && !screenshotErrorNotified) { + screenshotErrorNotified = true + AgentController.sendDesktopMessage("Unable to capture unattended screenshot, error $errorCode.") + } + scheduleNextCapture() + } } private fun scheduleNextCapture() { diff --git a/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt b/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt index 70bad1b..ba26b76 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshFirebaseMessagingService.kt @@ -49,8 +49,8 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { if ((remoteMessage.data["shash"] == null) || (serverLink == null) || (remoteMessage.data["shash"]!!.length < 12)) return; // Check the server's agent hash against the notification. - var x : List = serverLink!!.split(',') - if (!x[1].startsWith(remoteMessage.data["shash"]!!)) return; + val agentHash = serverLink!!.split(',').getOrNull(1) ?: return + if (!agentHash.startsWith(remoteMessage.data["shash"]!!)) return; // Get the notification URL if one is present var url : String? = null @@ -86,7 +86,8 @@ class MeshFirebaseMessagingService : FirebaseMessagingService() { var cmd : String? = remoteMessage.data["con"] var session : String? = remoteMessage.data["s"] var relayId : String? = remoteMessage.data["r"] - if ((cmd != null) && (session != null)) { processConsoleMessage(cmd, session, relayId, remoteMessage.from!!) } + val from = remoteMessage.from + if ((cmd != null) && (session != null) && (from != null)) { processConsoleMessage(cmd, session, relayId, from) } } } diff --git a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt index 418f26b..389304a 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt @@ -149,6 +149,11 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo _webSocket = null } catch (ex: Exception) { } } + // Close any in-flight upload so we don't leak the descriptor or leave a partial file + if (fileUpload != null) { + try { fileUpload?.close() } catch (ex: Exception) { } + fileUpload = null + } // Clear the connection timer if (connectionTimer != null) { connectionTimer?.cancel() @@ -195,13 +200,15 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo var type = json.optString("type") if (type == "options") { tunnelOptions = json } } else { - var xusage = text.toInt() + val xusage = text.toIntOrNull() ?: run { println("Invalid usage $text"); stopSocket(); return } if (((xusage < 1) || (xusage > 5)) && (xusage != 10)) { println("Invalid usage $text"); stopSocket(); return } - val serverExpectedUsage = if (serverData.has("usage")) serverData.getInt("usage") else null - if (!isTunnelUsageAllowed(serverExpectedUsage, xusage)) { - println("Unexpected usage $text != $serverExpectedUsage"); + val allowedUsages = serverData.optJSONObject("soptions")?.optJSONArray("usages")?.let { arr -> + List(arr.length()) { arr.getInt(it) } + } + if (!isTunnelUsageAllowed(allowedUsages, xusage)) { + println("Unexpected usage $text, allowed $allowedUsages"); stopSocket(); return } usage = xusage; // 2 = Desktop, 5 = Files, 10 = File transfer @@ -216,30 +223,27 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo if (usage == 2) { // If this is a remote desktop usage... if (!g_autoConsent && !AgentController.isRemoteDesktopRunning()) { - // asking for consent - if (meshAgent?.tunnels?.getOrNull(0) != null) { - val json = JSONObject() - val msg = if (!AgentController.isAccessibilityServiceEnabled() && g_mainActivity == null) { - "Open the Android app to approve screen capture, or enable Accessibility Remote Control for unattended desktop." - } else { - "Waiting for user to grant access..." - } - json.put("type", "console") - json.put("msg", msg) - json.put("msgid", 1) - meshAgent!!.tunnels[0].sendCtrlResponse(json) + // Ask for consent over this desktop tunnel, so the response reaches the + // viewer that opened it rather than whichever tunnel happens to be first. + val json = JSONObject() + val msg = if (!AgentController.isAccessibilityServiceEnabled() && g_mainActivity == null) { + "Open the Android app to approve screen capture, or enable Accessibility Remote Control for unattended desktop." + } else { + "Waiting for user to grant access..." } + json.put("type", "console") + json.put("msg", msg) + json.put("msgid", 1) + sendCtrlResponse(json) } if (!AgentController.isRemoteDesktopRunning()) { parent.parent.startProjection() } else { - if (meshAgent?.tunnels?.getOrNull(0) != null) { - val json = JSONObject() - json.put("type", "console") - json.put("msg", null) - json.put("msgid", 0) - meshAgent!!.tunnels[0].sendCtrlResponse(json) - } + val json = JSONObject() + json.put("type", "console") + json.put("msg", null) + json.put("msgid", 0) + sendCtrlResponse(json) // Send the display size and push a full frame of the current screen so the // reconnecting viewer sees it immediately instead of waiting for a change. updateDesktopDisplaySize() @@ -381,7 +385,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo writeShort(mHeight) // Height } } - _webSocket!!.send(bytesOut.toByteArray().toByteString()) + _webSocket?.send(bytesOut.toByteArray().toByteString()) } // Cause some data to be sent over the websocket control channel every 2 minutes to keep it open @@ -762,7 +766,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo if (len <= 0) { stopSocket(); break; } // Stream is done if (_webSocket == null) { stopSocket(); break; } // Web socket closed _webSocket?.send(buf.toByteString(0, len)) - if (_webSocket?.queueSize()!! > 655350) { Thread.sleep(100)} + if ((_webSocket?.queueSize() ?: 0) > 655350) { Thread.sleep(100)} } } return; @@ -809,7 +813,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo if (len <= 0) { stopSocket(); break; } // Stream is done if (_webSocket == null) { stopSocket(); break; } // Web socket closed _webSocket?.send(buf.toByteString(0, len)) - if (_webSocket?.queueSize()!! > 655350) { Thread.sleep(100)} + if ((_webSocket?.queueSize() ?: 0) > 655350) { Thread.sleep(100)} } } return; diff --git a/app/src/main/java/com/meshcentral/agent/ProtocolValidation.kt b/app/src/main/java/com/meshcentral/agent/ProtocolValidation.kt index f786fee..e2010fa 100644 --- a/app/src/main/java/com/meshcentral/agent/ProtocolValidation.kt +++ b/app/src/main/java/com/meshcentral/agent/ProtocolValidation.kt @@ -13,8 +13,11 @@ internal fun isMeshServerLinkValid(value: String): Boolean { parts[2].length >= 3 } -internal fun isTunnelUsageAllowed(expectedUsage: Int?, actualUsage: Int): Boolean { - return expectedUsage == null || expectedUsage == actualUsage +// The server restricts what a relay connection may do through soptions.usages, a list of allowed +// protocol numbers derived from the session's rights. It's only sent for limited sessions (e.g. +// guest shares); an absent list means unrestricted, matching meshcore.js onTunnelData. +internal fun isTunnelUsageAllowed(allowedUsages: List?, actualUsage: Int): Boolean { + return allowedUsages == null || allowedUsages.contains(actualUsage) } internal fun isSafeFileName(name: String): Boolean { diff --git a/app/src/main/java/com/meshcentral/agent/ScannerFragment.kt b/app/src/main/java/com/meshcentral/agent/ScannerFragment.kt index 5b1bd24..1e31053 100644 --- a/app/src/main/java/com/meshcentral/agent/ScannerFragment.kt +++ b/app/src/main/java/com/meshcentral/agent/ScannerFragment.kt @@ -23,7 +23,6 @@ import com.karumi.dexter.listener.PermissionGrantedResponse import com.karumi.dexter.listener.PermissionRequest import com.karumi.dexter.listener.single.PermissionListener import com.google.zxing.BarcodeFormat -import java.util.jar.Manifest /** * A simple [Fragment] subclass as the second destination in the navigation. diff --git a/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt b/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt index eb2b28d..b553596 100644 --- a/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt +++ b/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt @@ -133,7 +133,10 @@ class ScreenCaptureService : Service(), RemoteDesktopProvider { try { // Clean up if (mVirtualDisplay != null) mVirtualDisplay!!.release() - if (mImageReader != null) mImageReader!!.setOnImageAvailableListener(null, null) + if (mImageReader != null) { + mImageReader!!.setOnImageAvailableListener(null, null) + mImageReader!!.close() + } // Re-create virtual display depending on device width / height createVirtualDisplay() @@ -418,6 +421,7 @@ class ScreenCaptureService : Service(), RemoteDesktopProvider { mVirtualDisplay?.release() mVirtualDisplay = null mImageReader?.setOnImageAvailableListener(null, null) + mImageReader?.close() mImageReader = null if (mMediaProjection != null) createVirtualDisplay() } catch (e: Exception) { diff --git a/app/src/main/java/com/meshcentral/agent/WebViewFragment.kt b/app/src/main/java/com/meshcentral/agent/WebViewFragment.kt index 7ae29ac..577eb36 100644 --- a/app/src/main/java/com/meshcentral/agent/WebViewFragment.kt +++ b/app/src/main/java/com/meshcentral/agent/WebViewFragment.kt @@ -38,15 +38,17 @@ class WebViewFragment : Fragment() { visibleScreen = 3; browser = view.findViewById(R.id.mainWebView) as WebView browser?.settings?.javaScriptEnabled = true + browser?.settings?.allowFileAccess = false + browser?.settings?.allowContentAccess = false browser?.webViewClient = object : WebViewClient(){ @Suppress("OVERRIDE_DEPRECATION", "DEPRECATION") override fun shouldOverrideUrlLoading( view: WebView?, url: String? ): Boolean { - //println("shouldOverrideUrlLoading: $url") - pageUrl = url; - view?.loadUrl(url!!) + if (url == null) return false + pageUrl = url + view?.loadUrl(url) return true } @@ -73,7 +75,7 @@ class WebViewFragment : Fragment() { } */ } - browser?.loadUrl(pageUrl!!) + pageUrl?.let { browser?.loadUrl(it) } } fun navigate(url: String) { diff --git a/app/src/test/java/com/meshcentral/agent/ProtocolValidationTest.kt b/app/src/test/java/com/meshcentral/agent/ProtocolValidationTest.kt index c36cf55..63c9b82 100644 --- a/app/src/test/java/com/meshcentral/agent/ProtocolValidationTest.kt +++ b/app/src/test/java/com/meshcentral/agent/ProtocolValidationTest.kt @@ -22,14 +22,16 @@ class ProtocolValidationTest { } @Test - fun acceptsAbsentOrMatchingTunnelUsage() { + fun acceptsAbsentOrAllowedTunnelUsage() { assertTrue(isTunnelUsageAllowed(null, 2)) - assertTrue(isTunnelUsageAllowed(5, 5)) + assertTrue(isTunnelUsageAllowed(listOf(5, 10), 5)) + assertTrue(isTunnelUsageAllowed(listOf(1, 6, 8, 9, 2), 2)) } @Test - fun rejectsMismatchedTunnelUsage() { - assertFalse(isTunnelUsageAllowed(2, 5)) + fun rejectsUsageOutsideAllowedList() { + assertFalse(isTunnelUsageAllowed(listOf(5, 10), 2)) + assertFalse(isTunnelUsageAllowed(emptyList(), 2)) } @Test @@ -52,4 +54,22 @@ class ProtocolValidationTest { assertNull(resolveSdcardChild(root, "Sdcard/Pictures", "nested/photo.jpg")) assertFalse(isSafeFileName("..")) } + + @Test + fun rejectsSiblingDirectorySharingRootPrefix() { + val root = File("build/test-sdcard").canonicalFile + + // Escapes to a sibling whose path shares the root's string prefix; only the trailing + // separator in the containment check keeps this from passing. + assertNull(resolveSdcardPath(root, "Sdcard/../test-sdcardEvil/photo.jpg")) + } + + @Test + fun validatesSafeFileNames() { + assertTrue(isSafeFileName("photo.jpg")) + assertFalse(isSafeFileName("")) + assertFalse(isSafeFileName(".")) + assertFalse(isSafeFileName("dir/photo.jpg")) + assertFalse(isSafeFileName("photo\u0000.jpg")) + } } \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 576093f..d8348ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,8 +9,9 @@ requests. The Android agent is a native Kotlin application and is separate from the MeshCentral agents for Windows, Linux, macOS, and FreeBSD. Remote desktop is -currently **view only**: an operator can see the shared display, but cannot tap, -swipe, type, or otherwise control the Android device. +**view only** through Android's screen-capture path; when the user enables the +bundled accessibility service, an operator can also tap, swipe, scroll, and send +keys for unattended control. ## Get MeshAgent diff --git a/docs/overview.md b/docs/overview.md index a22326e..d3d06e5 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -23,9 +23,11 @@ are: - Approving or rejecting MeshCentral push-based two-factor authentication requests. -The remote desktop implementation is currently **view only**. Protocol handlers -for keyboard, mouse, Unicode key, and input-lock messages exist, but they are -no-ops. This app does not currently provide general remote input control. +Remote desktop is **view only** through the MediaProjection screen-capture path: +the keyboard, mouse, and Unicode-key handlers are no-ops there. When the user +enables the bundled `MeshAccessibilityService`, those messages are injected as +accessibility gestures and key events for unattended control; input-lock remains +a no-op. ## Project Snapshot @@ -36,11 +38,11 @@ no-ops. This app does not currently provide general remote input control. | Application ID | `com.meshcentral.agent2` | | Kotlin namespace | `com.meshcentral.agent` | | Minimum Android SDK | 23 (Android 6.0) | -| Compile/target SDK | 35 (Android 15) | +| Compile/target SDK | 37 | | Version | `1.0.23` (`versionCode` 30) | -| Kotlin | 1.9.10 | -| Android Gradle Plugin | 8.6.1 | -| Gradle wrapper | 8.7 | +| Kotlin | Bundled with the Android Gradle Plugin | +| Android Gradle Plugin | 9.3.1 | +| Gradle wrapper | 9.5.0 | | Java/Kotlin target | JVM 17 | The package namespace and installed application ID intentionally differ in the @@ -291,11 +293,9 @@ app/ ## Building and Verification -Use JDK 17 or Android Studio's bundled JDK 21 for the current Android Gradle -Plugin 8.6.1 and Gradle 8.7 combination. Confirm that `JAVA_HOME` and -`java -version` select one of those JDKs before building. Java 24 is not -supported by this wrapper and fails during Gradle settings evaluation with -`Unsupported class file major version 68`. +Build with JDK 17 or newer for the current Android Gradle Plugin 9.3.1 and +Gradle 9.5 combination. Confirm that `JAVA_HOME` and `java -version` select a +supported JDK before building. From the repository root on Windows: diff --git a/docs/remote-desktop.md b/docs/remote-desktop.md index 810f8e6..7dacb92 100644 --- a/docs/remote-desktop.md +++ b/docs/remote-desktop.md @@ -6,9 +6,10 @@ non-root Android application. ## Summary -Remote desktop is currently **screen sharing only**. A MeshCentral operator can -see the device display, but cannot tap, swipe, type, press navigation buttons, -lock input, or otherwise control the device. +Remote desktop is **screen sharing only** through the MediaProjection path: a +MeshCentral operator sees the device display but cannot control it. Enabling the +bundled accessibility service adds tap, swipe, scroll, and key input for +unattended control, within the limits described under Non-Root Limitations. The agent uses Android's public [MediaProjection API](https://developer.android.com/media/grow/media-projection) @@ -106,24 +107,21 @@ restarts. ## Non-Root Limitations -### No remote input +### Remote input requires the accessibility service -Android does not let an ordinary application inject arbitrary touch or keyboard -events into other applications. The MeshCentral protocol messages for legacy -keys, mouse input, Unicode keys, pause, refresh, and input lock are recognized -by `MeshTunnel`, but their handlers intentionally do nothing. +Android does not let an ordinary application inject touch or keyboard events into +other applications. On the plain MediaProjection screen share the MeshCentral +mouse, touch, and key messages are recognized by `MeshTunnel` but do nothing, so +that path is view-only. -The app does not declare an `AccessibilityService`, is not a system-signed app, -and does not use a rooted input-injection mechanism. As a result, the remote -desktop is view-only. - -An accessibility service could implement a limited set of gestures and global -actions after the device user explicitly enables it in Android settings. That -would still not be equivalent to root-level input: support varies by Android -version and device vendor, some screens reject accessibility actions, text and -key handling are incomplete, and Android displays persistent privacy indicators -and warnings. Accessibility must not be enabled or treated as a way to bypass -user consent. +When the device user explicitly enables the bundled `MeshAccessibilityService` +in Android settings, those messages are injected as tap, long-press, swipe, and +scroll gestures and key events, giving unattended control. This is not equivalent +to root-level input: support varies by Android version and device vendor, some +screens reject accessibility gestures, text and key handling are incomplete, +`FLAG_SECURE` windows still capture blank, and Android shows persistent privacy +indicators. The service is opt-in and must not be treated as a way to bypass user +consent. ### Protected content may be blank @@ -177,7 +175,7 @@ inspection and support rather than smooth video playback. | Multiple viewers | Supported; frames are broadcast to active desktop tunnels | Device and network load | | Rotation | Supported by recreating the virtual display | Brief update interruption | | Quality and scaling | Supported | Server settings and device cost | -| Remote tap, swipe, or typing | Not supported | No input implementation or privileged injection access | +| Remote tap, swipe, or typing | Supported with the accessibility service enabled; otherwise a no-op | Requires user-enabled `MeshAccessibilityService` | | Secure or DRM content | Not capturable | Android secure-surface policy | | Silent capture after restart | Not supported | MediaProjection authorization and lifecycle rules | | Hide sharing notification | Not supported | Foreground-service requirement | From 6243b55acd1ba69497c0dda37ff679eaae21a8cb Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Sat, 29 Aug 2026 07:19:18 +1000 Subject: [PATCH 6/9] Improve system bar handling and AppBar styling - Add `android:fitsSystemWindows="true"` to adjust layout for system bars. - Set `app:statusBarForeground` in `AppBarLayout` for better visual consistency. --- app/src/main/res/layout/activity_main.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index cf67a59..12184c3 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -5,11 +5,13 @@ xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" + android:fitsSystemWindows="true" tools:context=".MainActivity"> Date: Sat, 29 Aug 2026 09:45:36 +1000 Subject: [PATCH 7/9] Refactor accessibility service for enhanced input handling and reliable gesture execution - Add drag gesture support with motion path tracking and duration-based gestures. - Introduce gesture queueing mechanism to ensure sequential input handling. - Optimise screenshot capture timing with dynamic wake handling and backoff adjustments. - Enhance text input handling with precise cursor control and focused field management. - Refine global action handling and keyboard interaction for seamless remote desktop experience. --- .../agent/MeshAccessibilityService.kt | 285 +++++++++++++++--- .../java/com/meshcentral/agent/MeshTunnel.kt | 3 +- 2 files changed, 240 insertions(+), 48 deletions(-) diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt index df8c71f..a3d0c66 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -8,6 +8,7 @@ import android.os.Build import android.os.Bundle import android.os.Handler import android.os.Looper +import android.os.SystemClock import android.view.Display import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo @@ -30,9 +31,16 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { @Volatile private var lastHeight = 0 private var pointerDownX: Int? = null private var pointerDownY: Int? = null + // In-progress drag: path points and start time, so the gesture follows real motion. + private var dragPoints: ArrayList? = null + private var dragStartUptimeMs = 0L private var unsupportedKeyboardNotified = false @Volatile private var nextFrameDelayMs = MIN_FRAME_DELAY_MS @Volatile private var screenshotErrorNotified = false + @Volatile private var lastCaptureUptimeMs = 0L + // Accessibility runs one gesture at a time; queue them so quick taps aren't dropped. + private val gestureQueue = ArrayDeque() + private var gestureInFlight = false override val isRunning: Boolean get() = active @@ -62,7 +70,8 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { override fun onAccessibilityEvent(event: AccessibilityEvent?) { if (!active) return - nextFrameDelayMs = MIN_FRAME_DELAY_MS + // A visible change just happened; pull the next capture forward instead of waiting out the backoff. + wakeCapture() } override fun onInterrupt() { @@ -98,8 +107,8 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { } override fun requestFullFrame() { - nextFrameDelayMs = MIN_FRAME_DELAY_MS encoder.requestFullFrame() + wakeCapture() } override fun handleMouseCommand(msg: ByteString): Boolean { @@ -128,26 +137,78 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { true } flags == 0x02 || flags == 0x08 || flags == 0x20 -> { - pointerDownX = x - pointerDownY = y + beginDrag(x, y) + true + } + // Move with a button held = drag; no button = hover. + flags == 0x00 -> { + extendDrag(x, y) true } flags == 0x04 || flags == 0x10 || flags == 0x40 -> { - val startX = pointerDownX ?: x - val startY = pointerDownY ?: y - pointerDownX = null - pointerDownY = null - if ((startX - x).absoluteValue < 8 && (startY - y).absoluteValue < 8) { - dispatchTap(x, y) - } else { - dispatchSwipe(startX, startY, x, y, 350) - } + endDrag(x, y) true } else -> true } } + private fun beginDrag(x: Int, y: Int) { + pointerDownX = x + pointerDownY = y + dragStartUptimeMs = SystemClock.uptimeMillis() + dragPoints = arrayListOf(x.toFloat(), y.toFloat()) + } + + private fun extendDrag(x: Int, y: Int) { + val pts = dragPoints ?: return + val n = pts.size + if (n >= 2 && pts[n - 2] == x.toFloat() && pts[n - 1] == y.toFloat()) return + if (pts.size < MAX_DRAG_POINTS * 2) { + pts.add(x.toFloat()) + pts.add(y.toFloat()) + } + } + + private fun endDrag(x: Int, y: Int) { + val startX = pointerDownX ?: x + val startY = pointerDownY ?: y + val pts = dragPoints + pointerDownX = null + pointerDownY = null + dragPoints = null + val moved = (startX - x).absoluteValue >= 8 || (startY - y).absoluteValue >= 8 + if (!moved || pts == null) { + dispatchTap(x, y) + return + } + pts.add(x.toFloat()) + pts.add(y.toFloat()) + // Real hold time drives gesture duration, so a flick stays a flick. + val elapsed = (SystemClock.uptimeMillis() - dragStartUptimeMs) + .coerceIn(MIN_DRAG_DURATION_MS, MAX_DRAG_DURATION_MS) + dispatchDrag(pts, elapsed) + } + + private fun dispatchDrag(pts: List, durationMs: Long) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || pts.size < 4) return + val path = Path() + path.moveTo(pts[0], pts[1]) + var i = 2 + while (i + 1 < pts.size) { + path.lineTo(pts[i], pts[i + 1]) + i += 2 + } + val gesture = try { + GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs)) + .build() + } catch (ex: Exception) { + return + } + dispatchGestureQueued(gesture) + } + override fun handleTouchCommand(msg: ByteString): Boolean { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || msg.size < 14 || u(msg[4]) != 1) return false val flags = readInt(msg, 6) @@ -180,16 +241,19 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { } override fun handleKeyCommand(cmd: Int, msg: ByteString): Boolean { - return when (cmd) { + val handled = when (cmd) { 1 -> handleLegacyKey(msg) 85 -> handleUnicodeKey(msg) else -> false } + if (handled) wakeCapture() + return handled } private fun captureFrame() { if (!active || capturing || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return capturing = true + lastCaptureUptimeMs = SystemClock.uptimeMillis() try { takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, screenshotCallback) } catch (ex: Exception) { @@ -245,9 +309,14 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { override fun onFailure(errorCode: Int) { capturing = false - // Throttled or transient error; back off quietly instead of flooding the console. + if (errorCode == AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT) { + // We asked too soon; retry at the throttle interval rather than backing off toward idle. + scheduleNextCapture() + return + } + // Transient error; back off quietly instead of flooding the console. nextFrameDelayMs = min(nextFrameDelayMs * 2, MAX_IDLE_FRAME_DELAY_MS) - if (errorCode != AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT && !screenshotErrorNotified) { + if (!screenshotErrorNotified) { screenshotErrorNotified = true AgentController.sendDesktopMessage("Unable to capture unattended screenshot, error $errorCode.") } @@ -263,51 +332,170 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { mainHandler.postDelayed(captureRunnable, delay) } + // Pull the next capture forward on activity so an idle-backed-off loop isn't stuck on a stale frame. + private fun wakeCapture() { + if (!active) return + nextFrameDelayMs = MIN_FRAME_DELAY_MS + if (capturing) return + val sinceLast = SystemClock.uptimeMillis() - lastCaptureUptimeMs + val delay = max(0L, MIN_SCREENSHOT_INTERVAL_MS - sinceLast) + mainHandler.removeCallbacks(captureRunnable) + mainHandler.postDelayed(captureRunnable, delay) + } + + // dispatchGesture drops overlapping gestures, so serialize them; quick taps aren't lost. + private fun dispatchGestureQueued(gesture: GestureDescription) { + wakeCapture() + mainHandler.post { + if (gestureQueue.size >= MAX_QUEUED_GESTURES) gestureQueue.removeFirst() + gestureQueue.addLast(gesture) + pumpGestures() + } + } + + private fun pumpGestures() { + if (gestureInFlight) return + val gesture = gestureQueue.removeFirstOrNull() ?: return + gestureInFlight = true + dispatchGesture(gesture, object : AccessibilityService.GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription?) { + gestureInFlight = false + pumpGestures() + } + override fun onCancelled(gestureDescription: GestureDescription?) { + gestureInFlight = false + pumpGestures() + } + }, mainHandler) + } + private fun handleLegacyKey(msg: ByteString): Boolean { if (msg.size < 6) return false val action = u(msg[4]) val keyCode = u(msg[5]) if (action != 0) return true when (keyCode) { - 8 -> return editFocusedText { if (it.isNotEmpty()) it.dropLast(1) else it } - 13 -> return editFocusedText { "$it\n" } - 27 -> { - performGlobalAction(GLOBAL_ACTION_BACK) - return true - } - 36 -> { - performGlobalAction(GLOBAL_ACTION_HOME) - return true - } - 93 -> { - performGlobalAction(GLOBAL_ACTION_RECENTS) - return true - } + 8 -> return backspaceFocused() + 13 -> return insertFocused("\n") + 37 -> return moveFocusedCursor(-1) + 39 -> return moveFocusedCursor(1) + 38 -> return moveFocusedCursorLine(-1) + 40 -> return moveFocusedCursorLine(1) + 27 -> return globalAction(GLOBAL_ACTION_BACK) + 36 -> return globalAction(GLOBAL_ACTION_HOME) + 93 -> return globalAction(GLOBAL_ACTION_RECENTS) + // Codes above the keyboard range are the desktop panel's Android action buttons. + 200 -> return globalAction(GLOBAL_ACTION_ACCESSIBILITY_ALL_APPS, Build.VERSION_CODES.S) // App drawer + 201 -> return globalAction(GLOBAL_ACTION_NOTIFICATIONS) + 202 -> return globalAction(GLOBAL_ACTION_QUICK_SETTINGS) + 203 -> return globalAction(GLOBAL_ACTION_LOCK_SCREEN, Build.VERSION_CODES.P) + 204 -> return globalAction(GLOBAL_ACTION_POWER_DIALOG) } notifyUnsupportedKeyboard() return false } + private fun globalAction(action: Int, minSdk: Int = 0): Boolean { + if (Build.VERSION.SDK_INT < minSdk) return false + performGlobalAction(action) + return true + } + private fun handleUnicodeKey(msg: ByteString): Boolean { if (msg.size < 7) return false - val action = u(msg[4]) - if (action != 0) return true + // Insert on key-up (action 1). The browser keypress that carries the key-down is deprecated and + // often doesn't fire, but key-up always does; the panel and physical typing both send an up. + if (u(msg[4]) != 1) return true val charCode = readShort(msg, 5) - val char = charCode.toChar().toString() - return editFocusedText { it + char }.also { + return insertFocused(charCode.toChar().toString()).also { if (!it) notifyUnsupportedKeyboard() } } - private fun editFocusedText(transform: (String) -> String): Boolean { - val node = rootInActiveWindow?.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: return false - if (!node.isEditable) return false + private fun focusedEditable(): AccessibilityNodeInfo? { + val node = rootInActiveWindow?.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: return null + return if (node.isEditable) node else null + } + + // A shown hint reads back as node text; treat it as empty so it isn't captured as real content. + private fun fieldText(node: AccessibilityNodeInfo): String { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && node.isShowingHintText) return "" + return node.text?.toString() ?: "" + } + + private fun selectionRange(node: AccessibilityNodeInfo, len: Int): Pair { + var s = node.textSelectionStart + var e = node.textSelectionEnd + if (s < 0 || s > len) s = len + if (e < 0 || e > len) e = len + return if (s <= e) Pair(s, e) else Pair(e, s) + } + + private fun setCursor(node: AccessibilityNodeInfo, pos: Int): Boolean { val args = Bundle() - args.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - transform(node.text?.toString() ?: "") - ) - return node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) + args.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, pos) + args.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, pos) + return node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, args) + } + + private fun replaceText(node: AccessibilityNodeInfo, text: String, cursor: Int): Boolean { + val args = Bundle() + args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) + if (!node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)) return false + setCursor(node, cursor) + return true + } + + private fun insertFocused(insert: String): Boolean { + val node = focusedEditable() ?: return false + val text = fieldText(node) + val (s, e) = selectionRange(node, text.length) + return replaceText(node, text.substring(0, s) + insert + text.substring(e), s + insert.length) + } + + private fun backspaceFocused(): Boolean { + val node = focusedEditable() ?: return false + val text = fieldText(node) + val (s, e) = selectionRange(node, text.length) + return when { + s != e -> replaceText(node, text.substring(0, s) + text.substring(e), s) + s > 0 -> replaceText(node, text.substring(0, s - 1) + text.substring(s), s - 1) + else -> true + } + } + + private fun moveFocusedCursor(delta: Int): Boolean { + val node = focusedEditable() ?: return true + val text = fieldText(node) + val (s, e) = selectionRange(node, text.length) + // A selection collapses to its near edge; otherwise step one character. + val pos = when { + s != e && delta < 0 -> s + s != e && delta > 0 -> e + else -> (e + delta).coerceIn(0, text.length) + } + return setCursor(node, pos) + } + + private fun moveFocusedCursorLine(dir: Int): Boolean { + val node = focusedEditable() ?: return true + val text = fieldText(node) + val (_, e) = selectionRange(node, text.length) + val lineStart = text.lastIndexOf('\n', (e - 1).coerceAtLeast(0)).let { if (it < 0) 0 else it + 1 } + val col = e - lineStart + val pos = if (dir < 0) { + if (lineStart == 0) 0 else { + val prevStart = text.lastIndexOf('\n', lineStart - 2).let { if (it < 0) 0 else it + 1 } + (prevStart + col).coerceAtMost(lineStart - 1) + } + } else { + val lineEnd = text.indexOf('\n', e) + if (lineEnd < 0) text.length else { + val nextEnd = text.indexOf('\n', lineEnd + 1).let { if (it < 0) text.length else it } + (lineEnd + 1 + col).coerceAtMost(nextEnd) + } + } + return setCursor(node, pos) } private fun notifyUnsupportedKeyboard() { @@ -323,7 +511,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { val gesture = GestureDescription.Builder() .addStroke(GestureDescription.StrokeDescription(path, 0, 80)) .build() - dispatchGesture(gesture, null, null) + dispatchGestureQueued(gesture) } private fun dispatchSwipe(startX: Int, startY: Int, endX: Int, endY: Int, duration: Long) { @@ -334,7 +522,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { val gesture = GestureDescription.Builder() .addStroke(GestureDescription.StrokeDescription(path, 0, duration)) .build() - dispatchGesture(gesture, null, null) + dispatchGestureQueued(gesture) } private fun updateTunnelDisplaySize() { @@ -368,8 +556,13 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { var instance: MeshAccessibilityService? = null private set private const val MIN_FRAME_DELAY_MS = 100L - private const val MAX_IDLE_FRAME_DELAY_MS = 10_000L - // System throttles takeScreenshot() faster than ~3 fps. + // Idle cap; activity wakes capture immediately, so this only bounds silent-change latency. + private const val MAX_IDLE_FRAME_DELAY_MS = 2_000L + // System throttles takeScreenshot() faster than ~3 fps (AOSP interval is 333ms). private const val MIN_SCREENSHOT_INTERVAL_MS = 350L + private const val MAX_QUEUED_GESTURES = 16 + private const val MAX_DRAG_POINTS = 64 + private const val MIN_DRAG_DURATION_MS = 40L + private const val MAX_DRAG_DURATION_MS = 1500L } } diff --git a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt index 291ba43..50bab1a 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt @@ -223,8 +223,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo if (usage == 2) { // If this is a remote desktop usage... if (!g_autoConsent && !AgentController.isRemoteDesktopRunning()) { - // Ask for consent over this desktop tunnel, so the response reaches the - // viewer that opened it rather than whichever tunnel happens to be first. + // Consent goes over this desktop tunnel so it reaches the viewer that opened it. val json = JSONObject() val msg = if (!AgentController.isAccessibilityServiceEnabled() && g_mainActivity == null) { "Open the Android app to approve screen capture, or enable Accessibility Remote Control for unattended desktop." From b34f7b8cae90ec5084b5e5da34004ef842c4990a Mon Sep 17 00:00:00 2001 From: Patrick O'Connell Date: Tue, 15 Sep 2026 22:41:29 +1000 Subject: [PATCH 8/9] Add comprehensive desktop input and storage handling for remote agent - Introduce `DesktopInput` for refined mouse input handling, including button presses, releases, movement, and double-click decoding. - Add `MeshHttp` for centralised HTTP client configuration with connection pool reuse. - Implement `UploadStorage` for file upload and shared storage management, with support for raw public file access and MediaStore integration. - Update `README.md` to document remote desktop capabilities and extended user consent handling. - Enhance accessibility service (`MeshAccessibilityService`) to support expanded input gestures, including taps, drags, and double-taps, with queued gesture processing for improved input fidelity. --- README.md | 15 +- app/build.gradle | 7 + app/src/main/AndroidManifest.xml | 17 + .../agent/AgentForegroundService.kt | 43 +- .../com/meshcentral/agent/AgentRuntime.kt | 215 ++++++- .../com/meshcentral/agent/DesktopInput.kt | 30 + .../com/meshcentral/agent/MainActivity.kt | 72 ++- .../agent/MeshAccessibilityService.kt | 569 ++++++++++++------ .../java/com/meshcentral/agent/MeshAgent.kt | 177 +++++- .../java/com/meshcentral/agent/MeshHttp.kt | 17 + .../java/com/meshcentral/agent/MeshTunnel.kt | 540 ++++++++++------- .../meshcentral/agent/ScreenCaptureService.kt | 7 + .../com/meshcentral/agent/SettingsFragment.kt | 27 + .../com/meshcentral/agent/UploadStorage.kt | 322 ++++++++++ app/src/main/res/values-de/strings.xml | 45 ++ app/src/main/res/values/strings.xml | 10 +- app/src/main/res/xml/root_preferences.xml | 4 + .../com/meshcentral/agent/DesktopInputTest.kt | 19 + .../meshcentral/agent/UploadStorageTest.kt | 46 ++ docs/index.md | 8 +- docs/overview.md | 70 ++- docs/remote-desktop.md | 60 +- docs/tunnel-authentication.md | 20 +- 23 files changed, 1836 insertions(+), 504 deletions(-) create mode 100644 app/src/main/java/com/meshcentral/agent/DesktopInput.kt create mode 100644 app/src/main/java/com/meshcentral/agent/MeshHttp.kt create mode 100644 app/src/main/java/com/meshcentral/agent/UploadStorage.kt create mode 100644 app/src/test/java/com/meshcentral/agent/DesktopInputTest.kt create mode 100644 app/src/test/java/com/meshcentral/agent/UploadStorageTest.kt diff --git a/README.md b/README.md index 8b871eb..1a1b7e0 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,20 @@ link, or entering the link manually. After enrollment, the app maintains an authenticated connection to the server and can: - Report device, network, storage, and battery information. -- Share the device screen after Android MediaProjection consent. +- Share the device screen, and control it once the bundled accessibility + service is enabled. - Browse and transfer media and files available to the app. - Receive server notifications and a limited set of console commands. - Approve or reject MeshCentral push-based two-factor authentication requests. -Remote desktop is currently **view only**. The app can stream the display, but -it cannot tap, swipe, type, or otherwise control the device. Android displays a -foreground notification while screen sharing is active, and the user can deny -or stop capture at any time. +Remote desktop works two ways. Without extra setup it is **view only** through +Android's screen-capture consent dialog. Once the device user enables the +bundled Accessibility Remote Control service, the agent captures the screen in +the background and injects taps, drags, long presses, scrolling and typing, so +an operator can control the device unattended. Android shows a persistent +notification while a session is active, consent prompts follow the server's +policy and the app's Automatic Consent setting, and the user can deny or stop +sharing at any time. ## Install diff --git a/app/build.gradle b/app/build.gradle index 5465a7b..6701c29 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -32,6 +32,13 @@ android { testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" def enterpriseEnforced = (project.findProperty("meshEnterpriseEnforced") ?: "false").toString().toBoolean() buildConfigField "boolean", "ENTERPRISE_ENFORCED", enterpriseEnforced.toString() + // Optional "All files access" (MANAGE_EXTERNAL_STORAGE) for full file transfer on Android 11+. + // Off by default: Google Play only accepts that permission with an approved use-case + // declaration, so declare it just for enterprise or sideloaded builds. When off, the + // placeholder resolves to a permission the manifest already has, so nothing new is added. + def allFilesAccess = (project.findProperty("meshAllFilesAccess") ?: "false").toString().toBoolean() + buildConfigField "boolean", "ALL_FILES_ACCESS", allFilesAccess.toString() + manifestPlaceholders["allFilesAccessPermission"] = allFilesAccess ? "android.permission.MANAGE_EXTERNAL_STORAGE" : "android.permission.INTERNET" } buildTypes { diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index cfd67c6..d23801f 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -14,13 +14,30 @@ + + + + + + + + + + + + + + + + + { + cancelFilesConsentNotification(this) + AgentController.confirmFilesConsent() + } + ACTION_DENY_FILES -> { + cancelFilesConsentNotification(this) + AgentController.denyFilesConsent() + } ACTION_STOP -> { if (!AgentController.enterpriseEnforced) { if (meshAgent != null) AgentController.toggleAgentConnection(true) @@ -140,11 +148,14 @@ class AgentForegroundService : Service() { private const val RUNTIME_NOTIFICATION_ID = 2402 private const val SESSION_NOTIFICATION_ID = 2403 private const val CONSENT_NOTIFICATION_ID = 2404 + private const val FILES_CONSENT_NOTIFICATION_ID = 2405 private const val ACTION_CONNECT = "com.meshcentral.agent.action.CONNECT" private const val ACTION_DISCONNECT = "com.meshcentral.agent.action.DISCONNECT" private const val ACTION_STOP_SCREEN_SHARING = "com.meshcentral.agent.action.STOP_SCREEN_SHARING" private const val ACTION_APPROVE_SCREEN_SHARING = "com.meshcentral.agent.action.APPROVE_SCREEN_SHARING" private const val ACTION_DENY_SCREEN_SHARING = "com.meshcentral.agent.action.DENY_SCREEN_SHARING" + private const val ACTION_APPROVE_FILES = "com.meshcentral.agent.action.APPROVE_FILES" + private const val ACTION_DENY_FILES = "com.meshcentral.agent.action.DENY_FILES" private const val ACTION_STOP = "com.meshcentral.agent.action.STOP" fun start(context: Context) { @@ -182,24 +193,33 @@ class AgentForegroundService : Service() { } // Consent request with Approve/Deny actions, for when the app isn't foregrounded. - fun showConsentNotification(context: Context) { + fun showConsentNotification(context: Context, message: String) { + showConsentNotification(context, CONSENT_NOTIFICATION_ID, context.getString(R.string.approve_screen_sharing_title), message, + servicePendingIntent(context, ACTION_APPROVE_SCREEN_SHARING, 4), servicePendingIntent(context, ACTION_DENY_SCREEN_SHARING, 5)) + } + + fun showFilesConsentNotification(context: Context, message: String) { + showConsentNotification(context, FILES_CONSENT_NOTIFICATION_ID, context.getString(R.string.approve_files_title), message, + servicePendingIntent(context, ACTION_APPROVE_FILES, 6), servicePendingIntent(context, ACTION_DENY_FILES, 7)) + } + + private fun showConsentNotification(context: Context, id: Int, title: String, message: String, approve: PendingIntent, deny: PendingIntent) { createConsentNotificationChannel(context) - val body = context.getString(R.string.approve_screen_sharing_body) val notification = NotificationCompat.Builder(context, CONSENT_CHANNEL_ID) .setSmallIcon(R.drawable.ic_cloud) - .setContentTitle(context.getString(R.string.approve_screen_sharing_title)) - .setContentText(body) - .setStyle(NotificationCompat.BigTextStyle().bigText(body)) + .setContentTitle(title) + .setContentText(message) + .setStyle(NotificationCompat.BigTextStyle().bigText(message)) .setContentIntent(openAppPendingIntent(context, null)) .setCategory(Notification.CATEGORY_CALL) .setPriority(NotificationCompat.PRIORITY_HIGH) .setOngoing(true) .setAutoCancel(false) - .addAction(R.drawable.ic_cloud, context.getString(R.string.approve), servicePendingIntent(context, ACTION_APPROVE_SCREEN_SHARING, 4)) - .addAction(R.drawable.ic_cloud, context.getString(R.string.deny), servicePendingIntent(context, ACTION_DENY_SCREEN_SHARING, 5)) + .addAction(R.drawable.ic_cloud, context.getString(R.string.approve), approve) + .addAction(R.drawable.ic_cloud, context.getString(R.string.deny), deny) .build() try { - NotificationManagerCompat.from(context).notify(CONSENT_NOTIFICATION_ID, notification) + NotificationManagerCompat.from(context).notify(id, notification) } catch (_: SecurityException) { } } @@ -211,6 +231,13 @@ class AgentForegroundService : Service() { } } + fun cancelFilesConsentNotification(context: Context) { + try { + NotificationManagerCompat.from(context).cancel(FILES_CONSENT_NOTIFICATION_ID) + } catch (_: SecurityException) { + } + } + private fun buildNotification(context: Context): Notification { val state = when (meshAgent?.state ?: 0) { 1 -> context.getString(R.string.connecting) diff --git a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt index bdfa0f8..57a688c 100644 --- a/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt +++ b/app/src/main/java/com/meshcentral/agent/AgentRuntime.kt @@ -13,9 +13,11 @@ import android.content.pm.PackageManager import android.net.Uri import android.os.Build import android.os.Bundle +import android.os.Environment import android.os.Handler import android.os.Looper import android.os.PowerManager +import android.os.SystemClock import android.provider.Settings import android.security.keystore.KeyGenParameterSpec import android.security.keystore.KeyProperties @@ -30,8 +32,6 @@ import androidx.lifecycle.Lifecycle import androidx.preference.PreferenceManager import com.google.firebase.messaging.FirebaseMessaging import okio.ByteString -import okio.ByteString.Companion.toByteString -import org.json.JSONObject import java.io.ByteArrayInputStream import java.math.BigInteger import java.security.KeyFactory @@ -83,6 +83,9 @@ object AgentController : AgentHost { private const val TAG = "AgentController" private const val INITIAL_RETRY_DELAY_MS = 10_000L private const val MAX_RETRY_DELAY_MS = 300_000L + private const val DESKTOP_NOTICE_TIMEOUT_SECONDS = 8 + private const val SCREEN_AWAKE_MS = 60_000L + private const val SCREEN_AWAKE_REFRESH_MS = 15_000L private const val ANDROID_KEYSTORE = "AndroidKeyStore" private const val AGENT_KEY_ALIAS = "meshcentral-agent-identity" private const val ONE_DAY_MILLIS = 24L * 60L * 60L * 1000L @@ -100,11 +103,30 @@ object AgentController : AgentHost { private var handlingSettingsChange = false private var projectionRetryRunnable: Runnable? = null private var projectionRetryCount = 0 + // Consent prompts expire like the other agents' do: the server's timeout, 30 s by default. + private var desktopConsentTimeout: Runnable? = null + private var filesConsentTimeout: Runnable? = null + private var screenWakeLock: PowerManager.WakeLock? = null + @Volatile private var screenAwakeUntilUptimeMs = 0L private val MAX_PROJECTION_RETRIES = 12 val enterpriseEnforced: Boolean get() = BuildConfig.ENTERPRISE_ENFORCED + // Only builds made with -PmeshAllFilesAccess=true declare the permission; the user still has + // to grant it in system settings. + val allFilesAccessAvailable: Boolean + get() = BuildConfig.ALL_FILES_ACCESS && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R + + fun hasAllFilesAccess(): Boolean { + if (!allFilesAccessAvailable || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return false + return try { + Environment.isExternalStorageManager() + } catch (ex: Exception) { + false + } + } + override val contentResolver: ContentResolver get() = appContext.contentResolver @@ -303,23 +325,27 @@ object AgentController : AgentHost { private fun startProjectionOnHostThread() { if (meshAgent == null || meshAgent?.state != 3) return if (!hasActiveDesktopTunnel()) return + keepScreenAwake() if (isRemoteDesktopRunning()) return val accessibility = MeshAccessibilityService.instance if (accessibility != null) { cancelProjectionRetry() - if (g_autoConsent) { + val tunnel = activeDesktopTunnel() + if (tunnel == null || !tunnel.consentPromptRequired()) { if (accessibility.startDesktop()) return } else { - // Automatic Consent off: require explicit approval before capturing. + // Explicit approval before capturing: the app setting or the server's policy asks for it. + val message = tunnel.consentMessage(appContext) val mainActivity = activity val resumed = mainActivity?.lifecycle?.currentState?.isAtLeast(Lifecycle.State.RESUMED) == true if (mainActivity != null && resumed) { - mainActivity.promptUnattendedConsent() + mainActivity.promptUnattendedConsent(message) } else { // No foreground activity to host a dialog, so ask via the notification. - sendDesktopMessage("Waiting for the device user to approve screen sharing.") - AgentForegroundService.showConsentNotification(appContext) + sendDesktopConsentPending() + AgentForegroundService.showConsentNotification(appContext, message) } + armDesktopConsentTimeout(tunnel) return } } else if (isAccessibilityServiceEnabled() && waitForAccessibilityProjection()) { @@ -343,7 +369,7 @@ object AgentController : AgentHost { return } - sendDesktopMessage("Remote desktop requires Accessibility unattended access or an open app screen for Android capture consent.") + sendDesktopMessage("Remote desktop requires Accessibility unattended access or an open app screen for Android capture consent.", timeoutSeconds = null) showToastMessage("Enable unattended access in settings to share the screen in the background.") showRuntimeNotification( appContext.getString(R.string.unattended_access_required), @@ -354,22 +380,166 @@ object AgentController : AgentHost { fun confirmUnattendedConsent() { if (::appContext.isInitialized) AgentForegroundService.cancelConsentNotification(appContext) + cancelDesktopConsentTimeout() if (meshAgent?.state != 3 || !hasActiveDesktopTunnel() || isRemoteDesktopRunning()) return + activeDesktopTunnel()?.let { + it.logSessionEvent(30, "Starting remote desktop after local user accepted") + it.notifySessionStart() + } runOnHostThread { MeshAccessibilityService.instance?.startDesktop() } } fun denyUnattendedConsent() { + if (::appContext.isInitialized) AgentForegroundService.cancelConsentNotification(appContext) + cancelDesktopConsentTimeout() + activity?.dismissConsentPrompt(MainActivity.CONSENT_DESKTOP) val tunnel = activeDesktopTunnel() ?: return - val json = JSONObject() - json.put("type", "console") - json.put("msg", "denied") - json.put("msgid", 2) - tunnel.sendCtrlResponse(json) + tunnel.logSessionEvent(34, "Failed to start remote desktop after local user rejected") + tunnel.sendConsoleMessage("denied", MeshTunnel.MSGID_CONSENT_DENIED) tunnel.Stop() } + private fun armDesktopConsentTimeout(tunnel: MeshTunnel) { + if (desktopConsentTimeout != null) return + val runnable = Runnable { + desktopConsentTimeout = null + if (tunnel.consentAutoAcceptOnTimeout()) confirmUnattendedConsent() else denyUnattendedConsent() + } + desktopConsentTimeout = runnable + mainHandler.postDelayed(runnable, tunnel.consentTimeoutMs()) + } + + private fun cancelDesktopConsentTimeout() { + desktopConsentTimeout?.let { mainHandler.removeCallbacks(it) } + desktopConsentTimeout = null + } + + // Files sessions: same approval flow as screen sharing, but nothing to capture, so the tunnel + // just holds the viewer's requests until the user answers. + fun requestFilesConsent(tunnel: MeshTunnel) { + runOnHostThread { + val message = tunnel.consentMessage(appContext) + val mainActivity = activity + val resumed = mainActivity?.lifecycle?.currentState?.isAtLeast(Lifecycle.State.RESUMED) == true + if (mainActivity != null && resumed) { + mainActivity.promptFilesConsent(message) + } else { + AgentForegroundService.showFilesConsentNotification(appContext, message) + } + if (filesConsentTimeout == null) { + val runnable = Runnable { + filesConsentTimeout = null + if (tunnel.consentAutoAcceptOnTimeout()) confirmFilesConsent() else denyFilesConsent() + } + filesConsentTimeout = runnable + mainHandler.postDelayed(runnable, tunnel.consentTimeoutMs()) + } + } + } + + // Re-shows the dialog after the activity comes back, e.g. the prompt was answered on the + // notification's screen or lost to a rotation. + fun showPendingFilesConsent() { + val tunnel = pendingFilesConsentTunnels().firstOrNull() ?: return + activity?.promptFilesConsent(tunnel.consentMessage(appContext)) + } + + fun confirmFilesConsent() { + runOnHostThread { + clearFilesConsentPrompt() + for (t in pendingFilesConsentTunnels()) t.approveFilesConsent() + refreshInfo() + } + } + + fun denyFilesConsent() { + runOnHostThread { + clearFilesConsentPrompt() + for (t in pendingFilesConsentTunnels()) t.denyFilesConsent() + } + } + + // The waiting tunnel went away on its own (viewer closed it), so drop the prompt. + fun filesConsentResolved() { + runOnHostThread { + if (pendingFilesConsentTunnels().isEmpty()) clearFilesConsentPrompt() + } + } + + private fun pendingFilesConsentTunnels(): List { + val agent = meshAgent ?: return emptyList() + return agent.tunnels.filter { (it.state == 2) && (it.usage == 5) && it.filesConsentPending } + } + + private fun clearFilesConsentPrompt() { + if (::appContext.isInitialized) AgentForegroundService.cancelFilesConsentNotification(appContext) + filesConsentTimeout?.let { mainHandler.removeCallbacks(it) } + filesConsentTimeout = null + activity?.dismissConsentPrompt(MainActivity.CONSENT_FILES) + } + + // A remote session needs the display on: wake it when a session starts and keep it on for a + // minute after each operator action, then let the device sleep as it normally would. A sleeping + // or dozing screen captures black and ignores injected touches. + @Suppress("DEPRECATION") + fun keepScreenAwake() { + if (!::appContext.isInitialized) return + val now = SystemClock.uptimeMillis() + if (now < screenAwakeUntilUptimeMs - SCREEN_AWAKE_MS + SCREEN_AWAKE_REFRESH_MS) return + screenAwakeUntilUptimeMs = now + SCREEN_AWAKE_MS + runOnHostThread { + try { + val lock = screenWakeLock ?: run { + val powerManager = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager + powerManager.newWakeLock( + PowerManager.SCREEN_BRIGHT_WAKE_LOCK or PowerManager.ACQUIRE_CAUSES_WAKEUP or PowerManager.ON_AFTER_RELEASE, + "MeshCentral:remoteSession" + ).also { + it.setReferenceCounted(false) + screenWakeLock = it + } + } + lock.acquire(SCREEN_AWAKE_MS) + } catch (ex: Exception) { + Log.w(TAG, "Unable to wake the screen", ex) + } + } + } + + private fun releaseScreenAwake() { + screenAwakeUntilUptimeMs = 0L + runOnHostThread { + try { + val lock = screenWakeLock + if (lock != null && lock.isHeld) lock.release() + } catch (ex: Exception) { + } + } + } + + // A capture provider just started: take the consent banner off every viewer of this device. + fun desktopProviderStarted() { + keepScreenAwake() + val agent = meshAgent ?: return + for (t in agent.tunnels.toList()) { + if ((t.state == 2) && (t.usage == 2)) t.sendConsoleMessage(null) + } + refreshInfo() + } + + private fun sendDesktopConsentPending() { + val agent = meshAgent ?: return + for (t in agent.tunnels.toList()) { + if ((t.state == 2) && (t.usage == 2)) { + t.sendConsoleMessage(MeshTunnel.CONSENT_PENDING_MESSAGE, MeshTunnel.MSGID_CONSENT_PENDING) + } + } + } + override fun stopProjection() { if (::appContext.isInitialized) AgentForegroundService.cancelConsentNotification(appContext) + cancelDesktopConsentTimeout() + releaseScreenAwake() val provider = g_remoteDesktopProvider if (provider is MeshAccessibilityService) { provider.stopDesktop() @@ -501,17 +671,18 @@ object AgentController : AgentHost { } } - fun sendDesktopMessage(message: String) { - val bytes = message.toByteArray(Charsets.UTF_8) - val data = ByteArray(4 + bytes.size) - data[1] = 17 - data[2] = ((data.size shr 8) and 0xFF).toByte() - data[3] = (data.size and 0xFF).toByte() - bytes.copyInto(data, 4) - sendDesktopTunnelData(data.toByteString()) + // Notice on every viewer's desktop overlay. The web UI never displays the binary KVM message + // command, so this goes over the control channel. A null timeout keeps it up until capture + // starts and clears it. + fun sendDesktopMessage(message: String, timeoutSeconds: Int? = DESKTOP_NOTICE_TIMEOUT_SECONDS) { + val agent = meshAgent ?: return + for (t in agent.tunnels.toList()) { + if ((t.state == 2) && (t.usage == 2)) t.sendConsoleMessage(message, timeoutSeconds = timeoutSeconds) + } } fun handleDesktopMouseCommand(msg: ByteString): Boolean { + keepScreenAwake() val provider = activeInputProvider() if (provider != null && provider.handleMouseCommand(msg)) return true sendDesktopMessage("Remote input requires Accessibility unattended access.") @@ -519,6 +690,7 @@ object AgentController : AgentHost { } fun handleDesktopTouchCommand(msg: ByteString): Boolean { + keepScreenAwake() val provider = activeInputProvider() if (provider != null && provider.handleTouchCommand(msg)) return true sendDesktopMessage("Remote touch input requires Accessibility unattended access.") @@ -526,6 +698,7 @@ object AgentController : AgentHost { } fun handleDesktopKeyCommand(cmd: Int, msg: ByteString): Boolean { + keepScreenAwake() val provider = activeInputProvider() if (provider != null && provider.handleKeyCommand(cmd, msg)) return true sendDesktopMessage("Remote keyboard input is limited on Android and requires Accessibility unattended access.") diff --git a/app/src/main/java/com/meshcentral/agent/DesktopInput.kt b/app/src/main/java/com/meshcentral/agent/DesktopInput.kt new file mode 100644 index 0000000..838b2ac --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/DesktopInput.kt @@ -0,0 +1,30 @@ +package com.meshcentral.agent + +// MeshCentral mouse message flags (byte 5). A button's up flag is its down flag doubled and +// 0x88 is the viewer's double-click event, which follows the two down/up pairs it already sent. +internal enum class MouseButton { LEFT, RIGHT, MIDDLE } + +internal sealed class MouseInput { + data class Down(val button: MouseButton) : MouseInput() + data class Up(val button: MouseButton) : MouseInput() + object Move : MouseInput() + object DoubleClick : MouseInput() + object Unknown : MouseInput() +} + +internal fun decodeMouseFlags(flags: Int): MouseInput = when (flags) { + 0x00 -> MouseInput.Move + 0x02 -> MouseInput.Down(MouseButton.LEFT) + 0x04 -> MouseInput.Up(MouseButton.LEFT) + 0x08 -> MouseInput.Down(MouseButton.RIGHT) + 0x10 -> MouseInput.Up(MouseButton.RIGHT) + 0x20 -> MouseInput.Down(MouseButton.MIDDLE) + 0x40 -> MouseInput.Up(MouseButton.MIDDLE) + 0x88 -> MouseInput.DoubleClick + else -> MouseInput.Unknown +} + +// Touch message pointer flags (Windows POINTER_FLAG_* values as sent by the viewer). +internal const val TOUCH_FLAG_DOWN = 0x00010000 +internal const val TOUCH_FLAG_UPDATE = 0x00020000 +internal const val TOUCH_FLAG_UP = 0x00040000 diff --git a/app/src/main/java/com/meshcentral/agent/MainActivity.kt b/app/src/main/java/com/meshcentral/agent/MainActivity.kt index a830af7..8a3f733 100644 --- a/app/src/main/java/com/meshcentral/agent/MainActivity.kt +++ b/app/src/main/java/com/meshcentral/agent/MainActivity.kt @@ -29,7 +29,6 @@ import androidx.appcompat.app.AppCompatActivity import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import com.google.firebase.messaging.FirebaseMessaging -import org.json.JSONObject import java.security.PrivateKey import java.security.cert.X509Certificate @@ -77,6 +76,9 @@ var g_auth_url : Uri? = null class MainActivity : AppCompatActivity() { var alert : AlertDialog? = null + // The consent dialog on screen, if any, and which session kind it belongs to. + private var consentAlert: AlertDialog? = null + private var consentAlertKind = 0 // Set when the user taps "Later" on the unattended setup prompt; suppresses it for this session // only, so it returns on the next launch/resume while items are still missing. private var unattendedPromptDismissed = false @@ -90,21 +92,10 @@ class MainActivity : AppCompatActivity() { ActivityResultContracts.StartActivityForResult() ) { result -> if (result.resultCode == RESULT_OK) { + // The capture service clears the viewer's consent banner once projection is running. ContextCompat.startForegroundService(this, ScreenCaptureService.getStartIntent(this, result.resultCode, result.data)) - AgentController.activeDesktopTunnel()?.sendCtrlResponse(JSONObject().apply { - put("type", "console") - put("msg", null) - put("msgid", 0) - }) } else { - AgentController.activeDesktopTunnel()?.let { tunnel -> - tunnel.sendCtrlResponse(JSONObject().apply { - put("type", "console") - put("msg", "denied") - put("msgid", 2) - }) - tunnel.Stop() - } + AgentController.denyUnattendedConsent() } } @@ -153,6 +144,7 @@ class MainActivity : AppCompatActivity() { if (AgentController.hasActiveDesktopTunnel() && !AgentController.isRemoteDesktopRunning()) { AgentController.startProjection() } + AgentController.showPendingFilesConsent() } invalidateOptionsMenu() } @@ -291,6 +283,8 @@ class MainActivity : AppCompatActivity() { alert?.dismiss() alert = null } + consentAlert?.dismiss() + consentAlert = null super.onDestroy() } @@ -627,27 +621,57 @@ class MainActivity : AppCompatActivity() { .show() } - // Per-connection consent prompt shown when Automatic Consent is off. - fun promptUnattendedConsent() { + // Per-connection consent prompt for screen sharing. + fun promptUnattendedConsent(message: String) { if (AgentController.isRemoteDesktopRunning() || (meshAgent == null) || (meshAgent!!.state != 3)) return + showConsentPrompt(CONSENT_DESKTOP, R.string.share_screen_choice_title, message, R.string.share_screen_once, + onApprove = { AgentController.confirmUnattendedConsent() }, + onDeny = { AgentController.denyUnattendedConsent() }) + } + + fun promptFilesConsent(message: String) { + if ((meshAgent == null) || (meshAgent!!.state != 3)) return + showConsentPrompt(CONSENT_FILES, R.string.approve_files_title, message, R.string.approve, + onApprove = { AgentController.confirmFilesConsent() }, + onDeny = { AgentController.denyFilesConsent() }) + } + + private fun showConsentPrompt(kind: Int, titleRes: Int, message: String, approveRes: Int, onApprove: () -> Unit, onDeny: () -> Unit) { if (isFinishing || isDestroyed) return + consentAlert?.dismiss() + consentAlert = null if (alert != null) { alert?.dismiss() alert = null } - alert = AlertDialog.Builder(this) - .setTitle(R.string.share_screen_choice_title) - .setMessage(R.string.unattended_consent_message) - .setPositiveButton(R.string.share_screen_once) { _, _ -> - AgentController.confirmUnattendedConsent() + consentAlertKind = kind + consentAlert = AlertDialog.Builder(this) + .setTitle(titleRes) + .setMessage(message) + .setPositiveButton(approveRes) { _, _ -> + consentAlert = null + onApprove() } - .setNegativeButton(android.R.string.cancel) { dialog, _ -> - sendDesktopConsentDenied() + .setNegativeButton(R.string.deny) { dialog, _ -> + consentAlert = null + onDeny() dialog.dismiss() } + // Backing out of the prompt is a refusal, not a silent wait for the timeout. + .setOnCancelListener { + consentAlert = null + onDeny() + } .show() } + fun dismissConsentPrompt(kind: Int) { + if (consentAlertKind != kind) return + val dialog = consentAlert ?: return + consentAlert = null + dialog.dismiss() + } + private fun sendDesktopConsentDenied() { AgentController.denyUnattendedConsent() } @@ -663,5 +687,7 @@ class MainActivity : AppCompatActivity() { companion object { const val REQUEST_ALL_PERMISSIONS = 1 const val REQUEST_LOCAL_NETWORK_PERMISSION = 2 + const val CONSENT_DESKTOP = 1 + const val CONSENT_FILES = 2 } } diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt index a3d0c66..944b993 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -2,6 +2,10 @@ package com.meshcentral.agent import android.accessibilityservice.AccessibilityService import android.accessibilityservice.GestureDescription +import android.app.KeyguardManager +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Path import android.os.Build @@ -12,10 +16,10 @@ import android.os.SystemClock import android.view.Display import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo +import androidx.annotation.RequiresApi import okio.ByteString import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import kotlin.math.absoluteValue import kotlin.math.max import kotlin.math.min @@ -29,18 +33,35 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { @Volatile private var capturing = false @Volatile private var lastWidth = 0 @Volatile private var lastHeight = 0 - private var pointerDownX: Int? = null - private var pointerDownY: Int? = null - // In-progress drag: path points and start time, so the gesture follows real motion. - private var dragPoints: ArrayList? = null - private var dragStartUptimeMs = 0L private var unsupportedKeyboardNotified = false @Volatile private var nextFrameDelayMs = MIN_FRAME_DELAY_MS @Volatile private var screenshotErrorNotified = false @Volatile private var lastCaptureUptimeMs = 0L - // Accessibility runs one gesture at a time; queue them so quick taps aren't dropped. - private val gestureQueue = ArrayDeque() + + // Pointer input is streamed as continued strokes: button down puts a finger on the screen, + // each move drags it and button up lifts it. Drags happen live, holding the button is a long + // press and long-press-then-drag works. Android runs one injected gesture at a time, so steps + // queue here and the completion callback pumps the next one. Main thread only. + private sealed class InputStep { + class Down(val x: Int, val y: Int) : InputStep() + class Move(val x: Int, val y: Int) : InputStep() + class Up(val x: Int, val y: Int) : InputStep() + class DoubleTap(val x: Int, val y: Int) : InputStep() + class Gesture(val gesture: GestureDescription) : InputStep() + } + private val inputSteps = ArrayDeque() private var gestureInFlight = false + // The last dispatched stroke whose finger is still down, and where it left it. + private var heldStroke: GestureDescription.StrokeDescription? = null + private var heldX = 0f + private var heldY = 0f + private var fingerMoved = false + private var lastSegmentUptimeMs = 0L + private val recentTapUptimes = ArrayDeque() + private var keyguardNoticeSent = false + // What the operator typed into the focused password field; Android masks the field's own text. + private val passwordBuffer = StringBuilder() + private var passwordNodeKey: String? = null override val isRunning: Boolean get() = active @@ -79,7 +100,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { fun startDesktop(): Boolean { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { - AgentController.sendDesktopMessage("Unattended screenshots require Android 11 or later.") + AgentController.sendDesktopMessage("Unattended screenshots require Android 11 or later.", timeoutSeconds = null) return false } if (active) return true @@ -89,6 +110,8 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { nextFrameDelayMs = MIN_FRAME_DELAY_MS encoder.requestFullFrame() updateTunnelDisplaySize() + AgentController.desktopProviderStarted() + noteKeyguard() captureFrame() meshAgent?.sendConsoleResponse("Started unattended display sharing", null) return true @@ -98,6 +121,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { val wasActive = active active = false mainHandler.removeCallbacks(captureRunnable) + releaseHeldPointer() if (g_remoteDesktopProvider === this) { g_remoteDesktopProvider = null } @@ -113,141 +137,271 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { override fun handleMouseCommand(msg: ByteString): Boolean { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || msg.size < 10) return false - val flags = u(msg[5]) - var x = readShort(msg, 6) - var y = readShort(msg, 8) - if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { - x = (x * 1024) / g_desktop_scalingLevel - y = (y * 1024) / g_desktop_scalingLevel - } + val (x, y) = toScreen(readShort(msg, 6), readShort(msg, 8)) if (msg.size >= 12) { val delta = readSignedShort(msg, 10) if (delta != 0) { - val distance = if (delta > 0) -350 else 350 - dispatchSwipe(x, y, x, y + distance, 250) + // Wheel: swipe the content under the cursor; a positive delta scrolls up. + val distance = if (delta > 0) -SCROLL_STEP_PX else SCROLL_STEP_PX + val endY = (y + distance).coerceIn(0, max(0, height - 1)) + swipeGesture(x, y, x, endY, SCROLL_SWIPE_MS)?.let { enqueue(InputStep.Gesture(it)) } return true } } - return when { - flags == 0x88 -> { - dispatchTap(x, y) - mainHandler.postDelayed({ dispatchTap(x, y) }, 120) - true - } - flags == 0x02 || flags == 0x08 || flags == 0x20 -> { - beginDrag(x, y) - true - } - // Move with a button held = drag; no button = hover. - flags == 0x00 -> { - extendDrag(x, y) - true + when (val input = decodeMouseFlags(u(msg[5]))) { + is MouseInput.Down -> if (input.button == MouseButton.LEFT) enqueue(InputStep.Down(x, y)) + is MouseInput.Up -> when (input.button) { + MouseButton.LEFT -> enqueue(InputStep.Up(x, y)) + // Android has no secondary buttons; follow the scrcpy and Vysor convention instead. + MouseButton.RIGHT -> globalAction(GLOBAL_ACTION_BACK) + MouseButton.MIDDLE -> globalAction(GLOBAL_ACTION_HOME) } - flags == 0x04 || flags == 0x10 || flags == 0x40 -> { - endDrag(x, y) - true - } - else -> true + MouseInput.Move -> enqueue(InputStep.Move(x, y)) + MouseInput.DoubleClick -> enqueue(InputStep.DoubleTap(x, y)) + MouseInput.Unknown -> {} } + return true } - private fun beginDrag(x: Int, y: Int) { - pointerDownX = x - pointerDownY = y - dragStartUptimeMs = SystemClock.uptimeMillis() - dragPoints = arrayListOf(x.toFloat(), y.toFloat()) + override fun handleTouchCommand(msg: ByteString): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || msg.size < 14 || u(msg[4]) != 1) return false + val flags = readInt(msg, 6) + val (x, y) = toScreen(readShort(msg, 10), readShort(msg, 12)) + when { + (flags and TOUCH_FLAG_DOWN) != 0 -> enqueue(InputStep.Down(x, y)) + (flags and TOUCH_FLAG_UP) != 0 -> enqueue(InputStep.Up(x, y)) + (flags and TOUCH_FLAG_UPDATE) != 0 -> enqueue(InputStep.Move(x, y)) + } + return true } - private fun extendDrag(x: Int, y: Int) { - val pts = dragPoints ?: return - val n = pts.size - if (n >= 2 && pts[n - 2] == x.toFloat() && pts[n - 1] == y.toFloat()) return - if (pts.size < MAX_DRAG_POINTS * 2) { - pts.add(x.toFloat()) - pts.add(y.toFloat()) + override fun handleKeyCommand(cmd: Int, msg: ByteString): Boolean { + val handled = when (cmd) { + 1 -> handleLegacyKey(msg) + 85 -> handleUnicodeKey(msg) + else -> false + } + if (handled) wakeCapture() + return handled + } + + // Undo viewer scaling and keep the point on screen: a drag released past the viewer's edge + // arrives with out-of-range coordinates that Android would refuse. + private fun toScreen(rawX: Int, rawY: Int): Pair { + var x = rawX + var y = rawY + if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { + x = (x * 1024) / g_desktop_scalingLevel + y = (y * 1024) / g_desktop_scalingLevel + } + return Pair(x.coerceIn(0, max(0, width - 1)), y.coerceIn(0, max(0, height - 1))) + } + + private fun enqueue(step: InputStep) { + mainHandler.post { + // Moves are coalesced anyway; drop a surplus one rather than a press or release. + if (inputSteps.size >= MAX_QUEUED_STEPS && step is InputStep.Move) return@post + inputSteps.addLast(step) + pumpInput() + } + } + + private fun pumpInput() { + while (!gestureInFlight) { + val step = inputSteps.removeFirstOrNull() ?: return + when (step) { + is InputStep.Down -> { + noteKeyguard() + if (heldStroke != null) { + // A second down without an up means the release was lost: lift, then redo it. + inputSteps.addFirst(step) + liftPointer(heldX.toInt(), heldY.toInt()) + } else { + pressPointer(step.x, step.y) + } + } + is InputStep.Move -> { + var move = step + // Only the newest position matters once a segment is already in flight. + while (inputSteps.firstOrNull() is InputStep.Move) { + move = inputSteps.removeFirst() as InputStep.Move + } + if (heldStroke != null && (move.x.toFloat() != heldX || move.y.toFloat() != heldY)) { + movePointer(move.x, move.y) + } + } + is InputStep.Up -> if (heldStroke != null) liftPointer(step.x, step.y) + is InputStep.DoubleTap -> { + // The viewer sends both clicks as down/up pairs before this flag, so only + // synthesize the taps when they didn't come through. + val cutoff = SystemClock.uptimeMillis() - DOUBLE_TAP_WINDOW_MS + if (recentTapUptimes.count { it >= cutoff } < 2) { + inputSteps.addFirst(InputStep.Up(step.x, step.y)) + inputSteps.addFirst(InputStep.Down(step.x, step.y)) + inputSteps.addFirst(InputStep.Up(step.x, step.y)) + inputSteps.addFirst(InputStep.Down(step.x, step.y)) + } + } + is InputStep.Gesture -> { + if (heldStroke != null) { + inputSteps.addFirst(step) + liftPointer(heldX.toInt(), heldY.toInt()) + } else { + dispatchQueued(step.gesture) + } + } + } } } - private fun endDrag(x: Int, y: Int) { - val startX = pointerDownX ?: x - val startY = pointerDownY ?: y - val pts = dragPoints - pointerDownX = null - pointerDownY = null - dragPoints = null - val moved = (startX - x).absoluteValue >= 8 || (startY - y).absoluteValue >= 8 - if (!moved || pts == null) { - dispatchTap(x, y) + private fun pressPointer(x: Int, y: Int) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + // No continued strokes before Android 8: a press is a plain tap and moves are dropped. + tapGesture(x, y)?.let { dispatchQueued(it) } + return + } + val fx = x.toFloat() + val fy = y.toFloat() + val stroke = try { + GestureDescription.StrokeDescription(pointPath(fx, fy), 0, PRESS_MS, true) + } catch (ex: Exception) { return } - pts.add(x.toFloat()) - pts.add(y.toFloat()) - // Real hold time drives gesture duration, so a flick stays a flick. - val elapsed = (SystemClock.uptimeMillis() - dragStartUptimeMs) - .coerceIn(MIN_DRAG_DURATION_MS, MAX_DRAG_DURATION_MS) - dispatchDrag(pts, elapsed) + fingerMoved = false + dispatchStroke(stroke, fx, fy, keepsFinger = true) } - private fun dispatchDrag(pts: List, durationMs: Long) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || pts.size < 4) return - val path = Path() - path.moveTo(pts[0], pts[1]) - var i = 2 - while (i + 1 < pts.size) { - path.lineTo(pts[i], pts[i + 1]) - i += 2 + private fun movePointer(x: Int, y: Int) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + val held = heldStroke ?: return + val fx = x.toFloat() + val fy = y.toFloat() + val stroke = try { + held.continueStroke(linePath(heldX, heldY, fx, fy), 0, segmentDurationMs(), true) + } catch (ex: Exception) { + heldStroke = null + return } + fingerMoved = true + dispatchStroke(stroke, fx, fy, keepsFinger = true) + } + + private fun liftPointer(x: Int, y: Int) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + val held = heldStroke ?: return + val fx = x.toFloat() + val fy = y.toFloat() + val stationary = fx == heldX && fy == heldY + val stroke = try { + if (stationary) { + held.continueStroke(pointPath(fx, fy), 0, LIFT_MS, false) + } else { + held.continueStroke(linePath(heldX, heldY, fx, fy), 0, segmentDurationMs(), false) + } + } catch (ex: Exception) { + heldStroke = null + return + } + if (stationary && !fingerMoved) { + recentTapUptimes.addLast(SystemClock.uptimeMillis()) + while (recentTapUptimes.size > 4) recentTapUptimes.removeFirst() + } + dispatchStroke(stroke, fx, fy, keepsFinger = false) + } + + private fun dispatchStroke(stroke: GestureDescription.StrokeDescription, endX: Float, endY: Float, keepsFinger: Boolean) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return val gesture = try { - GestureDescription.Builder() - .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs)) - .build() + GestureDescription.Builder().addStroke(stroke).build() } catch (ex: Exception) { + heldStroke = null return } - dispatchGestureQueued(gesture) + heldStroke = if (keepsFinger) stroke else null + heldX = endX + heldY = endY + lastSegmentUptimeMs = SystemClock.uptimeMillis() + if (!dispatchQueued(gesture)) heldStroke = null } - override fun handleTouchCommand(msg: ByteString): Boolean { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N || msg.size < 14 || u(msg[4]) != 1) return false - val flags = readInt(msg, 6) - var x = readShort(msg, 10) - var y = readShort(msg, 12) - if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { - x = (x * 1024) / g_desktop_scalingLevel - y = (y * 1024) / g_desktop_scalingLevel + // Hands one gesture to Android; the callback resumes the queue. False when it was refused. + private fun dispatchQueued(gesture: GestureDescription): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return false + wakeCapture() + gestureInFlight = true + val callback = gestureCallback ?: GestureCallback().also { gestureCallback = it } + val accepted = try { + dispatchGesture(gesture, callback, mainHandler) + } catch (ex: Exception) { + false } - return when { - (flags and 0x00010000) != 0 -> { - pointerDownX = x - pointerDownY = y - true - } - (flags and 0x00040000) != 0 -> { - val startX = pointerDownX ?: x - val startY = pointerDownY ?: y - pointerDownX = null - pointerDownY = null - if ((startX - x).absoluteValue < 8 && (startY - y).absoluteValue < 8) { - dispatchTap(x, y) - } else { - dispatchSwipe(startX, startY, x, y, 350) - } - true + if (!accepted) gestureInFlight = false + return accepted + } + + // Both callbacks are created on first use: instantiating them in a field initializer would + // reference classes older Android releases lack and crash the service as it binds. + private var gestureCallback: AccessibilityService.GestureResultCallback? = null + private var screenshotCallback: AccessibilityService.TakeScreenshotCallback? = null + + @RequiresApi(Build.VERSION_CODES.N) + private inner class GestureCallback : AccessibilityService.GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription?) { + gestureInFlight = false + pumpInput() + } + + override fun onCancelled(gestureDescription: GestureDescription?) { + // Android dropped the finger, usually because the device user touched the screen. + gestureInFlight = false + heldStroke = null + pumpInput() + } + } + + // Segment length follows the real time between moves, so a slow drag stays slow and a flick + // stays a flick; the cap keeps a move after a pause from crawling. + private fun segmentDurationMs(): Long { + return (SystemClock.uptimeMillis() - lastSegmentUptimeMs).coerceIn(MIN_SEGMENT_MS, MAX_SEGMENT_MS) + } + + // Lift a finger left on screen when the session ends so it doesn't stay pressed. + private fun releaseHeldPointer() { + mainHandler.post { + inputSteps.clear() + if (heldStroke != null) { + inputSteps.addLast(InputStep.Up(heldX.toInt(), heldY.toInt())) + pumpInput() } - else -> true } } - override fun handleKeyCommand(cmd: Int, msg: ByteString): Boolean { - val handled = when (cmd) { - 1 -> handleLegacyKey(msg) - 85 -> handleUnicodeKey(msg) - else -> false + private fun pointPath(x: Float, y: Float): Path = Path().apply { moveTo(x, y) } + + private fun linePath(x1: Float, y1: Float, x2: Float, y2: Float): Path = Path().apply { + moveTo(x1, y1) + lineTo(x2, y2) + } + + private fun strokeGesture(path: Path, durationMs: Long): GestureDescription? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return null + return try { + GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs)) + .build() + } catch (ex: Exception) { + null } - if (handled) wakeCapture() - return handled + } + + private fun tapGesture(x: Int, y: Int): GestureDescription? { + return strokeGesture(pointPath(x.toFloat(), y.toFloat()), TAP_MS) + } + + private fun swipeGesture(startX: Int, startY: Int, endX: Int, endY: Int, durationMs: Long): GestureDescription? { + return strokeGesture(linePath(startX.toFloat(), startY.toFloat(), endX.toFloat(), endY.toFloat()), durationMs) } private fun captureFrame() { @@ -255,14 +409,16 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { capturing = true lastCaptureUptimeMs = SystemClock.uptimeMillis() try { - takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, screenshotCallback) + val callback = screenshotCallback ?: ScreenshotCallback().also { screenshotCallback = it } + takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, callback) } catch (ex: Exception) { capturing = false scheduleNextCapture() } } - private val screenshotCallback = object : AccessibilityService.TakeScreenshotCallback { + @RequiresApi(Build.VERSION_CODES.R) + private inner class ScreenshotCallback : AccessibilityService.TakeScreenshotCallback { override fun onSuccess(screenshot: AccessibilityService.ScreenshotResult) { // Recovered: allow the next error to be reported again. screenshotErrorNotified = false @@ -343,40 +499,14 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { mainHandler.postDelayed(captureRunnable, delay) } - // dispatchGesture drops overlapping gestures, so serialize them; quick taps aren't lost. - private fun dispatchGestureQueued(gesture: GestureDescription) { - wakeCapture() - mainHandler.post { - if (gestureQueue.size >= MAX_QUEUED_GESTURES) gestureQueue.removeFirst() - gestureQueue.addLast(gesture) - pumpGestures() - } - } - - private fun pumpGestures() { - if (gestureInFlight) return - val gesture = gestureQueue.removeFirstOrNull() ?: return - gestureInFlight = true - dispatchGesture(gesture, object : AccessibilityService.GestureResultCallback() { - override fun onCompleted(gestureDescription: GestureDescription?) { - gestureInFlight = false - pumpGestures() - } - override fun onCancelled(gestureDescription: GestureDescription?) { - gestureInFlight = false - pumpGestures() - } - }, mainHandler) - } - private fun handleLegacyKey(msg: ByteString): Boolean { if (msg.size < 6) return false val action = u(msg[4]) val keyCode = u(msg[5]) if (action != 0) return true when (keyCode) { - 8 -> return backspaceFocused() - 13 -> return insertFocused("\n") + 8 -> return backspaceFocused() || (keyguardLocked() && keyguardKey("delete_button", null)) + 13 -> return enterFocused() 37 -> return moveFocusedCursor(-1) 39 -> return moveFocusedCursor(1) 38 -> return moveFocusedCursorLine(-1) @@ -385,7 +515,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { 36 -> return globalAction(GLOBAL_ACTION_HOME) 93 -> return globalAction(GLOBAL_ACTION_RECENTS) // Codes above the keyboard range are the desktop panel's Android action buttons. - 200 -> return globalAction(GLOBAL_ACTION_ACCESSIBILITY_ALL_APPS, Build.VERSION_CODES.S) // App drawer + 200 -> return openAppDrawer() 201 -> return globalAction(GLOBAL_ACTION_NOTIFICATIONS) 202 -> return globalAction(GLOBAL_ACTION_QUICK_SETTINGS) 203 -> return globalAction(GLOBAL_ACTION_LOCK_SCREEN, Build.VERSION_CODES.P) @@ -401,17 +531,129 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { return true } + // GLOBAL_ACTION_ACCESSIBILITY_ALL_APPS exists since Android 12, but only Android 14 SystemUI + // wires it to the launcher; on 12 and 13 it injects a key the stock launcher ignores. So use it + // only where it can work, check that the launcher actually came up, and otherwise do what a + // user does: go home and swipe up. + private fun openAppDrawer(): Boolean { + val launcher = defaultLauncherPackage() + val front = rootInActiveWindow?.packageName?.toString() + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE && launcher != null && front != launcher) { + performGlobalAction(GLOBAL_ACTION_ACCESSIBILITY_ALL_APPS) + mainHandler.postDelayed({ + if (rootInActiveWindow?.packageName?.toString() != launcher) swipeUpFromHome() + }, APP_DRAWER_VERIFY_MS) + } else { + swipeUpFromHome() + } + return true + } + + private fun swipeUpFromHome() { + performGlobalAction(GLOBAL_ACTION_HOME) + mainHandler.postDelayed({ + val x = width / 2 + val gesture = swipeGesture(x, height * 4 / 5, x, height * 3 / 10, APP_DRAWER_SWIPE_MS) ?: return@postDelayed + enqueue(InputStep.Gesture(gesture)) + }, HOME_SETTLE_MS) + } + + private fun defaultLauncherPackage(): String? { + val intent = Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_HOME) + return try { + packageManager.resolveActivity(intent, PackageManager.MATCH_DEFAULT_ONLY)?.activityInfo?.packageName + } catch (ex: Exception) { + null + } + } + private fun handleUnicodeKey(msg: ByteString): Boolean { if (msg.size < 7) return false // Insert on key-up (action 1). The browser keypress that carries the key-down is deprecated and // often doesn't fire, but key-up always does; the panel and physical typing both send an up. if (u(msg[4]) != 1) return true - val charCode = readShort(msg, 5) - return insertFocused(charCode.toChar().toString()).also { + val ch = readShort(msg, 5).toChar() + // Lock screen PIN pad: no editable field, so press its buttons by hand. + if (ch.isDigit() && keyguardLocked() && focusedEditable() == null && keyguardKey("key$ch", ch.toString())) return true + return insertFocused(ch.toString()).also { if (!it) notifyUnsupportedKeyboard() } } + private fun keyguardLocked(): Boolean { + val keyguard = getSystemService(Context.KEYGUARD_SERVICE) as? KeyguardManager ?: return false + return keyguard.isKeyguardLocked + } + + // The lock screen captures normally but Android blanks the PIN and password entry, so tell the + // operator how to unlock blind. Repeats after the notice expires while the device stays locked. + private fun noteKeyguard() { + if (!keyguardLocked()) { + keyguardNoticeSent = false + return + } + if (keyguardNoticeSent) return + keyguardNoticeSent = true + AgentController.sendDesktopMessage( + "The device is locked. Android hides the PIN pad from screen capture: swipe up, then type the PIN or password on your keyboard and press Enter.", + timeoutSeconds = KEYGUARD_NOTICE_S + ) + mainHandler.postDelayed({ keyguardNoticeSent = false }, KEYGUARD_NOTICE_S * 1000L) + } + + // Presses one of the keyguard's own buttons (key0..key9, delete_button, key_enter on AOSP), by + // resource id first and by exact label as a fallback for vendor lock screens. + private fun keyguardKey(idSuffix: String, label: String?): Boolean { + val root = rootInActiveWindow ?: return false + val byId = try { + root.findAccessibilityNodeInfosByViewId("com.android.systemui:id/$idSuffix") + } catch (ex: Exception) { + null + } + var node = byId?.firstOrNull { it.isEnabled } + if (node == null && label != null) { + val byText = try { root.findAccessibilityNodeInfosByText(label) } catch (ex: Exception) { null } + node = byText?.firstOrNull { + it.isClickable && (it.text?.toString() == label || it.contentDescription?.toString() == label) + } + } + return node?.performAction(AccessibilityNodeInfo.ACTION_CLICK) == true + } + + private fun enterFocused(): Boolean { + val node = focusedEditable() + if (node == null) { + // No text field: on the lock screen this confirms the PIN. + return keyguardLocked() && keyguardKey("key_enter", null) + } + // A single-line field treats Enter as its keyboard action (sign in, search, unlock); only + // multi-line text takes a newline. + if (!node.isMultiLine) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && + node.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_IME_ENTER.id)) return true + if (keyguardLocked() && keyguardKey("key_enter", null)) return true + } + return insertFocused("\n") + } + + // Android masks a password field's text to accessibility, so rebuilding it from the node would + // insert into dots; keep what the operator typed since the field got focus and rewrite it whole. + private fun typeIntoPassword(node: AccessibilityNodeInfo, insert: String?): Boolean { + val key = "${node.windowId}:${node.hashCode()}" + if (key != passwordNodeKey) { + passwordNodeKey = key + passwordBuffer.setLength(0) + } + if (insert == null) { + if (passwordBuffer.isNotEmpty()) passwordBuffer.setLength(passwordBuffer.length - 1) + } else { + passwordBuffer.append(insert) + } + val args = Bundle() + args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, passwordBuffer.toString()) + return node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) + } + private fun focusedEditable(): AccessibilityNodeInfo? { val node = rootInActiveWindow?.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) ?: return null return if (node.isEditable) node else null @@ -448,6 +690,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun insertFocused(insert: String): Boolean { val node = focusedEditable() ?: return false + if (node.isPassword) return typeIntoPassword(node, insert) val text = fieldText(node) val (s, e) = selectionRange(node, text.length) return replaceText(node, text.substring(0, s) + insert + text.substring(e), s + insert.length) @@ -455,6 +698,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun backspaceFocused(): Boolean { val node = focusedEditable() ?: return false + if (node.isPassword) return typeIntoPassword(node, null) val text = fieldText(node) val (s, e) = selectionRange(node, text.length) return when { @@ -504,27 +748,6 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { AgentController.sendDesktopMessage("Android unattended keyboard input is limited to focused editable text and basic navigation keys.") } - private fun dispatchTap(x: Int, y: Int) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return - val path = Path() - path.moveTo(x.toFloat(), y.toFloat()) - val gesture = GestureDescription.Builder() - .addStroke(GestureDescription.StrokeDescription(path, 0, 80)) - .build() - dispatchGestureQueued(gesture) - } - - private fun dispatchSwipe(startX: Int, startY: Int, endX: Int, endY: Int, duration: Long) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) return - val path = Path() - path.moveTo(startX.toFloat(), startY.toFloat()) - path.lineTo(endX.toFloat(), endY.toFloat()) - val gesture = GestureDescription.Builder() - .addStroke(GestureDescription.StrokeDescription(path, 0, duration)) - .build() - dispatchGestureQueued(gesture) - } - private fun updateTunnelDisplaySize() { val agent = meshAgent ?: return for (t in agent.tunnels) { @@ -560,9 +783,19 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private const val MAX_IDLE_FRAME_DELAY_MS = 2_000L // System throttles takeScreenshot() faster than ~3 fps (AOSP interval is 333ms). private const val MIN_SCREENSHOT_INTERVAL_MS = 350L - private const val MAX_QUEUED_GESTURES = 16 - private const val MAX_DRAG_POINTS = 64 - private const val MIN_DRAG_DURATION_MS = 40L - private const val MAX_DRAG_DURATION_MS = 1500L + private const val MAX_QUEUED_STEPS = 64 + // Finger timing for streamed strokes. + private const val PRESS_MS = 40L + private const val LIFT_MS = 10L + private const val MIN_SEGMENT_MS = 16L + private const val MAX_SEGMENT_MS = 200L + private const val TAP_MS = 80L + private const val DOUBLE_TAP_WINDOW_MS = 700L + private const val SCROLL_STEP_PX = 350 + private const val SCROLL_SWIPE_MS = 250L + private const val APP_DRAWER_VERIFY_MS = 700L + private const val HOME_SETTLE_MS = 450L + private const val APP_DRAWER_SWIPE_MS = 300L + private const val KEYGUARD_NOTICE_S = 45 } } diff --git a/app/src/main/java/com/meshcentral/agent/MeshAgent.kt b/app/src/main/java/com/meshcentral/agent/MeshAgent.kt index 5498725..be3304c 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAgent.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAgent.kt @@ -1,9 +1,12 @@ package com.meshcentral.agent import android.annotation.SuppressLint +import android.content.ClipData +import android.content.ClipboardManager import android.content.Context import android.content.Intent import android.content.IntentFilter +import android.content.pm.ApplicationInfo import android.content.pm.PackageManager import android.graphics.* import android.hardware.camera2.CameraAccessException @@ -27,7 +30,10 @@ import java.security.cert.CertificateFactory import java.security.cert.CertificateException import java.security.cert.X509Certificate import java.security.interfaces.RSAPublicKey -import java.util.concurrent.TimeUnit +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.concurrent.CopyOnWriteArrayList import javax.net.ssl.HostnameVerifier import javax.net.ssl.SSLContext import javax.net.ssl.TrustManager @@ -65,7 +71,9 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S private var connectionTimer: CountDownTimer? = null private var lastBattState : JSONObject? = null private var lastNetInfo : String? = null - var tunnels : ArrayList = ArrayList() + // Tunnels come and go on OkHttp threads while capture and UI code iterate; copy-on-write keeps + // every iteration safe without locking. + val tunnels : MutableList = CopyOnWriteArrayList() var userinfo : HashMap = HashMap() // UserID -> MeshUserInfo init { @@ -118,10 +126,7 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S val sslSocketFactory = sslContext.socketFactory - return OkHttpClient.Builder() - .connectTimeout(20, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.MINUTES) - .writeTimeout(60, TimeUnit.MINUTES) + return MeshHttp.base.newBuilder() .hostnameVerifier(hostnameVerifier = HostnameVerifier { _, _ -> true }) .sslSocketFactory(sslSocketFactory, trustAllCerts[0] as X509TrustManager) .build() @@ -386,26 +391,22 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S "netinfo" -> { sendNetworkUpdate(true) } + "software" -> { + // The Software tab. Package lookups can be slow, so answer off the socket thread. + thread(name = "MeshSoftware") { sendSoftwareInventory(json) } + } "openUrl" -> { - /* - if (visibleScreen != 2) { // Device is busy in QR code scanner - // Open the URL - var xurl = json.optString("url") - //println("Opening: $xurl") - if ((xurl != null) && (parent.openUrl(xurl))) { - // Event to the server - var eventArgs = JSONArray() - eventArgs.put(xurl) - logServerEventEx(20, eventArgs, "Opening: ${xurl}", json); + // The server's "open URL on device" feature; only web links are handed to Android, + // so a server can't fire tel:, sms: or app-specific schemes at the device. + val xurl = json.optString("url") + if (xurl.startsWith("https://") || xurl.startsWith("http://")) { + try { + parent.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(xurl))) + logServerEventEx(20, JSONArray().put(xurl), "Opening: $xurl", json) + } catch (ex: Exception) { + println("openUrl failed: $ex") } } - */ - - var xurl = json.optString("url") - if (xurl.isNotEmpty()) { - var getintent: Intent = Intent(Intent.ACTION_VIEW, Uri.parse(xurl)); - parent.startActivity(getintent); - } } "msg" -> { var msgtype = json.getString("type") @@ -413,6 +414,12 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S "console" -> { processConsoleMessage(json.getString("value"), json.getString("sessionid"), json) } + "getclip" -> { + parent.runOnHostThread { sendClipboard(json) } + } + "setclip" -> { + parent.runOnHostThread { receiveClipboard(json) } + } "tunnel" -> { /* {"action":"msg", @@ -529,6 +536,120 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S } } + private fun sendSoftwareInventory(json: JSONObject) { + val value: Any = when (json.optString("type")) { + "installedapps" -> try { + installedApps() + } catch (ex: Exception) { + JSONObject().put("error", ex.toString()) + } + else -> JSONObject().put("success", false).put("error", "Not supported on Android") + } + val r = JSONObject() + r.put("action", "software") + r.put("value", value.toString()) + r.put("sessionid", json.optString("sessionid")) + if (_webSocket != null) { _webSocket?.send(r.toString().toByteArray().toByteString()) } + } + + // Every app with a launcher entry, which the manifest queries for; that covers what a user + // would call installed apps without needing QUERY_ALL_PACKAGES and its Play declaration. + @Suppress("DEPRECATION") + private fun installedApps(): JSONArray { + val packageManager = parent.getApplicationContext().packageManager + val launcher = Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER) + val packages = packageManager.queryIntentActivities(launcher, 0).map { it.activityInfo.packageName }.toSortedSet() + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + val apps = ArrayList() + for (pkg in packages) { + val info = try { packageManager.getPackageInfo(pkg, 0) } catch (ex: Exception) { continue } + val app = info.applicationInfo ?: continue + val entry = JSONObject() + entry.put("name", app.loadLabel(packageManager).toString()) + entry.put("version", info.versionName ?: "") + entry.put("publisher", installerName(packageManager, pkg, app)) + entry.put("date", dateFormat.format(Date(info.lastUpdateTime))) + entry.put("location", pkg) + apps.add(entry) + } + apps.sortBy { it.optString("name").lowercase() } + return JSONArray(apps) + } + + @Suppress("DEPRECATION") + private fun installerName(packageManager: PackageManager, pkg: String, app: ApplicationInfo): String { + val installer = try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + packageManager.getInstallSourceInfo(pkg).installingPackageName + } else { + packageManager.getInstallerPackageName(pkg) + } + } catch (ex: Exception) { + null + } + return when { + installer == "com.android.vending" -> "Google Play" + installer != null -> installer + (app.flags and ApplicationInfo.FLAG_SYSTEM) != 0 -> "System" + else -> "Sideloaded" + } + } + + // Android 10+ only hands the clipboard to the focused app or the keyboard, so reading usually + // works only while the agent's own screen is open. Polls (tag 3) stay silent; a manual request + // gets told why nothing came back. + private fun sendClipboard(json: JSONObject) { + val tag = json.opt("tag") + val text = readClipboardText() + if (text == null) { + if (tag != 3) AgentController.sendDesktopMessage("Android only lets the agent read the clipboard while the MeshCentral Agent app is open on the device.") + return + } + if (tag != 3) logServerEventEx(21, JSONArray().put(text.length), "Getting clipboard content, ${text.length} byte(s)", json) + val r = JSONObject() + r.put("action", "msg") + r.put("type", "getclip") + r.put("sessionid", json.optString("sessionid")) + r.put("data", text) + if (tag != null) r.put("tag", tag) + if (_webSocket != null) { _webSocket?.send(r.toString().toByteArray().toByteString()) } + } + + private fun receiveClipboard(json: JSONObject) { + val text = if (json.isNull("data")) null else json.optString("data") + val ok = (text != null) && writeClipboardText(text) + if (ok) logServerEventEx(22, JSONArray().put(text!!.length), "Setting clipboard content, ${text.length} byte(s)", json) + val r = JSONObject() + r.put("action", "msg") + r.put("type", "setclip") + r.put("sessionid", json.optString("sessionid")) + r.put("success", ok) + if (_webSocket != null) { _webSocket?.send(r.toString().toByteArray().toByteString()) } + } + + private fun readClipboardText(): String? { + val context = parent.getApplicationContext() + val manager = context.getSystemService(Context.CLIPBOARD_SERVICE) as? ClipboardManager ?: return null + return try { + val clip = manager.primaryClip ?: return null + if (clip.itemCount == 0) return null + clip.getItemAt(0).coerceToText(context)?.toString() + } catch (ex: Exception) { + null + } + } + + private fun writeClipboardText(text: String): Boolean { + val context = parent.getApplicationContext() + val manager = context.getSystemService(Context.CLIPBOARD_SERVICE) as? ClipboardManager ?: return false + return try { + manager.setPrimaryClip(ClipData.newPlainText("MeshCentral", text)) + true + } catch (ex: Exception) { + false + } + } + // Send the latest core information to the server fun sendCoreInfo() { val r = JSONObject() @@ -579,6 +700,16 @@ class MeshAgent(parent: AgentHost, host: String, certHash: String, devGroupId: S parent.refreshInfo() } + // Downloads run on a server-opened tunnel that has no way to report errors, so tell the + // operator on their files session instead. + fun sendFilesMessage(message: String, userid: String?) { + for (t in tunnels) { + if ((t.state == 2) && (t.usage == 5) && (userid.isNullOrEmpty() || t.userid == userid)) { + t.sendConsoleMessage(message, timeoutSeconds = 20) + } + } + } + fun sendNetworkUpdate(force: Boolean) : Boolean { var netinfo = getNetInfo(); if ((force == false) && (lastNetInfo != null)) { diff --git a/app/src/main/java/com/meshcentral/agent/MeshHttp.kt b/app/src/main/java/com/meshcentral/agent/MeshHttp.kt new file mode 100644 index 0000000..ea79fed --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/MeshHttp.kt @@ -0,0 +1,17 @@ +package com.meshcentral.agent + +import okhttp3.OkHttpClient +import java.util.concurrent.TimeUnit + +// One dispatcher and connection pool for the control channel and every relay tunnel. Each +// connection derives its own certificate-pinning client from this with newBuilder(), which keeps +// the shared thread pools instead of spinning up new ones per tunnel. +internal object MeshHttp { + val base: OkHttpClient by lazy { + OkHttpClient.Builder() + .connectTimeout(20, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.MINUTES) + .writeTimeout(60, TimeUnit.MINUTES) + .build() + } +} diff --git a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt index 50bab1a..4c14865 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshTunnel.kt @@ -4,7 +4,7 @@ import android.annotation.SuppressLint import android.app.RecoverableSecurityException import android.content.ContentResolver import android.content.ContentUris -import android.content.ContentValues +import android.content.Context import android.database.Cursor import android.net.Uri import android.os.Build @@ -24,12 +24,13 @@ import java.security.MessageDigest import java.security.cert.CertificateException import java.security.cert.X509Certificate import java.util.* -import java.util.concurrent.TimeUnit import javax.net.ssl.HostnameVerifier import javax.net.ssl.SSLContext import javax.net.ssl.TrustManager import javax.net.ssl.X509TrustManager import kotlin.collections.ArrayList +import java.util.concurrent.CopyOnWriteArrayList +import kotlin.concurrent.thread import kotlin.math.absoluteValue import kotlin.random.Random @@ -55,7 +56,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo var usage: Int = 0 // 2 = Desktop, 5 = Files, 10 = File transfer private var tunnelOptions : JSONObject? = null private var lastDirRequest : JSONObject? = null - private var fileUpload : OutputStream? = null + private var fileUpload : UploadSink? = null private var fileUploadName : String? = null private var fileUploadReqId : Int = 0 private var fileUploadSize : Int = 0 @@ -63,6 +64,12 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo var guestname : String? = null var sessionUserName : String? = null // UserID + GuestName in Base64 if this is a shared session. var sessionUserName2 : String? = null // UserID/GuestName + // Server-side consent policy for this session, a bitmask from the tunnel command. + val consentFlags: Int = serverData.optInt("consent", 0) + // A files session waiting for the device user; its commands are held until they decide. + @Volatile var filesConsentPending = false + private set + private val heldFileCommands = CopyOnWriteArrayList() init { } @@ -76,7 +83,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo // Set the userid and request more data about this user guestname = serverData.optString("guestname") userid = serverData.optString("userid") - if (userid != null) parent.sendUserImageRequest(userid!!) + if (!userid.isNullOrEmpty()) parent.sendUserImageRequest(userid!!) sessionUserName = userid sessionUserName2 = userid if ((userid != "") && (guestname != "")) { @@ -124,10 +131,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo sslContext.init(null, trustAllCerts, java.security.SecureRandom()) val sslSocketFactory = sslContext.socketFactory - return OkHttpClient.Builder() - .connectTimeout(20, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.MINUTES) - .writeTimeout(60, TimeUnit.MINUTES) + return MeshHttp.base.newBuilder() .hostnameVerifier ( hostnameVerifier = HostnameVerifier{ _, _ -> true }) .sslSocketFactory(sslSocketFactory, trustAllCerts[0] as X509TrustManager) .build() @@ -149,10 +153,15 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo _webSocket = null } catch (ex: Exception) { } } - // Close any in-flight upload so we don't leak the descriptor or leave a partial file - if (fileUpload != null) { - try { fileUpload?.close() } catch (ex: Exception) { } - fileUpload = null + // Drop any in-flight upload so no descriptor leaks and no partial file is left behind + fileUpload?.discard() + fileUpload = null + closeBlockDownload() + // A files session closed while waiting for approval no longer needs the prompt + if (filesConsentPending) { + filesConsentPending = false + heldFileCommands.clear() + AgentController.filesConsentResolved() } // Clear the connection timer if (connectionTimer != null) { @@ -181,6 +190,135 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo companion object { const val NORMAL_CLOSURE_STATUS = 1000 + // Console message ids the web UI translates itself (agentConsoleMessages in the views). + const val MSGID_CONSENT_PENDING = 1 + const val MSGID_CONSENT_DENIED = 2 + const val CONSENT_PENDING_MESSAGE = "Waiting for user to grant access..." + // Consent bitmask bits, as defined by the MeshCentral server. + const val CONSENT_DESKTOP_NOTIFY = 1 + const val CONSENT_FILES_NOTIFY = 4 + const val CONSENT_DESKTOP_PROMPT = 8 + const val CONSENT_FILES_PROMPT = 32 + private const val DEFAULT_CONSENT_TIMEOUT_S = 30 + private const val MAX_HELD_FILE_COMMANDS = 32 + // Block payload the web UI expects (16 KB minus the 4-byte header). + private const val DOWNLOAD_BLOCK_SIZE = 16380 + } + + // The app setting forces a prompt; with automatic consent on, the server's flags still + // decide, as they do for the other agents. + fun consentPromptRequired(): Boolean { + val promptBit = if (usage == 5) CONSENT_FILES_PROMPT else CONSENT_DESKTOP_PROMPT + return !g_autoConsent || (consentFlags and promptBit) != 0 + } + + fun consentNotifyRequested(): Boolean { + val notifyBit = if (usage == 5) CONSENT_FILES_NOTIFY else CONSENT_DESKTOP_NOTIFY + return (consentFlags and notifyBit) != 0 + } + + fun consentTimeoutMs(): Long { + val seconds = serverData.optJSONObject("soptions")?.optInt("consentTimeout", 0) ?: 0 + return (if (seconds > 0) seconds else DEFAULT_CONSENT_TIMEOUT_S) * 1000L + } + + fun consentAutoAcceptOnTimeout(): Boolean { + return serverData.optJSONObject("soptions")?.optBoolean("consentAutoAccept", false) ?: false + } + + fun remoteUserName(context: Context): String { + val realname = serverData.optString("realname") + if (realname.isNotEmpty()) return realname + val username = serverData.optString("username") + if (username.isNotEmpty()) return username + val guest = guestname + if (!guest.isNullOrEmpty()) return guest + return context.getString(R.string.remote_user) + } + + fun consentMessage(context: Context): String { + val serverKey = if (usage == 5) "consentMsgFiles" else "consentMsgDesktop" + val defaultRes = if (usage == 5) R.string.files_consent_message else R.string.unattended_consent_message + return serverText(serverKey) ?: context.getString(defaultRes, remoteUserName(context)) + } + + // Server-configured text uses {0} for the operator's real name and {1} for the account name. + private fun serverText(key: String): String? { + val template = serverData.optJSONObject("soptions")?.optString(key, "") ?: "" + if (template.isEmpty()) return null + val username = serverData.optString("username") + val realname = serverData.optString("realname").ifEmpty { username } + return template.replace("{0}", realname).replace("{1}", username) + } + + // Toast for the server's notify flag, shown once the session actually starts. + fun notifySessionStart() { + if (!consentNotifyRequested()) return + val context = parent.parent.getApplicationContext() + val serverKey = if (usage == 5) "notifyMsgFiles" else "notifyMsgDesktop" + val defaultRes = if (usage == 5) R.string.files_session_started else R.string.desktop_session_started + parent.parent.showToastMessage(serverText(serverKey) ?: context.getString(defaultRes, remoteUserName(context))) + } + + fun logSessionEvent(id: Int, msg: String) { + parent.logServerEventEx(id, null, "$msg (${serverData.optString("remoteaddr")})", serverData) + } + + private fun startFilesSession() { + if (consentPromptRequired()) { + filesConsentPending = true + sendConsoleMessage(CONSENT_PENDING_MESSAGE, MSGID_CONSENT_PENDING) + AgentController.requestFilesConsent(this) + } else { + logSessionEvent(if (consentNotifyRequested()) 42 else 43, "Started remote files " + (if (consentNotifyRequested()) "with toast notification" else "without notification")) + notifySessionStart() + } + } + + fun approveFilesConsent() { + if (!filesConsentPending) return + filesConsentPending = false + logSessionEvent(40, "Starting remote files after local user accepted") + sendConsoleMessage(null) + notifySessionStart() + val held = heldFileCommands.toList() + heldFileCommands.clear() + if (held.isEmpty()) return + // Listings and transfers touch storage, so keep them off the main thread the approval came from. + thread(name = "MeshFilesConsent") { + for (command in held) { + try { + processTunnelData(command) + } catch (ex: Exception) { + println("Tunnel-Exception: $ex") + } + } + } + } + + fun denyFilesConsent() { + if (!filesConsentPending) return + filesConsentPending = false + heldFileCommands.clear() + logSessionEvent(41, "Failed to start remote files after local user rejected") + sendConsoleMessage("denied", MSGID_CONSENT_DENIED) + Stop() + } + + // Overlay text on this session's viewer. With msgid the web UI shows its own translated string, + // otherwise the text as given. A null msg clears the overlay; timeoutSeconds lets the viewer + // clear it by itself. + fun sendConsoleMessage(msg: String?, msgid: Int? = null, timeoutSeconds: Int? = null) { + val json = JSONObject() + json.put("type", "console") + json.put("msg", msg ?: JSONObject.NULL) + if (msg == null) { + json.put("msgid", 0) + } else if (msgid != null) { + json.put("msgid", msgid) + } + if (timeoutSeconds != null) json.put("timeout", timeoutSeconds) + sendCtrlResponse(json) } override fun onOpen(webSocket: WebSocket, response: Response) { @@ -222,32 +360,24 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo startConnectionTimer() if (usage == 2) { // If this is a remote desktop usage... - if (!g_autoConsent && !AgentController.isRemoteDesktopRunning()) { + if (consentPromptRequired() && !AgentController.isRemoteDesktopRunning()) { // Consent goes over this desktop tunnel so it reaches the viewer that opened it. - val json = JSONObject() - val msg = if (!AgentController.isAccessibilityServiceEnabled() && g_mainActivity == null) { - "Open the Android app to approve screen capture, or enable Accessibility Remote Control for unattended desktop." - } else { - "Waiting for user to grant access..." - } - json.put("type", "console") - json.put("msg", msg) - json.put("msgid", 1) - sendCtrlResponse(json) + sendConsoleMessage(CONSENT_PENDING_MESSAGE, MSGID_CONSENT_PENDING) + } else { + logSessionEvent(if (consentNotifyRequested()) 35 else 36, "Started remote desktop " + (if (consentNotifyRequested()) "with toast notification" else "without notification")) + notifySessionStart() } if (!AgentController.isRemoteDesktopRunning()) { parent.parent.startProjection() } else { - val json = JSONObject() - json.put("type", "console") - json.put("msg", null) - json.put("msgid", 0) - sendCtrlResponse(json) + sendConsoleMessage(null) // Send the display size and push a full frame of the current screen so the // reconnecting viewer sees it immediately instead of waiting for a change. updateDesktopDisplaySize() AgentController.requestDesktopRefresh() } + } else if (usage == 5) { + startFilesSession() } } else { // This is a file transfer @@ -276,28 +406,33 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo try { if (msg[0].toInt() == 123) { // If we are authenticated, process JSON data - processTunnelData(String(msg.toByteArray(), Charsets.UTF_8)) + val command = String(msg.toByteArray(), Charsets.UTF_8) + if (filesConsentPending) { + // Nothing runs until the device user approves; keep the request for then. + if (heldFileCommands.size < MAX_HELD_FILE_COMMANDS) heldFileCommands.add(command) + } else { + processTunnelData(command) + } } else if (fileUpload != null) { // If this is file upload data, process it here + val stream = fileUpload?.stream ?: return if (msg[0].toInt() == 0) { // If data starts with zero, skip the first byte. This is used to escape binary file data from JSON. fileUploadSize += (msg.size - 1); var buf = msg.toByteArray() try { - fileUpload?.write(buf, 1, buf.size - 1) + stream.write(buf, 1, buf.size - 1) } catch (ex : Exception) { - // Report a problem - uploadError() + uploadError("write failed, ${ex.message}") return } } else { // If data does not start with zero, save as-is. fileUploadSize += msg.size; try { - fileUpload?.write(msg.toByteArray()) + stream.write(msg.toByteArray()) } catch (ex : Exception) { - // Report a problem - uploadError() + uploadError("write failed, ${ex.message}") return } } @@ -308,7 +443,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo json.put("reqid", fileUploadReqId) if (_webSocket != null) { _webSocket?.send(json.toString().toByteArray().toByteString()) } } else { - if (msg.size < 2) return + if (msg.size < 4) return var cmd : Int = (msg[0].toInt() shl 8) + msg[1].toInt() var cmdsize : Int = (msg[2].toInt() shl 8) + msg[3].toInt() if (cmdsize != msg.size) return @@ -402,14 +537,15 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo } } - private fun uploadError() { + private fun uploadError(reason: String?) { val json = JSONObject() json.put("action", "uploaderror") json.put("reqid", fileUploadReqId) if (_webSocket != null) { _webSocket?.send(json.toString().toByteArray().toByteString()) } - try { fileUpload?.close() } catch (ex : Exception) {} + fileUpload?.discard() fileUpload = null - return + // The viewer only closes its dialog on uploaderror, so say why on the files overlay. + sendConsoleMessage("Upload of \"$fileUploadName\" failed: ${reason ?: "unknown error"}", timeoutSeconds = 20) } private fun processTunnelData(jsonStr: String) { @@ -441,88 +577,30 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo val filenames = json.getJSONArray("delfiles") deleteFile(path, filenames, json) } + "download" -> handleDownloadCommand(json) "upload" -> { - // {"action":"upload","reqid":0,"path":"Images","name":"00000000.JPG","size":1180231} + // {"action":"upload","reqid":0,"path":"Images","name":"00000000.JPG","size":1180231,"append":false} val path = json.getString("path") val name = json.getString("name") - //val size = json.getInt("size") val reqid = json.getInt("reqid") - - if (!isSafeFileName(name)) { - uploadError() - return - } + val append = json.optBoolean("append", false) // Close previous upload - if (fileUpload != null) { - fileUpload?.close() - fileUpload = null; - } + fileUpload?.discard() + fileUpload = null // Setup fileUploadName = name fileUploadReqId = reqid fileUploadSize = 0 - - if (path.startsWith("Sdcard")) { - val file = resolveSdcardChild(Environment.getExternalStorageDirectory(), path, name) - if (file == null) { - uploadError() - return - } - try { - fileUpload = FileOutputStream(file) - } catch (e: Exception) { - uploadError() - return - } - } else { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - val resolver: ContentResolver = parent.parent.contentResolver - val contentValues = ContentValues() - contentValues.put(MediaStore.MediaColumns.DISPLAY_NAME, name) - val (mimeType, relativePath, externalUri) = when { - name.lowercase().endsWith(".jpg") || name.lowercase().endsWith(".jpeg") -> Triple("image/jpg", Environment.DIRECTORY_PICTURES, MediaStore.Images.Media.EXTERNAL_CONTENT_URI) - name.lowercase().endsWith(".png") -> Triple("image/png", Environment.DIRECTORY_PICTURES, MediaStore.Images.Media.EXTERNAL_CONTENT_URI) - name.lowercase().endsWith(".bmp") -> Triple("image/bmp", Environment.DIRECTORY_PICTURES, MediaStore.Images.Media.EXTERNAL_CONTENT_URI) - name.lowercase().endsWith(".mp4") -> Triple("video/mp4", Environment.DIRECTORY_MOVIES, MediaStore.Video.Media.EXTERNAL_CONTENT_URI) - name.lowercase().endsWith(".mp3") -> Triple("audio/mpeg3", Environment.DIRECTORY_MUSIC, MediaStore.Audio.Media.EXTERNAL_CONTENT_URI) - name.lowercase().endsWith(".ogg") -> Triple("audio/ogg", Environment.DIRECTORY_MUSIC, MediaStore.Audio.Media.EXTERNAL_CONTENT_URI) - else -> { - println("Unsupported file type: $name") - Triple(null, null, null) - } - } - if (mimeType != null && relativePath != null && externalUri != null) { - contentValues.put(MediaStore.MediaColumns.MIME_TYPE, mimeType) - contentValues.put(MediaStore.MediaColumns.RELATIVE_PATH, relativePath) - val fileUri = resolver.insert(externalUri, contentValues) - try { - fileUpload = resolver.openOutputStream(fileUri!!) - } catch (e: Exception) { - uploadError() - return - } - } else { - uploadError() - return - } - } else { - val fileExtension = name.lowercase().substringAfterLast('.') - val fileDir: String = when (fileExtension) { - "jpg", "jpeg", "png" -> Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES).toString() - "mp4", "mkv" -> Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).toString() - "mp3", "wav" -> Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MUSIC).toString() - else -> Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS).toString() - } - val file = File(fileDir, name) - try { - fileUpload = FileOutputStream(file) - } catch (e: Exception) { - uploadError() - return - } - } + fileUpload = try { + UploadSink.open(parent.parent.contentResolver, path, name, append) + } catch (ex: UploadException) { + uploadError(ex.message) + return + } catch (ex: Exception) { + uploadError("cannot create \"$name\", ${ex.message}") + return } // Send response @@ -532,9 +610,15 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo if (_webSocket != null) { _webSocket?.send(respJson.toString().toByteArray().toByteString()) } } "uploaddone" -> { - if (fileUpload == null) return; - fileUpload?.close() - fileUpload = null; + val upload = fileUpload ?: return + fileUpload = null + try { + upload.finish() + } catch (ex: Exception) { + upload.discard() + uploadError("cannot save \"$fileUploadName\", ${ex.message}") + return + } // Send response val respJson = JSONObject() @@ -546,7 +630,31 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo var eventArgs = JSONArray() eventArgs.put(fileUploadName) eventArgs.put(fileUploadSize) - parent.logServerEventEx(105, eventArgs, "Upload: \"${fileUploadName}}\", Size: $fileUploadSize", serverData); + parent.logServerEventEx(105, eventArgs, "Upload: \"${fileUploadName}\", Size: $fileUploadSize", serverData); + } + "uploadcancel" -> { + val upload = fileUpload ?: return + fileUpload = null + upload.discard() + val respJson = JSONObject() + respJson.put("action", "uploadcancel") + respJson.put("reqid", fileUploadReqId) + if (_webSocket != null) { _webSocket?.send(respJson.toString().toByteArray().toByteString()) } + } + "uploadhash" -> { + // The viewer compares this with its local file to skip an identical upload. Without + // a reply its upload dialog waits forever. + val path = json.getString("path") + val name = json.getString("name") + val hash = hashExistingUpload(parent.parent.contentResolver, path, name) + val respJson = JSONObject() + respJson.put("action", "uploadhash") + respJson.put("reqid", json.get("reqid")) + respJson.put("path", path) + respJson.put("name", name) + respJson.put("tag", json.optJSONObject("tag") ?: JSONObject()) + respJson.put("hash", hash ?: JSONObject.NULL) + if (_webSocket != null) { _webSocket?.send(respJson.toString().toByteArray().toByteString()) } } else -> { // Unknown command, ignore it. @@ -573,6 +681,7 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo val mediaUri = uri ?: return r if (dir.startsWith("Sdcard")) { val directory = resolveSdcardPath(Environment.getExternalStorageDirectory(), dir) ?: return r + listIndexedFolder(parent.parent.contentResolver, dir)?.let { return it } val listOfFiles = directory.listFiles() for (file in listOfFiles.orEmpty()) { var f : JSONObject = JSONObject() @@ -581,7 +690,8 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo else f.put("t", 3) //f.put("t", 3) f.put("s", file.length()) - f.put("d", file.lastModified()) + // The web UI treats numeric dates as seconds. + f.put("d", file.lastModified() / 1000) r.put(f) } } else { @@ -600,8 +710,8 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo var f : JSONObject = JSONObject() f.put("n", cursor.getString(titleColumn)) f.put("t", 3) - f.put("s", cursor.getInt(sizeColumn)) - f.put("d", cursor.getInt(dateModified)) + f.put("s", cursor.getLong(sizeColumn)) + f.put("d", cursor.getLong(dateModified)) r.put(f) //println("${cursor.getString(titleColumn)}, ${cursor.getString(typeColumn)}") } @@ -727,113 +837,111 @@ class MeshTunnel(parent: MeshAgent, url: String, serverData: JSONObject) : WebSo } } + // Serves a file to the server's devicefile.ashx download over this usage-10 tunnel. fun startFileTransfer(filename: String) { - var filenameSplit = filename.split('/') - //println("startFileTransfer: $filenameSplit") + val name = filename.substringAfterLast('/') + val source = openSharedFile(parent.parent.contentResolver, filename.substringBeforeLast('/'), name) + if (source == null) { + parent.sendFilesMessage("Download of \"$name\" failed: the file is missing or Android does not let the agent read it", userid) + stopSocket() + return + } + val ws = _webSocket + if (ws == null) { + source.stream.close() + stopSocket() + return + } - val projection = arrayOf( - MediaStore.MediaColumns._ID, - MediaStore.MediaColumns.DISPLAY_NAME, - MediaStore.MediaColumns.SIZE - ) - var uri : Uri? = null; - if (filenameSplit[0].startsWith("Sdcard")) { uri = Uri.fromFile(Environment.getExternalStorageDirectory()) } - if (filenameSplit[0].equals("Images")) { uri = MediaStore.Images.Media.EXTERNAL_CONTENT_URI } - if (filenameSplit[0].equals("Audio")) { uri = MediaStore.Audio.Media.EXTERNAL_CONTENT_URI } - if (filenameSplit[0].equals("Videos")) { uri = MediaStore.Video.Media.EXTERNAL_CONTENT_URI } - //if (filenameSplit[0] == "Documents") { uri = MediaStore.Files. } - val mediaUri = uri ?: run { stopSocket(); return } - if (filenameSplit[0].startsWith("Sdcard")){ - val file = resolveSdcardPath(Environment.getExternalStorageDirectory(), filename) - ?: run { stopSocket(); return } - if (file.exists()) { - val fileName = file.name - val fileSize = file.length() - var eventArgs = JSONArray() - eventArgs.put(fileName) - eventArgs.put(fileSize) - parent.logServerEventEx(106, eventArgs, "Download: ${fileName}, Size: $fileSize", serverData); - val okJson = JSONObject() - okJson.put("op", "ok") - okJson.put("size", fileSize) - _webSocket?.send(okJson.toString()) - val contentUrl = Uri.fromFile(file) - try { - // Serve the file - parent.parent.contentResolver.openInputStream(contentUrl).use { stream -> - // Perform operation on stream - var buf = ByteArray(65535) - var len : Int - while (true) { - len = stream!!.read(buf, 0, 65535) - if (len <= 0) { stopSocket(); break; } // Stream is done - if (_webSocket == null) { stopSocket(); break; } // Web socket closed - _webSocket?.send(buf.toByteString(0, len)) - if ((_webSocket?.queueSize() ?: 0) > 655350) { Thread.sleep(100)} - } - } - return; - } catch (e: FileNotFoundException) { + val eventArgs = JSONArray() + eventArgs.put(source.name) + eventArgs.put(source.size) + parent.logServerEventEx(106, eventArgs, "Download: ${source.name}, Size: ${source.size}", serverData) + val okJson = JSONObject() + okJson.put("op", "ok") + okJson.put("size", source.size) + ws.send(okJson.toString()) + + // Stream on our own thread so the socket's reader keeps handling control frames and a + // close from the other side ends the transfer instead of waiting for it to finish. + thread(name = "MeshFileSend") { + try { + source.stream.use { stream -> + val buf = ByteArray(65535) + while (_webSocket === ws) { + val len = stream.read(buf, 0, buf.size) + if (len <= 0) break + ws.send(buf.toByteString(0, len)) + while ((_webSocket === ws) && (ws.queueSize() > 655350)) Thread.sleep(100) + } } - } else { - // file does not exist + } catch (ex: Exception) { + println("Tunnel-Exception: $ex") } - } else { - if (filenameSplit.size != 2 || !isSafeFileName(filenameSplit[1])) { - stopSocket() - return - } - parent.parent.contentResolver.query( - mediaUri, - projection, - null, - null, - null - )?.use { cursor -> - val idColumn: Int = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID) - val titleColumn: Int = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.DISPLAY_NAME) - val sizeColumn: Int = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.SIZE) - while (cursor.moveToNext()) { - var name = cursor.getString(titleColumn) - if (name == filenameSplit[1]) { - var contentUrl: Uri = ContentUris.withAppendedId(mediaUri, cursor.getLong(idColumn)) - var fileSize = cursor.getInt(sizeColumn) - - // Event to the server - var eventArgs = JSONArray() - eventArgs.put(filename) - eventArgs.put(fileSize) - parent.logServerEventEx(106, eventArgs, "Download: ${filename}, Size: $fileSize", serverData); - val okJson = JSONObject() - okJson.put("op", "ok") - okJson.put("size", fileSize) - _webSocket?.send(okJson.toString()) - - // Serve the file - parent.parent.contentResolver.openInputStream(contentUrl).use { stream -> - // Perform operation on stream - var buf = ByteArray(65535) - var len : Int - while (true) { - len = stream!!.read(buf, 0, 65535) - if (len <= 0) { - stopSocket() - break - } // Stream is done - if (_webSocket == null) { - stopSocket() - break - } // Web socket closed - _webSocket?.send(buf.toByteString(0, len)) - if ((_webSocket?.queueSize() ?: 0) > 655350) { Thread.sleep(100)} - } - } - return; + stopSocket() + } + } + + // The viewer's file editor pulls files over the files session in blocks, each prefixed with a + // 4-byte flag word whose low bit marks the last block. + private var blockDownload: SharedFile? = null + private var blockDownloadId: String? = null + + private fun handleDownloadCommand(json: JSONObject) { + val id = json.opt("id") + when (json.optString("sub")) { + "start" -> { + closeBlockDownload() + val path = json.optString("path") + val file = openSharedFile(parent.parent.contentResolver, path.substringBeforeLast('/'), path.substringAfterLast('/')) + val reply = JSONObject() + reply.put("action", "download") + reply.put("id", id) + if (file == null) { + reply.put("sub", "cancel") + } else { + blockDownload = file + blockDownloadId = id?.toString() + reply.put("sub", "start") + } + if (_webSocket != null) { _webSocket?.send(reply.toString().toByteArray().toByteString()) } + } + "startack", "ack" -> { + val file = blockDownload ?: return + if (id?.toString() != blockDownloadId) return + val buf = ByteArray(4 + DOWNLOAD_BLOCK_SIZE) + var len = 0 + try { + while (len < DOWNLOAD_BLOCK_SIZE) { + val read = file.stream.read(buf, 4 + len, DOWNLOAD_BLOCK_SIZE - len) + if (read <= 0) break + len += read } + } catch (ex: Exception) { + closeBlockDownload() + val reply = JSONObject() + reply.put("action", "download") + reply.put("sub", "cancel") + reply.put("id", id) + if (_webSocket != null) { _webSocket?.send(reply.toString().toByteArray().toByteString()) } + return } + val last = len < DOWNLOAD_BLOCK_SIZE + buf[0] = 1 + buf[1] = 0 + buf[2] = 0 + buf[3] = if (last) 1 else 0 + if (_webSocket != null) { _webSocket?.send(buf.toByteString(0, 4 + len)) } + if (last) closeBlockDownload() } + "cancel" -> closeBlockDownload() } - stopSocket() + } + + private fun closeBlockDownload() { + try { blockDownload?.stream?.close() } catch (ex: Exception) { } + blockDownload = null + blockDownloadId = null } override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { diff --git a/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt b/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt index b553596..1bd08c2 100644 --- a/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt +++ b/app/src/main/java/com/meshcentral/agent/ScreenCaptureService.kt @@ -86,6 +86,12 @@ class ScreenCaptureService : Service(), RemoteDesktopProvider { var bitmap = Bitmap.createBitmap(mWidth + rowPadding / pixelStride, mHeight, Bitmap.Config.ARGB_8888) bitmap.copyPixelsFromBuffer(buffer) + if (rowPadding > 0) { + // The stride padding would otherwise be streamed as garbage columns past the screen edge. + val cropped = Bitmap.createBitmap(bitmap, 0, 0, mWidth, mHeight) + bitmap.recycle() + bitmap = cropped + } if (g_desktop_scalingLevel != 1024 && g_desktop_scalingLevel > 0) { val newWidth = max(1, (mWidth * g_desktop_scalingLevel) / 1024) @@ -251,6 +257,7 @@ class ScreenCaptureService : Service(), RemoteDesktopProvider { g_ScreenCaptureService = this g_remoteDesktopProvider = this updateTunnelDisplaySize() + AgentController.desktopProviderStarted() sendAgentConsole("Started display sharing") } } diff --git a/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt b/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt index 9ea4956..36ecb3b 100644 --- a/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt +++ b/app/src/main/java/com/meshcentral/agent/SettingsFragment.kt @@ -37,6 +37,21 @@ class SettingsFragment : PreferenceFragmentCompat() { } true } + findPreference("pref_all_files_access")?.apply { + isVisible = AgentController.allFilesAccessAvailable + setOnPreferenceClickListener { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + try { + val intent = Intent(Settings.ACTION_MANAGE_APP_ALL_FILES_ACCESS_PERMISSION) + intent.data = Uri.parse("package:${requireContext().packageName}") + startActivity(intent) + } catch (ex: Exception) { + startActivity(Intent(Settings.ACTION_MANAGE_ALL_FILES_ACCESS_PERMISSION)) + } + } + true + } + } findPreference("pref_notification_permission")?.setOnPreferenceClickListener { val intent = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) @@ -57,6 +72,12 @@ class SettingsFragment : PreferenceFragmentCompat() { refreshStatus() } + // Summaries reflect system settings the user may just have changed on another screen. + override fun onResume() { + super.onResume() + refreshStatus() + } + override fun onDestroy() { if (settingsFragment === this) settingsFragment = null g_mainActivity?.settingsChanged() @@ -81,6 +102,12 @@ class SettingsFragment : PreferenceFragmentCompat() { } else { getString(R.string.battery_optimization_summary) } + findPreference("pref_all_files_access")?.summary = + if (AgentController.hasAllFilesAccess()) { + getString(R.string.ready) + } else { + getString(R.string.all_files_access_summary) + } findPreference("pref_notification_permission")?.summary = if (AgentController.areNotificationsEnabled()) { getString(R.string.ready) diff --git a/app/src/main/java/com/meshcentral/agent/UploadStorage.kt b/app/src/main/java/com/meshcentral/agent/UploadStorage.kt new file mode 100644 index 0000000..7965320 --- /dev/null +++ b/app/src/main/java/com/meshcentral/agent/UploadStorage.kt @@ -0,0 +1,322 @@ +package com.meshcentral.agent + +import android.annotation.SuppressLint +import android.content.ContentResolver +import android.content.ContentUris +import android.content.ContentValues +import android.net.Uri +import android.os.Build +import android.os.Environment +import android.os.ParcelFileDescriptor +import android.provider.MediaStore +import android.webkit.MimeTypeMap +import androidx.annotation.RequiresApi +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.InputStream +import java.io.OutputStream +import java.security.MessageDigest +import org.json.JSONArray +import org.json.JSONObject + +// The message is shown on the viewer's files overlay, so keep it operator-friendly. +internal class UploadException(message: String) : Exception(message) + +// Top-level shared-storage folders a scoped-storage app may create files in. Anything else under +// the storage root is off limits without MANAGE_EXTERNAL_STORAGE. +internal val standardSharedFolders = setOf( + "Alarms", "Audiobooks", "DCIM", "Documents", "Download", "Movies", "Music", + "Notifications", "Pictures", "Podcasts", "Recordings", "Ringtones" +) + +// "Sdcard/Download/sub" -> "Download/sub", "Sdcard" -> "", anything else -> null. +internal fun sdcardRelativeDirectory(virtualDir: String): String? { + if (virtualDir != "Sdcard" && !virtualDir.startsWith("Sdcard/")) return null + return virtualDir.removePrefix("Sdcard").trim('/') +} + +internal fun isStandardSharedFolder(relativeDir: String): Boolean { + return standardSharedFolders.contains(relativeDir.substringBefore('/')) +} + +// Media-only top-level folders, where Android's own index holds every file. Walking such a folder +// through FUSE is slow (4 s for a 14k-photo camera roll) while the index answers in about a second; +// Download and Documents stay on the directory walk because they hold other apps' unindexed files. +internal val mediaOnlySharedFolders = setOf( + "Alarms", "Audiobooks", "DCIM", "Movies", "Music", "Notifications", "Pictures", "Podcasts", + "Recordings", "Ringtones" +) + +internal fun isMediaOnlySharedFolder(relativeDir: String): Boolean { + return relativeDir.isNotEmpty() && mediaOnlySharedFolders.contains(relativeDir.substringBefore('/')) +} + +// The folder's entries in the file browser's format from MediaStore, or null when the index can't +// be used (older Android, not a media folder, query failure, nothing indexed) so the caller walks +// the directory instead. +@SuppressLint("InlinedApi") +internal fun listIndexedFolder(resolver: ContentResolver, virtualDir: String): JSONArray? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) return null + val relativeDir = sdcardRelativeDirectory(virtualDir) ?: return null + if (!isMediaOnlySharedFolder(relativeDir)) return null + val projection = arrayOf( + MediaStore.MediaColumns.DISPLAY_NAME, + MediaStore.MediaColumns.SIZE, + MediaStore.MediaColumns.DATE_MODIFIED, + MediaStore.MediaColumns.MIME_TYPE + ) + val result = JSONArray() + try { + val cursor = resolver.query( + MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY), + projection, + "${MediaStore.MediaColumns.RELATIVE_PATH} = ?", + arrayOf(relativeDir.trim('/') + "/"), + null + ) ?: return null + cursor.use { + val nameColumn = it.getColumnIndexOrThrow(MediaStore.MediaColumns.DISPLAY_NAME) + val sizeColumn = it.getColumnIndexOrThrow(MediaStore.MediaColumns.SIZE) + val dateColumn = it.getColumnIndexOrThrow(MediaStore.MediaColumns.DATE_MODIFIED) + val mimeColumn = it.getColumnIndexOrThrow(MediaStore.MediaColumns.MIME_TYPE) + while (it.moveToNext()) { + val name = it.getString(nameColumn) ?: continue + val entry = JSONObject() + entry.put("n", name) + // Subfolders are the only indexed rows without a MIME type. + entry.put("t", if (it.isNull(mimeColumn)) 2 else 3) + entry.put("s", it.getLong(sizeColumn)) + entry.put("d", it.getLong(dateColumn)) + result.put(entry) + } + } + } catch (ex: Exception) { + return null + } + return if (result.length() == 0) null else result +} + +// The flat media folders offered at the root of the file browser, backed by MediaStore. +internal enum class MediaFolder( + val virtualName: String, + private val mimePrefix: String, + val directory: String, + val label: String +) { + IMAGES("Images", "image/", "Pictures", "image"), + AUDIO("Audio", "audio/", "Music", "audio"), + VIDEOS("Videos", "video/", "Movies", "video"); + + fun accepts(mimeType: String): Boolean = mimeType.startsWith(mimePrefix) + + // Resolved on demand: the Uri constants need the Android runtime, which unit tests lack. + fun collectionUri(): Uri = when (this) { + IMAGES -> MediaStore.Images.Media.EXTERNAL_CONTENT_URI + AUDIO -> MediaStore.Audio.Media.EXTERNAL_CONTENT_URI + VIDEOS -> MediaStore.Video.Media.EXTERNAL_CONTENT_URI + } + + companion object { + fun fromVirtualName(name: String): MediaFolder? = values().firstOrNull { it.virtualName == name } + } +} + +internal fun mimeTypeForFileName(name: String): String { + val extension = name.substringAfterLast('.', "").lowercase() + val mapped = if (extension.isEmpty()) null else MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension) + return mapped ?: "application/octet-stream" +} + +// An open upload target: a plain file where Android still allows one, otherwise a MediaStore row. +internal class UploadSink private constructor( + val stream: OutputStream, + private val file: File?, + private val resolver: ContentResolver?, + private val uri: Uri?, + // This upload created the file or row, so a failed transfer removes it again. + private val created: Boolean, + // A row inserted as pending stays hidden from other apps until finish(). + private val pending: Boolean +) { + fun finish() { + stream.close() + if (pending && resolver != null && uri != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val values = ContentValues() + values.put(MediaStore.MediaColumns.IS_PENDING, 0) + resolver.update(uri, values, null, null) + } + } + + fun discard() { + try { stream.close() } catch (ex: Exception) { } + if (!created) return + try { + if (file != null) { + file.delete() + } else if (resolver != null && uri != null) { + resolver.delete(uri, null, null) + } + } catch (ex: Exception) { } + } + + companion object { + fun open(resolver: ContentResolver, path: String, name: String, append: Boolean): UploadSink { + if (!isSafeFileName(name)) throw UploadException("invalid file name") + val mimeType = mimeTypeForFileName(name) + + val media = MediaFolder.fromVirtualName(path) + if (media != null) { + if (!media.accepts(mimeType)) { + throw UploadException("only ${media.label} files can go in ${media.virtualName}, use Sdcard/Download for other files") + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + return openMediaStore(resolver, media.collectionUri(), media.directory, name, mimeType, append, matchDirectory = false) + } + @Suppress("DEPRECATION") + return openRawFile(File(Environment.getExternalStoragePublicDirectory(media.directory), name), append) + } + + val relativeDir = sdcardRelativeDirectory(path) ?: throw UploadException("unknown folder \"$path\"") + val file = resolveSdcardChild(Environment.getExternalStorageDirectory(), path, name) + ?: throw UploadException("invalid path") + val rawError = try { + return openRawFile(file, append) + } catch (ex: Exception) { + ex + } + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { + throw UploadException("cannot write \"$name\", ${rawError.message}") + } + if (!isStandardSharedFolder(relativeDir)) { + val hint = if (AgentController.allFilesAccessAvailable) ", or grant All files access in the app settings" else "" + throw UploadException("Android only lets the agent write inside standard folders such as Download, Documents, Pictures, Movies or Music, pick one of those under Sdcard$hint") + } + return openMediaStore(resolver, MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY), relativeDir, name, mimeType, append, matchDirectory = true) + } + + private fun openRawFile(file: File, append: Boolean): UploadSink { + val existed = file.exists() + val stream = FileOutputStream(file, append) + return UploadSink(stream, file, null, null, created = !existed, pending = false) + } + + @RequiresApi(Build.VERSION_CODES.Q) + private fun openMediaStore( + resolver: ContentResolver, + collection: Uri, + directory: String, + name: String, + mimeType: String, + append: Boolean, + matchDirectory: Boolean + ): UploadSink { + val relativePath = directory.trim('/') + "/" + val existing = findMediaRow(resolver, collection, if (matchDirectory) relativePath else null, name) + if (existing != null) { + val stream = try { + resolver.openOutputStream(existing, if (append) "wa" else "wt") + } catch (ex: SecurityException) { + throw UploadException("\"$name\" belongs to another app and cannot be replaced") + } ?: throw UploadException("cannot open \"$name\"") + return UploadSink(stream, null, resolver, existing, created = false, pending = false) + } + + val values = ContentValues() + values.put(MediaStore.MediaColumns.DISPLAY_NAME, name) + values.put(MediaStore.MediaColumns.MIME_TYPE, mimeType) + values.put(MediaStore.MediaColumns.RELATIVE_PATH, relativePath) + values.put(MediaStore.MediaColumns.IS_PENDING, 1) + val uri = try { + resolver.insert(collection, values) + } catch (ex: Exception) { + // MediaProvider refuses, for example, a text file in Pictures. + throw UploadException("Android does not allow $mimeType files in $directory, use Download or Documents") + } ?: throw UploadException("cannot create \"$name\" in $directory") + val stream = try { + resolver.openOutputStream(uri, "w") + } catch (ex: Exception) { + null + } + if (stream == null) { + try { resolver.delete(uri, null, null) } catch (ex: Exception) { } + throw UploadException("cannot open \"$name\" for writing") + } + return UploadSink(stream, null, resolver, uri, created = true, pending = true) + } + + @SuppressLint("InlinedApi") + fun findMediaRow(resolver: ContentResolver, collection: Uri, relativePath: String?, name: String): Uri? { + val selection = if (relativePath == null) { + "${MediaStore.MediaColumns.DISPLAY_NAME} = ?" + } else { + "${MediaStore.MediaColumns.RELATIVE_PATH} = ? AND ${MediaStore.MediaColumns.DISPLAY_NAME} = ?" + } + val args = if (relativePath == null) arrayOf(name) else arrayOf(relativePath, name) + return try { + resolver.query(collection, arrayOf(MediaStore.MediaColumns._ID), selection, args, null)?.use { cursor -> + if (cursor.moveToFirst()) ContentUris.withAppendedId(collection, cursor.getLong(0)) else null + } + } catch (ex: Exception) { + null + } + } + } +} + +internal class SharedFile(val name: String, val size: Long, val stream: InputStream) + +// Opens a file the viewer named by virtual folder and name for reading, wherever Android lets the +// agent reach it: the raw path, or the MediaStore row for media and for files the agent created. +internal fun openSharedFile(resolver: ContentResolver, path: String, name: String): SharedFile? { + if (!isSafeFileName(name)) return null + val media = MediaFolder.fromVirtualName(path) + if (media != null) { + val uri = UploadSink.findMediaRow(resolver, media.collectionUri(), null, name) ?: return null + return openMediaRow(resolver, uri, name) + } + val file = resolveSdcardChild(Environment.getExternalStorageDirectory(), path, name) ?: return null + if (file.isFile) { + try { + return SharedFile(name, file.length(), FileInputStream(file)) + } catch (ex: Exception) { + // Another app's file may still be reachable through its MediaStore row below. + } + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val relativeDir = sdcardRelativeDirectory(path) ?: return null + val collection = MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) + val uri = UploadSink.findMediaRow(resolver, collection, relativeDir.trim('/') + "/", name) ?: return null + return openMediaRow(resolver, uri, name) + } + return null +} + +private fun openMediaRow(resolver: ContentResolver, uri: Uri, name: String): SharedFile? { + return try { + val descriptor = resolver.openFileDescriptor(uri, "r") ?: return null + SharedFile(name, descriptor.statSize.coerceAtLeast(0), ParcelFileDescriptor.AutoCloseInputStream(descriptor)) + } catch (ex: Exception) { + null + } +} + +// SHA-384 of the file the viewer is about to replace, upper-case hex to match the web UI's +// comparison, or null when nothing readable is there. The viewer skips identical uploads. +internal fun hashExistingUpload(resolver: ContentResolver, path: String, name: String): String? { + val input = try { openSharedFile(resolver, path, name)?.stream } catch (ex: Exception) { null } ?: return null + return try { + input.use { stream -> + val digest = MessageDigest.getInstance("SHA-384") + val buffer = ByteArray(65536) + while (true) { + val read = stream.read(buffer) + if (read <= 0) break + digest.update(buffer, 0, read) + } + digest.digest().joinToString("") { "%02X".format(it) } + } + } catch (ex: Exception) { + null + } +} diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index e433911..2ab039c 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -36,6 +36,51 @@ Automatische Zustimmung Dem Remote Agent automatisch Zustimmung erteilen Immer Zustimmung für Verbindung anfordern + Unbeaufsichtigter Zugriff + Fernsteuerung über Bedienungshilfen + Einmal in den Android-Einstellungen aktivieren, um den Bildschirm unbeaufsichtigt zu teilen und Eingaben aus der Ferne zu erlauben + Akku-Optimierung + Hintergrundbetrieb erlauben, damit die App zuverlässig startet und die Verbindung wiederherstellt + Benachrichtigungen + Statusbenachrichtigung des Vordergrunddienstes erlauben + Zugriff auf alle Dateien + Durchsuchen und Übertragen aller Dateien im gemeinsamen Speicher erlauben + Autostart + Startet nach einem Neustart automatisch, sobald das Gerät gekoppelt ist + Bereit + Einrichtung erforderlich + Unbeaufsichtigter Zugriff erforderlich + Öffnen Sie MeshCentral Agent, um die Einrichtung des unbeaufsichtigten Zugriffs abzuschließen + Erlaubt MeshCentral Agent nach der Einrichtung auf verwalteten Geräten, den Bildschirm zu teilen und Eingaben aus der Ferne auszuführen. + Durch Unternehmensrichtlinie festgelegt + Einrichtung des unbeaufsichtigten Zugriffs abschließen + Dieser Build ist %1$s.\n\nFehlende Einrichtung:\n%2$s\n\nDer Remote-Desktop zeigt einen schwarzen Bildschirm, bis die Fernsteuerung über Bedienungshilfen aktiviert ist oder die App geöffnet ist, damit Android nach der Zustimmung zur Bildschirmaufnahme fragen kann. + Bedienungshilfen öffnen + App-Einstellungen öffnen + Später + Diesen Bildschirm teilen? + Ein Remote-Benutzer möchte diesen Bildschirm sehen.\n\nFür unbeaufsichtigte Fernsteuerung aktivieren Sie die Fernsteuerung über Bedienungshilfen (empfohlen). Andernfalls können Sie die Android-Bildschirmaufnahme für diese Sitzung erlauben. + Bildschirm teilen + Fernsteuerung über Bedienungshilfen + uneingeschränkter Akku-Modus + Berechtigung für Benachrichtigungen + Einrichtung prüfen + Die Einrichtung des unbeaufsichtigten Zugriffs ist abgeschlossen. + Verbindungsbenachrichtigung + Benachrichtigung anzeigen, solange jemand verbunden ist + Keine Benachrichtigung, solange jemand verbunden ist + Remote-Sitzung aktiv + Verbunden: %1$s + %1$d Benutzer verbunden + Ein Remote-Benutzer + %1$s möchte diesen Bildschirm für diese Sitzung sehen und steuern. Bildschirmfreigabe erlauben? + Bildschirmfreigabe genehmigen + Dateizugriff genehmigen + %1$s möchte in dieser Sitzung Dateien auf diesem Gerät durchsuchen und übertragen. Dateizugriff erlauben? + %1$s hat eine Remote-Desktop-Sitzung gestartet. + %1$s hat eine Remote-Dateisitzung gestartet. + Genehmigen + Ablehnen Verbinden mit: %1$s? Server-Setup löschen? Server-Kopplungslink diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 96f9a9d..3e84e18 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -45,6 +45,8 @@ Allow background operation for reliable startup and reconnect Notifications Allow the foreground service status notification + All files access + Allow browsing and transferring any file on shared storage Startup Starts automatically after reboot when paired Ready @@ -72,9 +74,13 @@ Remote session active Connected %1$s %1$d users connected - A remote user wants to view and control this screen for this session. Allow screen sharing? + A remote user + %1$s wants to view and control this screen for this session. Allow screen sharing? Approve screen sharing - A remote user is requesting to view and control this screen. + Approve file access + %1$s wants to browse and transfer files on this device for this session. Allow file access? + %1$s started a remote desktop session. + %1$s started a remote files session. Approve Deny Setup to: %1$s? diff --git a/app/src/main/res/xml/root_preferences.xml b/app/src/main/res/xml/root_preferences.xml index bf85594..a49391e 100644 --- a/app/src/main/res/xml/root_preferences.xml +++ b/app/src/main/res/xml/root_preferences.xml @@ -37,6 +37,10 @@ app:key="pref_battery_optimization" app:title="@string/battery_optimization" app:summary="@string/battery_optimization_summary" /> + Date: Wed, 16 Sep 2026 07:31:45 +1000 Subject: [PATCH 9/9] Introduce enhanced input and drag selection support, improve accessibility service text handling, and optimise screen capture flow - Add drag-to-select functionality for text fields, including character bounds calculation and shift-based extension. - Improve text field handling with shadow state management to prevent input inconsistencies. - Refactor legacy key handling for modifiers, shortcuts, and focused actions, including robust Ctrl-based operations. - Optimise screenshot capture with sequence validation and fallback handling to address dropped requests. - Replace adler32 checks with FNV-1a hashing for improved tile hash accuracy. - Expand unit test coverage (`TileHashTest`) to validate hash and change detection logic. --- .../meshcentral/agent/DesktopFrameEncoder.kt | 86 ++-- .../agent/MeshAccessibilityService.kt | 380 ++++++++++++++++-- .../res/xml/mesh_accessibility_service.xml | 2 +- .../com/meshcentral/agent/TileHashTest.kt | 25 ++ docs/remote-desktop.md | 12 +- 5 files changed, 442 insertions(+), 63 deletions(-) create mode 100644 app/src/test/java/com/meshcentral/agent/TileHashTest.kt diff --git a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt index e1d9c9e..87eefc5 100644 --- a/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt +++ b/app/src/main/java/com/meshcentral/agent/DesktopFrameEncoder.kt @@ -6,6 +6,16 @@ import okio.ByteString.Companion.toByteString import java.io.ByteArrayOutputStream import java.io.DataOutputStream +// FNV-1a over the pixel words with an extra shift for diffusion. The previous checksum was a +// miscoded Adler-32 that swapped its halves every step and fed signed pixels through a signed +// remainder, which made it a weaker change detector than intended. +internal const val TILE_HASH_SEED = 0x811C9DC5.toInt() + +internal fun tileHash(pixel: Int, state: Int): Int { + val h = (state xor pixel) * 16777619 + return h xor (h ushr 15) +} + class DesktopFrameEncoder { private var tilesWide: Int = 0 private var tilesHigh: Int = 0 @@ -14,7 +24,7 @@ class DesktopFrameEncoder { private var tilesCount: Int = 0 private var oldcrcs: IntArray? = null private var newcrcs: IntArray? = null - // Reused per-tile pixel scratch, so a full-screen CRC pass doesn't allocate one array per tile. + // Reused per-tile pixel scratch, so a full-screen hash pass doesn't allocate one array per tile. private val tilePixels = IntArray(64 * 64) // Written from the tunnel/main thread, read on the capture thread. @Volatile private var forceFullFrame = true @@ -35,7 +45,7 @@ class DesktopFrameEncoder { forceFullFrame = true } - computeAllCRCs(bitmap) + computeAllHashes(bitmap) var changedTiles = 0 for (i in 0 until tilesCount) { if (forceFullFrame || oldcrcs!![i] != newcrcs!![i]) changedTiles++ @@ -43,7 +53,9 @@ class DesktopFrameEncoder { if (changedTiles == 0) return false if (forceFullFrame || ((changedTiles * 100) >= (tilesCount * 85))) { - sink(buildImageCommand(bitmap, 0, 0, bitmap.width, bitmap.height)) + // A failed encode leaves forceFullFrame set, so the next capture retries as a full frame. + val command = buildImageCommand(bitmap, 0, 0, bitmap.width, bitmap.height) ?: return false + sink(command) for (i in 0 until tilesCount) oldcrcs!![i] = newcrcs!![i] forceFullFrame = false return true @@ -77,7 +89,6 @@ class DesktopFrameEncoder { if (sendx != -1) { sendSubBitmapRow(bitmap, sendx, sendy, sendw, sink) } - forceFullFrame = false return true } @@ -103,11 +114,17 @@ class DesktopFrameEncoder { h++ } h -= y - sink(buildImageCommand(bitmap, x * 64, y * 64, w * 64, h * 64)) + val command = buildImageCommand(bitmap, x * 64, y * 64, w * 64, h * 64) + if (command == null) { + // These tiles are already marked as sent; resend the whole screen next time instead. + forceFullFrame = true + return + } + sink(command) } - private fun computeAllCRCs(bitmap: Bitmap) { - for (i in 0 until tilesCount) newcrcs!![i] = 1 + private fun computeAllHashes(bitmap: Bitmap) { + for (i in 0 until tilesCount) newcrcs!![i] = TILE_HASH_SEED for (y in 0 until tilesHigh) { var h = 64 if (((y * 64) + 64) > bitmap.height) h = bitmap.height - (y * 64) @@ -117,23 +134,31 @@ class DesktopFrameEncoder { val t = (y * tilesWide) + x val count = w * h bitmap.getPixels(tilePixels, 0, w, x * 64, y * 64, w, h) - var crc = newcrcs!![t] - for (i in 0 until count) crc = adler32(tilePixels[i], crc) - newcrcs!![t] = crc + var hash = newcrcs!![t] + for (i in 0 until count) hash = tileHash(tilePixels[i], hash) + newcrcs!![t] = hash } } } - private fun buildImageCommand(bitmap: Bitmap, x: Int, y: Int, w: Int, h: Int): ByteString { + // Null when the region can't be encoded, so callers never ship an empty image that would jam + // the viewer's in-order tile queue. + private fun buildImageCommand(bitmap: Bitmap, x: Int, y: Int, w: Int, h: Int): ByteString? { var ww = w var hh = h if (x + w > bitmap.width) ww = bitmap.width - x if (y + h > bitmap.height) hh = bitmap.height - y - val croppedBitmap = if (x == 0 && y == 0 && ww == bitmap.width && hh == bitmap.height) { - bitmap - } else { - Bitmap.createBitmap(bitmap, x, y, ww, hh) + val croppedBitmap = try { + if (x == 0 && y == 0 && ww == bitmap.width && hh == bitmap.height) { + bitmap + } else { + Bitmap.createBitmap(bitmap, x, y, ww, hh) + } + } catch (ex: Exception) { + return null } + // The screen is opaque; without this PNG and WebP would carry a pointless alpha plane. + croppedBitmap.setHasAlpha(false) val bytesOut = ByteArrayOutputStream() val dos = DataOutputStream(bytesOut) @@ -144,19 +169,24 @@ class DesktopFrameEncoder { dos.writeShort(0) dos.writeShort(x) dos.writeShort(y) - when (g_desktop_imageType) { - 4 -> { - if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { - croppedBitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, g_desktop_compressionLevel, dos) - } else { - @Suppress("DEPRECATION") - croppedBitmap.compress(Bitmap.CompressFormat.WEBP, g_desktop_compressionLevel, dos) + val encoded = try { + when (g_desktop_imageType) { + 4 -> { + if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.R) { + croppedBitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, g_desktop_compressionLevel, dos) + } else { + @Suppress("DEPRECATION") + croppedBitmap.compress(Bitmap.CompressFormat.WEBP, g_desktop_compressionLevel, dos) + } } + 2 -> croppedBitmap.compress(Bitmap.CompressFormat.PNG, g_desktop_compressionLevel, dos) + else -> croppedBitmap.compress(Bitmap.CompressFormat.JPEG, g_desktop_compressionLevel, dos) } - 2 -> croppedBitmap.compress(Bitmap.CompressFormat.PNG, g_desktop_compressionLevel, dos) - else -> croppedBitmap.compress(Bitmap.CompressFormat.JPEG, g_desktop_compressionLevel, dos) + } catch (ex: Exception) { + false } if (croppedBitmap !== bitmap) croppedBitmap.recycle() + if (!encoded) return null val data = bytesOut.toByteArray() val cmdSize = data.size - 8 @@ -166,12 +196,4 @@ class DesktopFrameEncoder { data[7] = cmdSize.toByte() return data.toByteString() } - - private fun adler32(n: Int, state: Int): Int { - var a = state shr 16 - var b = state and 0xFFFF - a = (a + n) % 65521 - b = (b + a) % 65521 - return (b shl 16) + a - } } diff --git a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt index 944b993..87ae856 100644 --- a/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt +++ b/app/src/main/java/com/meshcentral/agent/MeshAccessibilityService.kt @@ -2,24 +2,29 @@ package com.meshcentral.agent import android.accessibilityservice.AccessibilityService import android.accessibilityservice.GestureDescription +import android.accessibilityservice.InputMethod import android.app.KeyguardManager import android.content.Context import android.content.Intent import android.content.pm.PackageManager import android.graphics.Bitmap import android.graphics.Path +import android.graphics.Rect +import android.graphics.RectF import android.os.Build import android.os.Bundle import android.os.Handler import android.os.Looper import android.os.SystemClock import android.view.Display +import android.view.KeyEvent import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import androidx.annotation.RequiresApi import okio.ByteString import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import kotlin.math.absoluteValue import kotlin.math.max import kotlin.math.min @@ -37,6 +42,9 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { @Volatile private var nextFrameDelayMs = MIN_FRAME_DELAY_MS @Volatile private var screenshotErrorNotified = false @Volatile private var lastCaptureUptimeMs = 0L + // Each screenshot request gets a sequence number; a late or duplicate answer for an older + // request is dropped so it can't paint over a newer frame. + @Volatile private var captureSequence = 0 // Pointer input is streamed as continued strokes: button down puts a finger on the screen, // each move drags it and button up lifts it. Drags happen live, holding the button is a long @@ -62,6 +70,23 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { // What the operator typed into the focused password field; Android masks the field's own text. private val passwordBuffer = StringBuilder() private var passwordNodeKey: String? = null + // Modifier keys arrive as their own key messages; remembered for shortcuts and shift-selection. + private var shiftHeld = false + private var ctrlHeld = false + private var altHeld = false + // The last text written to a field with SET_TEXT, used while the app is still applying it. + private var shadowNodeKey: String? = null + private var shadowText: String? = null + private var shadowCursor = 0 + private var shadowUptimeMs = 0L + // Mouse-style text selection in progress (see beginDragSelect). + private var dragSelectNode: AccessibilityNodeInfo? = null + private var dragSelectRects: List? = null + private var dragSelectAnchor = -1 + private var dragSelectLastFocus = -1 + private var dragSelectActive = false + private var dragDownX = 0 + private var dragDownY = 0 override val isRunning: Boolean get() = active @@ -122,6 +147,9 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { active = false mainHandler.removeCallbacks(captureRunnable) releaseHeldPointer() + shiftHeld = false + ctrlHeld = false + altHeld = false if (g_remoteDesktopProvider === this) { g_remoteDesktopProvider = null } @@ -218,6 +246,10 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { // A second down without an up means the release was lost: lift, then redo it. inputSteps.addFirst(step) liftPointer(heldX.toInt(), heldY.toInt()) + } else if (beginDragSelect(step.x, step.y)) { + // Nothing touches the screen yet: a move makes this a selection, a release + // becomes a tap, and a hold becomes a real press once the deferral expires. + mainHandler.postDelayed(deferredPress, DRAG_SELECT_DEFER_MS) } else { pressPointer(step.x, step.y) } @@ -228,11 +260,33 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { while (inputSteps.firstOrNull() is InputStep.Move) { move = inputSteps.removeFirst() as InputStep.Move } - if (heldStroke != null && (move.x.toFloat() != heldX || move.y.toFloat() != heldY)) { + if (dragSelectActive) { + updateDragSelect(move.x, move.y) + } else if (dragSelectNode != null) { + if ((move.x - dragDownX).absoluteValue > DRAG_SELECT_SLOP || (move.y - dragDownY).absoluteValue > DRAG_SELECT_SLOP) { + mainHandler.removeCallbacks(deferredPress) + dragSelectActive = true + println("dragSelect: selecting from offset $dragSelectAnchor") + updateDragSelect(move.x, move.y) + } + } else if (heldStroke != null && (move.x.toFloat() != heldX || move.y.toFloat() != heldY)) { movePointer(move.x, move.y) } } - is InputStep.Up -> if (heldStroke != null) liftPointer(step.x, step.y) + is InputStep.Up -> { + if (dragSelectNode != null && !dragSelectActive) { + // Released without moving: deliver the click as a tap now. + val x = dragDownX + val y = dragDownY + endDragSelect() + recentTapUptimes.addLast(SystemClock.uptimeMillis()) + while (recentTapUptimes.size > 4) recentTapUptimes.removeFirst() + tapGesture(x, y)?.let { dispatchQueued(it) } + } else { + endDragSelect() + if (heldStroke != null) liftPointer(step.x, step.y) + } + } is InputStep.DoubleTap -> { // The viewer sends both clicks as down/up pairs before this flag, so only // synthesize the taps when they didn't come through. @@ -341,10 +395,9 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { return accepted } - // Both callbacks are created on first use: instantiating them in a field initializer would - // reference classes older Android releases lack and crash the service as it binds. + // Created on first use: instantiating it in a field initializer would reference a class older + // Android releases lack and crash the service as it binds. private var gestureCallback: AccessibilityService.GestureResultCallback? = null - private var screenshotCallback: AccessibilityService.TakeScreenshotCallback? = null @RequiresApi(Build.VERSION_CODES.N) private inner class GestureCallback : AccessibilityService.GestureResultCallback() { @@ -371,6 +424,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun releaseHeldPointer() { mainHandler.post { inputSteps.clear() + endDragSelect() if (heldStroke != null) { inputSteps.addLast(InputStep.Up(heldX.toInt(), heldY.toInt())) pumpInput() @@ -408,18 +462,40 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { if (!active || capturing || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return capturing = true lastCaptureUptimeMs = SystemClock.uptimeMillis() + val sequence = ++captureSequence + mainHandler.removeCallbacks(captureWatchdog) + mainHandler.postDelayed(captureWatchdog, CAPTURE_WATCHDOG_MS) try { - val callback = screenshotCallback ?: ScreenshotCallback().also { screenshotCallback = it } - takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, callback) + takeScreenshot(Display.DEFAULT_DISPLAY, captureExecutor, ScreenshotCallback(sequence)) } catch (ex: Exception) { - capturing = false + captureFinished(sequence) scheduleNextCapture() } } + // Android occasionally never answers a screenshot request; without this the loop would stop + // for the rest of the session. + private val captureWatchdog = Runnable { + if (!capturing) return@Runnable + println("takeScreenshot did not answer, retrying") + captureSequence++ + capturing = false + scheduleNextCapture() + } + + private fun captureFinished(sequence: Int) { + if (sequence != captureSequence) return + mainHandler.removeCallbacks(captureWatchdog) + capturing = false + } + @RequiresApi(Build.VERSION_CODES.R) - private inner class ScreenshotCallback : AccessibilityService.TakeScreenshotCallback { + private inner class ScreenshotCallback(private val sequence: Int) : AccessibilityService.TakeScreenshotCallback { override fun onSuccess(screenshot: AccessibilityService.ScreenshotResult) { + if (sequence != captureSequence) { + screenshot.hardwareBuffer.close() + return + } // Recovered: allow the next error to be reported again. screenshotErrorNotified = false var bitmap: Bitmap? = null @@ -458,13 +534,14 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { if (encodedBitmap != null && encodedBitmap !== bitmap) encodedBitmap.recycle() bitmap?.recycle() screenshot.hardwareBuffer.close() - capturing = false + captureFinished(sequence) scheduleNextCapture() } } override fun onFailure(errorCode: Int) { - capturing = false + if (sequence != captureSequence) return + captureFinished(sequence) if (errorCode == AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT) { // We asked too soon; retry at the throttle interval rather than backing off toward idle. scheduleNextCapture() @@ -501,16 +578,24 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private fun handleLegacyKey(msg: ByteString): Boolean { if (msg.size < 6) return false - val action = u(msg[4]) + val down = u(msg[4]) == 0 val keyCode = u(msg[5]) - if (action != 0) return true when (keyCode) { - 8 -> return backspaceFocused() || (keyguardLocked() && keyguardKey("delete_button", null)) - 13 -> return enterFocused() - 37 -> return moveFocusedCursor(-1) - 39 -> return moveFocusedCursor(1) - 38 -> return moveFocusedCursorLine(-1) - 40 -> return moveFocusedCursorLine(1) + 16 -> { shiftHeld = down; return true } + 17 -> { ctrlHeld = down; return true } + 18 -> { altHeld = down; return true } + } + if (!down) return true + if (ctrlHeld) return handleShortcut(keyCode) + when (keyCode) { + 8 -> return (keyguardLocked() && keyguardKey("delete_button", null)) || editorKey(KeyEvent.KEYCODE_DEL) || backspaceFocused() + 9 -> if (editorKey(KeyEvent.KEYCODE_TAB)) return true + 13 -> return (keyguardLocked() && keyguardKey("key_enter", null)) || editorKey(KeyEvent.KEYCODE_ENTER) || enterFocused() + 46 -> return editorKey(KeyEvent.KEYCODE_FORWARD_DEL) || deleteForwardFocused() + 37 -> return editorKey(KeyEvent.KEYCODE_DPAD_LEFT) || moveFocusedCursor(-1) + 39 -> return editorKey(KeyEvent.KEYCODE_DPAD_RIGHT) || moveFocusedCursor(1) + 38 -> return editorKey(KeyEvent.KEYCODE_DPAD_UP) || moveFocusedCursorLine(-1) + 40 -> return editorKey(KeyEvent.KEYCODE_DPAD_DOWN) || moveFocusedCursorLine(1) 27 -> return globalAction(GLOBAL_ACTION_BACK) 36 -> return globalAction(GLOBAL_ACTION_HOME) 93 -> return globalAction(GLOBAL_ACTION_RECENTS) @@ -575,11 +660,210 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { val ch = readShort(msg, 5).toChar() // Lock screen PIN pad: no editable field, so press its buttons by hand. if (ch.isDigit() && keyguardLocked() && focusedEditable() == null && keyguardKey("key$ch", ch.toString())) return true + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val connection = editorConnection() + if (connection != null) { + connection.commitText(ch.toString(), 1, null) + return true + } + } return insertFocused(ch.toString()).also { if (!it) notifyUnsupportedKeyboard() } } + // Android 13+ lets an accessibility service act as an input method: an InputConnection keeps + // keystrokes in order and offers the editor's own key handling and shortcuts, so nothing is + // dropped when the operator types fast. Null without an active text field or on older releases. + private fun editorConnection(): InputMethod.AccessibilityInputConnection? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) return null + // The connection can outlive the field it belonged to (the lock screen's PIN pad is one + // case), so only trust it while a text field actually has input focus. + if (focusedEditable() == null) return null + return try { + inputMethod?.currentInputConnection + } catch (ex: Exception) { + null + } + } + + // A hardware-style key press with the held modifiers, through the input connection. + private fun editorKey(keyCode: Int): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) return false + val connection = editorConnection() ?: return false + val meta = currentMeta() + val now = SystemClock.uptimeMillis() + connection.sendKeyEvent(KeyEvent(now, now, KeyEvent.ACTION_DOWN, keyCode, 0, meta)) + connection.sendKeyEvent(KeyEvent(now, now, KeyEvent.ACTION_UP, keyCode, 0, meta)) + return true + } + + private fun currentMeta(): Int { + var meta = 0 + if (shiftHeld) meta = meta or KeyEvent.META_SHIFT_ON or KeyEvent.META_SHIFT_LEFT_ON + if (ctrlHeld) meta = meta or KeyEvent.META_CTRL_ON or KeyEvent.META_CTRL_LEFT_ON + if (altHeld) meta = meta or KeyEvent.META_ALT_ON or KeyEvent.META_ALT_LEFT_ON + return meta + } + + // Ctrl shortcuts: the editor's own select-all, copy, cut and paste where an input connection + // exists, node actions otherwise; any other combination goes through as a key with Ctrl held. + private fun handleShortcut(keyCode: Int): Boolean { + val menuId = when (keyCode) { + 65 -> android.R.id.selectAll + 67 -> android.R.id.copy + 86 -> android.R.id.paste + 88 -> android.R.id.cut + else -> 0 + } + if (menuId != 0) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val connection = editorConnection() + if (connection != null) { + connection.performContextMenuAction(menuId) + return true + } + } + val node = focusedEditable() ?: return true + return when (keyCode) { + 65 -> setSelectionRange(node, 0, fieldText(node).length) + 67 -> node.performAction(AccessibilityNodeInfo.ACTION_COPY) + 86 -> node.performAction(AccessibilityNodeInfo.ACTION_PASTE) + else -> node.performAction(AccessibilityNodeInfo.ACTION_CUT) + } + } + val androidKey = when (keyCode) { + in 65..90 -> KeyEvent.KEYCODE_A + (keyCode - 65) + 37 -> KeyEvent.KEYCODE_DPAD_LEFT + 39 -> KeyEvent.KEYCODE_DPAD_RIGHT + 38 -> KeyEvent.KEYCODE_DPAD_UP + 40 -> KeyEvent.KEYCODE_DPAD_DOWN + 8 -> KeyEvent.KEYCODE_DEL + 46 -> KeyEvent.KEYCODE_FORWARD_DEL + else -> 0 + } + if (androidKey != 0) editorKey(androidKey) + return true + } + + // A drag that starts on the focused text field selects text, as a mouse does on a desktop, + // instead of the touch behaviour of moving the caret. The field's character bounds are + // fetched once per drag and matched against the pointer; fields that don't report them (some + // web views) keep the touch behaviour. + private fun beginDragSelect(x: Int, y: Int): Boolean { + endDragSelect() + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return false + val node = focusedEditable() ?: return false + if (node.isPassword) return false + val bounds = Rect() + node.getBoundsInScreen(bounds) + if (!bounds.contains(x, y)) return false + val rects = characterBounds(node) + if (rects == null) { + println("dragSelect: field reports no character bounds") + return false + } + val anchor = offsetAt(rects, x, y) + if (anchor == null) { + println("dragSelect: no text offset at $x,$y") + return false + } + dragSelectNode = node + dragSelectRects = rects + dragSelectAnchor = anchor + dragSelectLastFocus = anchor + dragDownX = x + dragDownY = y + return true + } + + private fun updateDragSelect(x: Int, y: Int) { + val node = dragSelectNode ?: return + val rects = dragSelectRects ?: return + val focus = offsetAt(rects, x, y) ?: return + if (focus == dragSelectLastFocus) return + dragSelectLastFocus = focus + if (!setSelectionRange(node, dragSelectAnchor, focus)) println("dragSelect: selection $dragSelectAnchor..$focus refused") + } + + private fun endDragSelect() { + mainHandler.removeCallbacks(deferredPress) + dragSelectNode = null + dragSelectRects = null + dragSelectActive = false + } + + // The pointer is still held on the text field with no movement: it was a press after all, so + // put the finger down now (a long press follows if it stays). + private val deferredPress = object : Runnable { + override fun run() { + if (dragSelectNode == null || dragSelectActive) return + if (gestureInFlight) { + mainHandler.postDelayed(this, 20) + return + } + val x = dragDownX + val y = dragDownY + endDragSelect() + pressPointer(x, y) + } + } + + @Suppress("DEPRECATION") + @RequiresApi(Build.VERSION_CODES.O) + private fun characterBounds(node: AccessibilityNodeInfo): List? { + val length = min(fieldText(node).length, AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_ARG_MAX_LENGTH) + if (length == 0) return null + val args = Bundle() + args.putInt(AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_ARG_START_INDEX, 0) + args.putInt(AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_ARG_LENGTH, length) + val fetched = try { + node.refreshWithExtraData(AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_KEY, args) + } catch (ex: Exception) { + false + } + if (!fetched) return null + val array = node.extras.getParcelableArray(AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_KEY) ?: return null + val rects = array.map { it as? RectF } + return if (rects.any { it != null }) rects else null + } + + // Text offset for a screen point: the nearest line by vertical distance, then the character + // whose bounds hold x, with the caret after it when x is past its middle. + private fun offsetAt(rects: List, x: Int, y: Int): Int? { + var lineTop = 0f + var lineBottom = 0f + var best = Float.MAX_VALUE + for (r in rects) { + if (r == null) continue + val distance = when { + y < r.top -> r.top - y + y > r.bottom -> y - r.bottom + else -> 0f + } + if (distance < best) { + best = distance + lineTop = r.top + lineBottom = r.bottom + } + } + if (best == Float.MAX_VALUE) return null + var firstIndex = -1 + var lastIndex = -1 + var firstLeft = 0f + for ((i, r) in rects.withIndex()) { + if (r == null || (r.top - lineTop).absoluteValue > 0.5f || (r.bottom - lineBottom).absoluteValue > 0.5f) continue + if (firstIndex < 0) { + firstIndex = i + firstLeft = r.left + } + lastIndex = i + if (x >= r.left && x <= r.right) return if (x < r.centerX()) i else i + 1 + } + if (firstIndex < 0) return null + return if (x < firstLeft) firstIndex else lastIndex + 1 + } + private fun keyguardLocked(): Boolean { val keyguard = getSystemService(Context.KEYGUARD_SERVICE) as? KeyguardManager ?: return false return keyguard.isKeyguardLocked @@ -673,34 +957,55 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { return if (s <= e) Pair(s, e) else Pair(e, s) } - private fun setCursor(node: AccessibilityNodeInfo, pos: Int): Boolean { + private fun setCursor(node: AccessibilityNodeInfo, pos: Int): Boolean = setSelectionRange(node, pos, pos) + + // TextView rejects a selection whose start is past its end, so order the two. + private fun setSelectionRange(node: AccessibilityNodeInfo, start: Int, end: Int): Boolean { val args = Bundle() - args.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, pos) - args.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, pos) + args.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, min(start, end)) + args.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, max(start, end)) return node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, args) } + private fun nodeKey(node: AccessibilityNodeInfo): String = "${node.windowId}:${node.hashCode()}" + private fun replaceText(node: AccessibilityNodeInfo, text: String, cursor: Int): Boolean { val args = Bundle() args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) if (!node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)) return false setCursor(node, cursor) + shadowNodeKey = nodeKey(node) + shadowText = text + shadowCursor = cursor + shadowUptimeMs = SystemClock.uptimeMillis() return true } + // The field's text and selection, or the text of a write the app hasn't applied yet: a + // keystroke that read the stale text would rebuild from it and drop the previous character. + private fun fieldState(node: AccessibilityNodeInfo): Pair> { + val text = fieldText(node) + val shadow = shadowText + if (shadow != null && shadowNodeKey == nodeKey(node) && text != shadow && + SystemClock.uptimeMillis() - shadowUptimeMs < SHADOW_TEXT_MS) { + return Pair(shadow, Pair(shadowCursor, shadowCursor)) + } + return Pair(text, selectionRange(node, text.length)) + } + private fun insertFocused(insert: String): Boolean { val node = focusedEditable() ?: return false if (node.isPassword) return typeIntoPassword(node, insert) - val text = fieldText(node) - val (s, e) = selectionRange(node, text.length) + val (text, selection) = fieldState(node) + val (s, e) = selection return replaceText(node, text.substring(0, s) + insert + text.substring(e), s + insert.length) } private fun backspaceFocused(): Boolean { val node = focusedEditable() ?: return false if (node.isPassword) return typeIntoPassword(node, null) - val text = fieldText(node) - val (s, e) = selectionRange(node, text.length) + val (text, selection) = fieldState(node) + val (s, e) = selection return when { s != e -> replaceText(node, text.substring(0, s) + text.substring(e), s) s > 0 -> replaceText(node, text.substring(0, s - 1) + text.substring(s), s - 1) @@ -708,10 +1013,28 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { } } + private fun deleteForwardFocused(): Boolean { + val node = focusedEditable() ?: return false + if (node.isPassword) return false + val (text, selection) = fieldState(node) + val (s, e) = selection + return when { + s != e -> replaceText(node, text.substring(0, s) + text.substring(e), s) + e < text.length -> replaceText(node, text.substring(0, e) + text.substring(e + 1), e) + else -> true + } + } + private fun moveFocusedCursor(delta: Int): Boolean { val node = focusedEditable() ?: return true val text = fieldText(node) val (s, e) = selectionRange(node, text.length) + if (shiftHeld) { + // Extend from the anchor, which Android keeps as the selection start. + val anchor = node.textSelectionStart.coerceIn(0, text.length) + val focus = (node.textSelectionEnd.coerceIn(0, text.length) + delta).coerceIn(0, text.length) + return setSelectionRange(node, anchor, focus) + } // A selection collapses to its near edge; otherwise step one character. val pos = when { s != e && delta < 0 -> s @@ -739,6 +1062,7 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { (lineEnd + 1 + col).coerceAtMost(nextEnd) } } + if (shiftHeld) return setSelectionRange(node, node.textSelectionStart.coerceIn(0, text.length), pos) return setCursor(node, pos) } @@ -797,5 +1121,9 @@ class MeshAccessibilityService : AccessibilityService(), RemoteDesktopProvider { private const val HOME_SETTLE_MS = 450L private const val APP_DRAWER_SWIPE_MS = 300L private const val KEYGUARD_NOTICE_S = 45 + private const val CAPTURE_WATCHDOG_MS = 4_000L + private const val DRAG_SELECT_SLOP = 8 + private const val DRAG_SELECT_DEFER_MS = 300L + private const val SHADOW_TEXT_MS = 800L } } diff --git a/app/src/main/res/xml/mesh_accessibility_service.xml b/app/src/main/res/xml/mesh_accessibility_service.xml index 221ed86..e01c537 100644 --- a/app/src/main/res/xml/mesh_accessibility_service.xml +++ b/app/src/main/res/xml/mesh_accessibility_service.xml @@ -2,7 +2,7 @@