diff --git a/app/build.gradle.kts b/app/build.gradle.kts index cf112f5..884bc98 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -10,10 +10,10 @@ android { defaultConfig { applicationId = "com.kevinluo.autoglm" - minSdk = 24 + minSdk = 27 targetSdk = 34 - versionCode = 5 - versionName = "0.0.5" + versionCode = 2 + versionName = "0.0.2" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } @@ -32,7 +32,8 @@ android { buildTypes { debug { - // 不再使用 applicationIdSuffix，与发行版使用相同包名 + applicationIdSuffix = ".debug" + versionNameSuffix = "-debug" resValue("string", "app_name", "AutoGLM Dev") } release { @@ -59,7 +60,7 @@ android { kotlinOptions { jvmTarget = "11" } - + // Enable JUnit 5 for Kotest property-based testing testOptions { unitTests.all { @@ -69,33 +70,25 @@ android { } } -// Copy dev_profiles.json to assets for debug builds only +// Copy dev_profiles.json to assets for debug builds android.applicationVariants.all { val variant = this - - // Custom APK file name - outputs.all { - val output = this as com.android.build.gradle.internal.api.BaseVariantOutputImpl - output.outputFileName = "AutoGLM-${variant.versionName}-${variant.buildType.name}.apk" - } - if (variant.buildType.name == "debug") { val copyDevProfiles = tasks.register("copyDevProfiles${variant.name.replaceFirstChar { it.uppercase() }}") { val devProfilesFile = rootProject.file("dev_profiles.json") - // Use debug-specific assets directory to avoid polluting release builds - val assetsDir = file("src/debug/assets") - + val assetsDir = file("src/main/assets") + doLast { if (devProfilesFile.exists()) { assetsDir.mkdirs() devProfilesFile.copyTo(File(assetsDir, "dev_profiles.json"), overwrite = true) - println("Copied dev_profiles.json to debug assets") + println("Copied dev_profiles.json to assets") } else { println("dev_profiles.json not found, skipping") } } } - + tasks.named("merge${variant.name.replaceFirstChar { it.uppercase() }}Assets") { dependsOn(copyDevProfiles) } @@ -111,29 +104,29 @@ dependencies { implementation(libs.androidx.constraintlayout) implementation(libs.shizuku.api) implementation(libs.shizuku.provider) - + // Kotlin Coroutines for async operations implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.coroutines.android) - + // Lifecycle & ViewModel implementation(libs.androidx.lifecycle.viewmodel) implementation(libs.androidx.lifecycle.runtime) - + // Security for encrypted preferences implementation(libs.androidx.security.crypto) - + // OkHttp for API communication implementation(libs.okhttp) implementation(libs.okhttp.sse) implementation(libs.okhttp.logging) - + // Retrofit for API communication implementation(libs.retrofit) - + // Kotlin Serialization for JSON parsing implementation(libs.kotlinx.serialization.json) - + // Testing testImplementation(libs.junit) testImplementation(libs.kotest.runner.junit5) @@ -141,4 +134,4 @@ dependencies { testImplementation(libs.kotest.assertions.core) androidTestImplementation(libs.androidx.junit) androidTestImplementation(libs.androidx.espresso.core) -} \ No newline at end of file +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index b4cd24b..31ac67f 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -3,9 +3,13 @@ xmlns:tools="http://schemas.android.com/tools"> + + + + + + + + + + + - \ No newline at end of file + diff --git a/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl b/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl index 09a179b..6ed15f5 100644 --- a/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl +++ b/app/src/main/aidl/com/kevinluo/autoglm/IUserService.aidl @@ -3,4 +3,10 @@ package com.kevinluo.autoglm; interface IUserService { void destroy() = 16777114; String executeCommand(String command) = 1; + // Capture screenshot to internal buffer and return base64 length + int captureScreenshotAndGetSize() = 2; + // Read a base64 chunk from the internal screenshot buffer + String readScreenshotChunk(int offset, int size) = 3; + // Clear internal screenshot buffer to free memory + void clearScreenshotBuffer() = 4; } diff --git a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt index 3f29cf0..354f3ab 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ComponentManager.kt @@ -7,7 +7,11 @@ import com.kevinluo.autoglm.agent.AgentConfig import com.kevinluo.autoglm.agent.PhoneAgent import com.kevinluo.autoglm.agent.PhoneAgentListener import com.kevinluo.autoglm.app.AppResolver +import com.kevinluo.autoglm.device.AccessibilityDeviceController +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.device.DeviceExecutor +import com.kevinluo.autoglm.device.IDeviceController +import com.kevinluo.autoglm.device.ShizukuDeviceController import com.kevinluo.autoglm.history.HistoryManager import com.kevinluo.autoglm.input.TextInputManager import com.kevinluo.autoglm.model.ModelClient @@ -17,29 +21,29 @@ import com.kevinluo.autoglm.screenshot.ScreenshotService import com.kevinluo.autoglm.settings.SettingsManager import com.kevinluo.autoglm.ui.FloatingWindowService import com.kevinluo.autoglm.util.HumanizedSwipeGenerator -import com.kevinluo.autoglm.util.Logger /** * Centralized component manager for dependency injection and lifecycle management. * Provides a single point of access for all major components in the application. - * + * * This class ensures: * - Proper dependency injection * - Lifecycle-aware component management * - Clean separation of concerns - * + * + * Requirements: All (integration) */ class ComponentManager private constructor(private val context: Context) { - + companion object { private const val TAG = "ComponentManager" - + @Volatile private var instance: ComponentManager? = null - + /** * Gets the singleton instance of ComponentManager. - * + * * @param context Application context * @return ComponentManager instance */ @@ -48,7 +52,7 @@ class ComponentManager private constructor(private val context: Context) { instance ?: ComponentManager(context.applicationContext).also { instance = it } } } - + /** * Clears the singleton instance. * Should be called when the application is being destroyed. @@ -60,66 +64,84 @@ class ComponentManager private constructor(private val context: Context) { } } } - + // Settings manager - always available val settingsManager: SettingsManager by lazy { SettingsManager(context) } - + // History manager - always available val historyManager: HistoryManager by lazy { HistoryManager.getInstance(context) } - + // User service reference - set when Shizuku connects private var userService: IUserService? = null - + + // Device controller - initialized based on control mode + private var _deviceController: IDeviceController? = null + // Lazily initialized components that depend on UserService private var _deviceExecutor: DeviceExecutor? = null private var _textInputManager: TextInputManager? = null private var _screenshotService: ScreenshotService? = null private var _actionHandler: ActionHandler? = null private var _phoneAgent: PhoneAgent? = null - + // Components that don't depend on UserService private var _modelClient: ModelClient? = null private var _appResolver: AppResolver? = null private var _swipeGenerator: HumanizedSwipeGenerator? = null - + /** - * Checks if the UserService is connected. + * Checks if the device controller is ready. + * For Shizuku mode: checks if Shizuku service is connected + * For Accessibility mode: checks if PhoneAgent is initialized */ val isServiceConnected: Boolean - get() = userService != null - + get() { + val controlMode = settingsManager.getDeviceControlMode() + return when (controlMode) { + DeviceControlMode.SHIZUKU -> userService != null + DeviceControlMode.ACCESSIBILITY -> _phoneAgent != null + } + } + /** * Gets the DeviceExecutor instance. * Requires UserService to be connected. */ val deviceExecutor: DeviceExecutor? get() = _deviceExecutor - + + /** + * Gets the DeviceController instance. + * Available in both Shizuku and Accessibility modes. + */ + val deviceControllerInstance: IDeviceController? + get() = _deviceController + /** * Gets the ScreenshotService instance. * Requires UserService to be connected. */ val screenshotService: ScreenshotService? get() = _screenshotService - + /** * Gets the ActionHandler instance. * Requires UserService to be connected. */ val actionHandler: ActionHandler? get() = _actionHandler - + /** * Gets the PhoneAgent instance. * Requires UserService to be connected. */ val phoneAgent: PhoneAgent? get() = _phoneAgent - + /** * Gets the ModelClient instance. * Creates a new instance if config has changed. @@ -132,7 +154,7 @@ class ComponentManager private constructor(private val context: Context) { } return _modelClient!! } - + /** * Gets the AppResolver instance. */ @@ -143,7 +165,7 @@ class ComponentManager private constructor(private val context: Context) { } return _appResolver!! } - + /** * Gets the HumanizedSwipeGenerator instance. */ @@ -154,57 +176,110 @@ class ComponentManager private constructor(private val context: Context) { } return _swipeGenerator!! } - + // Track current model config for change detection private var currentModelConfig: ModelConfig? = null - + /** * Called when UserService connects. - * Initializes all service-dependent components. - * + * Initializes all service-dependent components for Shizuku mode. + * * @param service The connected UserService */ fun onServiceConnected(service: IUserService) { - Logger.i(TAG, "UserService connected, initializing components") + android.util.Log.i(TAG, "UserService connected, initializing components") userService = service - initializeServiceDependentComponents() + + // Only initialize Shizuku components if in Shizuku mode + val controlMode = settingsManager.getDeviceControlMode() + if (controlMode == DeviceControlMode.SHIZUKU) { + initializeServiceDependentComponents() + } } - + /** * Called when UserService disconnects. * Cleans up service-dependent components. */ fun onServiceDisconnected() { - Logger.i(TAG, "UserService disconnected, cleaning up components") + android.util.Log.i(TAG, "UserService disconnected, cleaning up components") userService = null cleanupServiceDependentComponents() } - + /** * Initializes components that depend on UserService. */ private fun initializeServiceDependentComponents() { val service = userService ?: return - + // Create DeviceExecutor _deviceExecutor = DeviceExecutor(service) - + // Create TextInputManager _textInputManager = TextInputManager(service) - + // Create ScreenshotService with floating window controller provider // Use a provider function so it can get the current instance dynamically - _screenshotService = ScreenshotService(service) { FloatingWindowService.getInstance() } - - // Create ActionHandler with floating window provider to hide window during touch operations + _screenshotService = ScreenshotService( + userService = service, + screenshotProvider = null, // For Shizuku mode, use shell commands + floatingWindowControllerProvider = { FloatingWindowService.getInstance() } + ) + + // Create ShizukuDeviceController for Shizuku mode + _deviceController = ShizukuDeviceController( + userService = service, + textInputManager = _textInputManager!!, + screenshotProvider = { _screenshotService!! } + ) + + // Create ActionHandler with IDeviceController _actionHandler = ActionHandler( - deviceExecutor = _deviceExecutor!!, + deviceController = _deviceController!!, + appResolver = appResolver, + swipeGenerator = swipeGenerator, + floatingWindowProvider = { FloatingWindowService.getInstance() } + ) + + // Create PhoneAgent + val agentConfig = settingsManager.getAgentConfig() + _phoneAgent = PhoneAgent( + modelClient = modelClient, + actionHandler = _actionHandler!!, + screenshotService = _screenshotService!!, + config = agentConfig, + historyManager = historyManager + ) + + android.util.Log.i(TAG, "All service-dependent components initialized") + } + + /** + * Initializes components for accessibility mode. + * Creates a device controller when Accessibility Service is available. + */ + private fun initializeAccessibilityComponents() { + // Create AccessibilityDeviceController + _deviceController = AccessibilityDeviceController(context) + + // Create ScreenshotService for accessibility mode with proper screenshotProvider + // The screenshotProvider delegates to AccessibilityDeviceController for screenshots + _screenshotService = ScreenshotService( + userService = null, // No userService needed in accessibility mode + screenshotProvider = { _deviceController!!.captureScreen() }, + floatingWindowControllerProvider = { FloatingWindowService.getInstance() } + ) + + // Create ActionHandler with IDeviceController (AccessibilityDeviceController) + // No need for DeviceExecutor or TextInputManager, all operations go through the controller + _actionHandler = ActionHandler( + deviceController = _deviceController!!, appResolver = appResolver, swipeGenerator = swipeGenerator, - textInputManager = _textInputManager!!, floatingWindowProvider = { FloatingWindowService.getInstance() } ) - + // Create PhoneAgent val agentConfig = settingsManager.getAgentConfig() _phoneAgent = PhoneAgent( @@ -214,10 +289,15 @@ class ComponentManager private constructor(private val context: Context) { config = agentConfig, historyManager = historyManager ) - - Logger.i(TAG, "All service-dependent components initialized") + + // Check permission status and log accordingly + if (!_deviceController!!.checkPermission()) { + android.util.Log.w(TAG, "Accessibility service not yet connected, but components initialized for deferred startup") + } else { + android.util.Log.i(TAG, "Accessibility mode components initialized with service connected") + } } - + /** * Cleans up components that depend on UserService. */ @@ -228,68 +308,66 @@ class ComponentManager private constructor(private val context: Context) { _screenshotService = null _textInputManager = null _deviceExecutor = null - - Logger.i(TAG, "Service-dependent components cleaned up") + + android.util.Log.i(TAG, "Service-dependent components cleaned up") } - + /** * Reinitializes the PhoneAgent with updated configuration. * Call this after settings have been changed. - * - * Note: This will NOT reinitialize if a task is currently running or paused, - * to prevent accidentally cancelling user tasks. + * Handles mode switching between Shizuku and Accessibility modes. */ fun reinitializeAgent() { - if (userService == null) { - Logger.w(TAG, "Cannot reinitialize agent: UserService not connected") - return - } - - // Safety check: don't reinitialize while a task is active - _phoneAgent?.let { agent -> - if (agent.isRunning() || agent.isPaused()) { - Logger.w(TAG, "Cannot reinitialize agent: task is currently active (state: ${agent.getState()})") - return - } - } - - // Cancel any running task (should be IDLE at this point, but just in case) + val controlMode = settingsManager.getDeviceControlMode() + + // Cancel any running task _phoneAgent?.cancel() - + // Recreate model client with new config _modelClient = null - - // Recreate PhoneAgent - val agentConfig = settingsManager.getAgentConfig() - _phoneAgent = PhoneAgent( - modelClient = modelClient, - actionHandler = _actionHandler!!, - screenshotService = _screenshotService!!, - config = agentConfig, - historyManager = historyManager - ) - - Logger.i(TAG, "PhoneAgent reinitialized with new configuration") + + when (controlMode) { + DeviceControlMode.SHIZUKU -> { + if (userService == null) { + android.util.Log.w(TAG, "Cannot reinitialize agent: UserService not connected") + return + } + + // Re-initialize Shizuku components + cleanupServiceDependentComponents() + initializeServiceDependentComponents() + + android.util.Log.i(TAG, "PhoneAgent reinitialized for Shizuku mode") + } + + DeviceControlMode.ACCESSIBILITY -> { + // Re-initialize accessibility components + cleanupServiceDependentComponents() + initializeAccessibilityComponents() + + android.util.Log.i(TAG, "PhoneAgent reinitialized for Accessibility mode") + } + } } - + /** * Sets the listener for PhoneAgent events. - * + * * @param listener The listener to set */ fun setPhoneAgentListener(listener: PhoneAgentListener?) { _phoneAgent?.setListener(listener) } - + /** * Sets the confirmation callback for ActionHandler. - * + * * @param callback The callback to set */ fun setConfirmationCallback(callback: ActionHandler.ConfirmationCallback?) { _actionHandler?.setConfirmationCallback(callback) } - + /** * Checks if the model config has changed. */ @@ -300,20 +378,20 @@ class ComponentManager private constructor(private val context: Context) { } return changed } - + /** * Cleans up all components. * Should be called when the application is being destroyed. */ fun cleanup() { - Logger.i(TAG, "Cleaning up all components") + android.util.Log.i(TAG, "Cleaning up all components") cleanupServiceDependentComponents() _modelClient = null _appResolver = null _swipeGenerator = null currentModelConfig = null } - + /** * Gets the current state summary for debugging. */ diff --git a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt index c1fe57d..53c6f92 100644 --- a/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/MainActivity.kt @@ -1,28 +1,35 @@ package com.kevinluo.autoglm +import android.accessibilityservice.AccessibilityServiceInfo import android.content.ComponentName import android.content.Intent import android.content.ServiceConnection import android.content.pm.PackageManager import android.graphics.drawable.GradientDrawable +import android.os.Build import android.os.Bundle import android.os.IBinder +import android.provider.Settings import android.view.View +import android.view.accessibility.AccessibilityManager import android.widget.Button import android.widget.ImageButton import android.widget.TextView import android.widget.Toast -import androidx.activity.enableEdgeToEdge +import androidx.activity.result.contract.ActivityResultContracts import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AppCompatActivity import androidx.core.content.ContextCompat +import androidx.core.content.PermissionChecker import androidx.core.view.ViewCompat import androidx.core.view.WindowInsetsCompat import androidx.lifecycle.lifecycleScope +import com.kevinluo.autoglm.accessibility.AutoGLMAccessibilityService import com.kevinluo.autoglm.action.ActionHandler import com.kevinluo.autoglm.action.AgentAction import com.kevinluo.autoglm.agent.PhoneAgent import com.kevinluo.autoglm.agent.PhoneAgentListener +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.settings.SettingsActivity import com.kevinluo.autoglm.ui.FloatingWindowService import com.kevinluo.autoglm.ui.TaskStatus @@ -50,10 +57,12 @@ import rikka.shizuku.Shizuku * The activity implements [PhoneAgentListener] to receive callbacks * during task execution for UI updates. * + * Requirements: 1.1, 2.1, 2.2 */ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Shizuku status views + private lateinit var shizukuCard: View private lateinit var statusText: TextView private lateinit var shizukuStatusIndicator: View private lateinit var shizukuButtonsRow: View @@ -73,6 +82,12 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { private lateinit var keyboardStatusText: TextView private lateinit var enableKeyboardBtn: Button + // Accessibility permission views + private lateinit var accessibilityCard: View + private lateinit var accessibilityStatusIcon: android.widget.ImageView + private lateinit var accessibilityStatusText: TextView + private lateinit var requestAccessibilityBtn: Button + // Task input views private lateinit var taskInputLayout: TextInputLayout private lateinit var taskInput: TextInputEditText @@ -88,7 +103,22 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // Component manager for dependency injection private lateinit var componentManager: ComponentManager - + + private val requestNotificationPermissionLauncher = registerForActivityResult( + ActivityResultContracts.RequestPermission() + ) { isGranted: Boolean -> + if (isGranted) { + Logger.i(TAG, "Notification permission granted") + } else { + Logger.w(TAG, "Notification permission denied") + Toast.makeText( + this, + R.string.toast_notification_permission_required, + Toast.LENGTH_LONG + ).show() + } + } + // Current step tracking for floating window private var currentStepNumber = 0 private var currentThinking = "" @@ -108,26 +138,26 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { override fun onServiceConnected(name: ComponentName?, service: IBinder?) { val userService = IUserService.Stub.asInterface(service) Logger.i(TAG, "UserService connected") - + // Notify ComponentManager componentManager.onServiceConnected(userService) - + runOnUiThread { Toast.makeText(this@MainActivity, R.string.toast_user_service_connected, Toast.LENGTH_SHORT).show() - updateShizukuStatus() + updateDeviceServiceStatus() initializePhoneAgent() } } override fun onServiceDisconnected(name: ComponentName?) { Logger.i(TAG, "UserService disconnected") - + // Notify ComponentManager componentManager.onServiceDisconnected() - + runOnUiThread { Toast.makeText(this@MainActivity, R.string.toast_user_service_disconnected, Toast.LENGTH_SHORT).show() - updateShizukuStatus() + updateDeviceServiceStatus() updateTaskButtonStates() } } @@ -137,7 +167,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { Shizuku.OnRequestPermissionResultListener { requestCode, grantResult -> if (requestCode == SHIZUKU_PERMISSION_REQUEST_CODE) { if (grantResult == PackageManager.PERMISSION_GRANTED) { - updateShizukuStatus() + updateDeviceServiceStatus() bindUserService() Toast.makeText(this, R.string.toast_shizuku_permission_granted, Toast.LENGTH_SHORT).show() } else { @@ -147,7 +177,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { } private val binderReceivedListener = Shizuku.OnBinderReceivedListener { - updateShizukuStatus() + updateDeviceServiceStatus() if (hasShizukuPermission()) { bindUserService() } @@ -156,35 +186,82 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { private val binderDeadListener = Shizuku.OnBinderDeadListener { Logger.w(TAG, "Shizuku binder died") componentManager.onServiceDisconnected() - updateShizukuStatus() + updateDeviceServiceStatus() updateTaskButtonStates() } + // Accessibility state change listener + private val accessibilityStateChangeListener = + AccessibilityManager.AccessibilityStateChangeListener { enabled -> + Logger.d(TAG, "Accessibility state changed: enabled=$enabled") + runOnUiThread { + updateAccessibilityStatus() + } + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - enableEdgeToEdge() setContentView(R.layout.activity_main) - ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.main)) { v, insets -> - val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars()) - v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom) - insets - } - // Initialize ComponentManager componentManager = ComponentManager.getInstance(this) Logger.i(TAG, "ComponentManager initialized") - + + // Log all launchable apps at startup + logAllLaunchableApps() + initViews() setupListeners() + checkAndRequestNotificationPermission() setupShizukuListeners() - - updateShizukuStatus() + setupAccessibilityListener() + + // For accessibility mode, initialize components immediately + val controlMode = componentManager.settingsManager.getDeviceControlMode() + if (controlMode == DeviceControlMode.ACCESSIBILITY) { + componentManager.reinitializeAgent() + } + + updateDeviceServiceStatus() updateOverlayPermissionStatus() updateKeyboardStatus() + updateAccessibilityStatus() updateTaskStatus(TaskStatus.IDLE) + updateTaskButtonStates() } - + + /** + * Logs all launchable apps for debugging purposes. + * + * Queries the package manager for all apps with launcher activities + * and logs them for debugging. Only logs the first 20 apps to avoid + * excessive log output. + */ + private fun logAllLaunchableApps() { + // Query apps without loading icons to avoid excessive logging + val intent = android.content.Intent(android.content.Intent.ACTION_MAIN).apply { + addCategory(android.content.Intent.CATEGORY_LAUNCHER) + } + val resolveInfoList = packageManager.queryIntentActivities(intent, 0) + + val apps = resolveInfoList.mapNotNull { resolveInfo -> + val activityInfo = resolveInfo.activityInfo ?: return@mapNotNull null + val displayName = resolveInfo.loadLabel(packageManager)?.toString() ?: return@mapNotNull null + val packageName = activityInfo.packageName ?: return@mapNotNull null + displayName to packageName + }.distinctBy { it.second } + + Logger.i(TAG, "=== All Launchable Apps: ${apps.size} total ===") + // Only log first 20 apps to avoid log quota + apps.take(20).forEach { (name, pkg) -> + Logger.i(TAG, " $name -> $pkg") + } + if (apps.size > 20) { + Logger.i(TAG, " ... and ${apps.size - 20} more apps") + } + Logger.i(TAG, "=== End of App List ===") + } + /** * Updates the overlay permission status display. * @@ -193,7 +270,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun updateOverlayPermissionStatus() { val hasPermission = FloatingWindowService.canDrawOverlays(this) - + if (hasPermission) { overlayStatusText.text = getString(R.string.overlay_permission_granted) overlayStatusIcon.setColorFilter(ContextCompat.getColor(this, R.color.status_running)) @@ -204,7 +281,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { requestOverlayBtn.visibility = View.VISIBLE } } - + /** * Updates the keyboard status display. * @@ -212,7 +289,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun updateKeyboardStatus() { val status = com.kevinluo.autoglm.input.KeyboardHelper.getAutoGLMKeyboardStatus(this) - + when (status) { com.kevinluo.autoglm.input.KeyboardHelper.KeyboardStatus.ENABLED -> { keyboardStatusText.text = getString(R.string.keyboard_settings_subtitle) @@ -228,16 +305,145 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { } } + /** + * Updates the accessibility service status display. + * + * Checks if AutoGLM Accessibility Service is enabled and updates the UI accordingly. + * Also shows/hides relevant permission cards based on the device control mode. + */ + private fun updateAccessibilityStatus() { + val controlMode = componentManager.settingsManager.getDeviceControlMode() + + // Show/hide relevant cards based on control mode + when (controlMode) { + DeviceControlMode.SHIZUKU -> { + // Shizuku mode: show Shizuku card and keyboard card, hide accessibility card + shizukuCard.visibility = View.VISIBLE + keyboardCard.visibility = View.VISIBLE + accessibilityCard.visibility = View.GONE + } + DeviceControlMode.ACCESSIBILITY -> { + // Accessibility mode: hide Shizuku card and keyboard card, show accessibility card + shizukuCard.visibility = View.GONE + keyboardCard.visibility = View.GONE + accessibilityCard.visibility = View.VISIBLE + + // Update accessibility status + val isAccessibilityEnabled = AutoGLMAccessibilityService.isEnabled() + + if (isAccessibilityEnabled) { + accessibilityStatusText.text = getString(R.string.accessibility_permission_granted) + accessibilityStatusIcon.setColorFilter(ContextCompat.getColor(this, R.color.status_running)) + requestAccessibilityBtn.visibility = View.GONE + } else { + accessibilityStatusText.text = getString(R.string.accessibility_permission_denied) + accessibilityStatusIcon.setColorFilter(ContextCompat.getColor(this, R.color.status_waiting)) + requestAccessibilityBtn.visibility = View.VISIBLE + requestAccessibilityBtn.text = getString(R.string.request_accessibility_permission) + } + + // Check Android version for accessibility support + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { + accessibilityStatusText.text = getString(R.string.accessibility_not_supported) + requestAccessibilityBtn.visibility = View.GONE + } + } + } + + // Also update the main status display to reflect the current mode + updateDeviceServiceStatus() + } + + /** + * Attempts to automatically enable AutoGLM Accessibility Service when in accessibility mode + * and the app has WRITE_SECURE_SETTINGS permission. + * + * This writes the component into Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES and sets + * Settings.Secure.ACCESSIBILITY_ENABLED to 1. Requires privileged permission. + */ + private fun tryAutoEnableAccessibilityService() { + val controlMode = componentManager.settingsManager.getDeviceControlMode() + if (controlMode != DeviceControlMode.ACCESSIBILITY) return + + // Already enabled + if (AutoGLMAccessibilityService.isEnabled()) return + + // Check WRITE_SECURE_SETTINGS permission + val hasWss = PermissionChecker.checkSelfPermission( + this, + android.Manifest.permission.WRITE_SECURE_SETTINGS + ) == PermissionChecker.PERMISSION_GRANTED + + if (!hasWss) { + Logger.d(TAG, "WRITE_SECURE_SETTINGS not granted; skip auto-enable") + return + } + + val component = ComponentName(this, AutoGLMAccessibilityService::class.java) + val flattened = component.flattenToString() + try { + val current = Settings.Secure.getString( + contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES + ) + val items = current?.split(":")?.filter { it.isNotBlank() }?.toMutableSet() ?: mutableSetOf() + if (!items.contains(flattened)) { + items.add(flattened) + } + val newValue = items.joinToString(":") + val updated = Settings.Secure.putString( + contentResolver, + Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES, + newValue + ) + val enabled = Settings.Secure.putInt( + contentResolver, + Settings.Secure.ACCESSIBILITY_ENABLED, + 1 + ) + Logger.i( + TAG, + "Auto-enable accessibility attempted: updated=$updated enabledSet=$enabled" + ) + } catch (e: Exception) { + Logger.e(TAG, "Failed to auto-enable accessibility service", e) + } + + // Refresh UI after attempt + updateAccessibilityStatus() + } + + /** + * Checks and requests notification permission on Android 13+. + */ + private fun checkAndRequestNotificationPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + if (ContextCompat.checkSelfPermission( + this, + android.Manifest.permission.POST_NOTIFICATIONS + ) != PackageManager.PERMISSION_GRANTED + ) { + requestNotificationPermissionLauncher.launch(android.Manifest.permission.POST_NOTIFICATIONS) + } + } + } + override fun onResume() { super.onResume() Logger.d(TAG, "onResume - checking for settings changes") - + // Update overlay permission status (user may have granted it) updateOverlayPermissionStatus() - + // Update keyboard status (user may have enabled it) updateKeyboardStatus() - + + // Update accessibility status (user may have enabled it or mode changed) + updateAccessibilityStatus() + + // Attempt auto-enable accessibility if applicable + tryAutoEnableAccessibilityService() + // Re-setup floating window callbacks if service is running FloatingWindowService.getInstance()?.let { service -> service.setStopTaskCallback { @@ -260,36 +466,37 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { resumeTask() } } - - // Only reinitialize if service is connected and we need to refresh + + // Check if settings actually changed before reinitializing + if (componentManager.settingsManager.hasConfigChanged()) { + componentManager.reinitializeAgent() + } + + // Update button states if (componentManager.isServiceConnected) { - // Check if settings actually changed before reinitializing - // But NEVER reinitialize while a task is running or paused - this would cancel the task! - val isTaskActive = componentManager.phoneAgent?.let { - it.isRunning() || it.isPaused() - } ?: false - - if (!isTaskActive && componentManager.settingsManager.hasConfigChanged()) { - componentManager.reinitializeAgent() - } componentManager.setPhoneAgentListener(this) setupConfirmationCallback() - updateTaskButtonStates() } + updateTaskButtonStates() } override fun onDestroy() { Logger.i(TAG, "onDestroy - cleaning up") super.onDestroy() - + // Remove Shizuku listeners Shizuku.removeRequestPermissionResultListener(onRequestPermissionResultListener) Shizuku.removeBinderReceivedListener(binderReceivedListener) Shizuku.removeBinderDeadListener(binderDeadListener) + // Remove accessibility state change listener + (getSystemService(ACCESSIBILITY_SERVICE) as? AccessibilityManager)?.removeAccessibilityStateChangeListener( + accessibilityStateChangeListener + ) + // Cancel any running task componentManager.phoneAgent?.cancel() - + // Unbind user service if (componentManager.isServiceConnected) { try { @@ -298,7 +505,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { Logger.e(TAG, "Error unbinding user service", e) } } - + // Note: Don't stop FloatingWindowService here - it should run independently // The service will be stopped when user explicitly closes it or the app process is killed } @@ -310,6 +517,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun initViews() { // Shizuku status views + shizukuCard = findViewById(R.id.shizukuCard) statusText = findViewById(R.id.statusText) shizukuStatusIndicator = findViewById(R.id.shizukuStatusIndicator) shizukuButtonsRow = findViewById(R.id.shizukuButtonsRow) @@ -329,6 +537,12 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { keyboardStatusText = findViewById(R.id.keyboardStatusText) enableKeyboardBtn = findViewById(R.id.enableKeyboardBtn) + // Accessibility permission views + accessibilityCard = findViewById(R.id.accessibilityCard) + accessibilityStatusIcon = findViewById(R.id.accessibilityStatusIcon) + accessibilityStatusText = findViewById(R.id.accessibilityStatusText) + requestAccessibilityBtn = findViewById(R.id.requestAccessibilityBtn) + // Task input views taskInputLayout = findViewById(R.id.taskInputLayout) taskInput = findViewById(R.id.taskInput) @@ -359,7 +573,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { openShizukuApp() } - // Settings button + // Settings button - Requirements: 6.1 settingsBtn.setOnClickListener { startActivity(Intent(this, SettingsActivity::class.java)) } @@ -385,16 +599,23 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { com.kevinluo.autoglm.input.KeyboardHelper.openInputMethodSettings(this) } - // Start task button + // Accessibility permission button + requestAccessibilityBtn.setOnClickListener { + // Open accessibility settings + val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + startActivity(intent) + } + + // Start task button - Requirements: 1.1 startTaskBtn.setOnClickListener { startTask() } - // Cancel task button + // Cancel task button - Requirements: 1.4 cancelTaskBtn.setOnClickListener { cancelTask() } - + // Select template button btnSelectTemplate.setOnClickListener { showTemplateSelectionDialog() @@ -404,7 +625,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { taskInput.setOnFocusChangeListener { _, _ -> updateTaskButtonStates() } - + // Watch for text changes to enable/disable start button taskInput.addTextChangedListener(object : android.text.TextWatcher { override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) {} @@ -541,7 +762,17 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { Shizuku.addBinderReceivedListenerSticky(binderReceivedListener) Shizuku.addBinderDeadListener(binderDeadListener) } - + + /** + * Sets up accessibility state change listener. + * + * Registers listener to detect when accessibility service is enabled/disabled. + */ + private fun setupAccessibilityListener() { + val accessibilityManager = getSystemService(ACCESSIBILITY_SERVICE) as? AccessibilityManager + accessibilityManager?.addAccessibilityStateChangeListener(accessibilityStateChangeListener) + } + /** * Opens the Shizuku app or Play Store if not installed. * @@ -573,23 +804,24 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Called after UserService is connected. Sets up the agent listener * and confirmation callback for sensitive operations. * + * Requirements: 1.1, 2.1, 2.2 */ private fun initializePhoneAgent() { if (!componentManager.isServiceConnected) { Logger.w(TAG, "Cannot initialize PhoneAgent: service not connected") return } - + // Set up listener componentManager.setPhoneAgentListener(this) - + // Setup confirmation callback setupConfirmationCallback() - + updateTaskButtonStates() Logger.i(TAG, "PhoneAgent initialized successfully") } - + /** * Sets up the confirmation callback for sensitive operations. * @@ -635,31 +867,50 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Validates the task description, checks agent state, starts the * floating window service, and launches the task in a coroutine. * + * Requirements: 1.1, 2.1, 2.2 */ private fun startTask() { val taskDescription = taskInput.text?.toString()?.trim() ?: "" - + // Validate task description if (taskDescription.isBlank()) { Toast.makeText(this, R.string.toast_task_empty, Toast.LENGTH_SHORT).show() taskInputLayout.error = getString(R.string.toast_task_empty) return } - + taskInputLayout.error = null - + val agent = componentManager.phoneAgent if (agent == null) { Logger.e(TAG, "PhoneAgent not initialized") + Toast.makeText(this, "Device controller not initialized. Please check your settings.", Toast.LENGTH_SHORT).show() return } - + // Check if already running if (agent.isRunning()) { Logger.w(TAG, "Task already running") return } - + + // Check device controller permissions + val deviceController = componentManager.deviceControllerInstance + if (deviceController != null && !deviceController.checkPermission()) { + val mode = deviceController.getMode() + val message = when (mode) { + com.kevinluo.autoglm.device.DeviceControlMode.ACCESSIBILITY -> { + "Accessibility service not enabled. Please enable it in system settings." + } + com.kevinluo.autoglm.device.DeviceControlMode.SHIZUKU -> { + "Shizuku service not connected. Please ensure Shizuku is running." + } + } + Toast.makeText(this, message, Toast.LENGTH_LONG).show() + deviceController.requestPermission(this) + return + } + // Start floating window service if overlay permission granted if (FloatingWindowService.canDrawOverlays(this)) { Logger.d(TAG, "startTask: Starting floating window service") @@ -669,7 +920,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { FloatingWindowService.requestOverlayPermission(this) return } - + // Update UI state - manually set running state since agent.run() hasn't started yet updateTaskStatus(TaskStatus.RUNNING) // Manually update UI for running state @@ -677,9 +928,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { runningSection.visibility = View.VISIBLE cancelTaskBtn.isEnabled = true taskInput.isEnabled = false - + Logger.i(TAG, "Starting task: $taskDescription") - + // Run task in coroutine lifecycleScope.launch { // Set up callbacks immediately after service starts @@ -711,15 +962,15 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { floatingWindow?.updateStatus(TaskStatus.RUNNING) floatingWindow?.show() } - + // Minimize app to let agent work withContext(Dispatchers.Main) { moveTaskToBack(true) } - + try { val result = agent.run(taskDescription) - + withContext(Dispatchers.Main) { if (result.success) { Logger.i(TAG, "Task completed: ${result.message}") @@ -746,26 +997,27 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Cancels the agent, resets its state, and updates the UI * to reflect the cancelled status. * + * Requirements: 1.1, 2.1, 2.2 */ private fun cancelTask() { Logger.i(TAG, "Cancelling task") - + // Cancel the agent - this will cancel any ongoing network requests componentManager.phoneAgent?.cancel() - + // Reset the agent state so it can accept new tasks componentManager.phoneAgent?.reset() - + Toast.makeText(this, R.string.toast_task_cancelled, Toast.LENGTH_SHORT).show() updateTaskStatus(TaskStatus.FAILED) updateTaskButtonStates() - + // Update floating window to show cancelled state // Use the same message as PhoneAgent for consistency val cancellationMessage = PhoneAgent.CANCELLATION_MESSAGE FloatingWindowService.getInstance()?.showResult(cancellationMessage, false) } - + /** * Pauses the currently running task. * @@ -773,14 +1025,14 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun pauseTask() { Logger.i(TAG, "Pausing task") - + val paused = componentManager.phoneAgent?.pause() == true if (paused) { updateTaskStatus(TaskStatus.PAUSED) FloatingWindowService.getInstance()?.updateStatus(TaskStatus.PAUSED) } } - + /** * Resumes a paused task. * @@ -788,7 +1040,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun resumeTask() { Logger.i(TAG, "Resuming task") - + val resumed = componentManager.phoneAgent?.resume() == true if (resumed) { updateTaskStatus(TaskStatus.RUNNING) @@ -807,15 +1059,15 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { val hasAgent = componentManager.phoneAgent != null val hasTaskText = !taskInput.text.isNullOrBlank() val isRunning = componentManager.phoneAgent?.isRunning() == true - + // Show/hide sections based on running state startTaskBtn.visibility = if (isRunning) View.GONE else View.VISIBLE runningSection.visibility = if (isRunning) View.VISIBLE else View.GONE - + startTaskBtn.isEnabled = hasService && hasAgent && hasTaskText && !isRunning cancelTaskBtn.isEnabled = isRunning taskInput.isEnabled = !isRunning - + Logger.d( TAG, "Button states updated: service=$hasService, agent=$hasAgent, " + @@ -831,6 +1083,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param status The new task status to display * + * Requirements: 1.1, 2.1, 2.2 */ private fun updateTaskStatus(status: TaskStatus) { val (text, colorRes) = when (status) { @@ -842,14 +1095,14 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { TaskStatus.WAITING_CONFIRMATION -> "等待确认" to R.color.status_waiting TaskStatus.WAITING_TAKEOVER -> "等待接管" to R.color.status_waiting } - + taskStatusText.text = text - + val drawable = taskStatusIndicator.background as? GradientDrawable ?: GradientDrawable().also { taskStatusIndicator.background = it } drawable.shape = GradientDrawable.OVAL drawable.setColor(ContextCompat.getColor(this, colorRes)) - + // Also update floating window FloatingWindowService.getInstance()?.updateStatus(status) } @@ -863,6 +1116,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param stepNumber The current step number * + * Requirements: 1.1, 2.1, 2.2 */ override fun onStepStarted(stepNumber: Int) { runOnUiThread { @@ -880,6 +1134,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param thinking The model's thinking text * + * Requirements: 1.1, 2.1, 2.2 */ override fun onThinkingUpdate(thinking: String) { runOnUiThread { @@ -894,6 +1149,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param action The action that was executed * + * Requirements: 1.1, 2.1, 2.2 */ override fun onActionExecuted(action: AgentAction) { runOnUiThread { @@ -910,6 +1166,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param message The completion message * + * Requirements: 1.1, 2.1, 2.2 */ override fun onTaskCompleted(message: String) { runOnUiThread { @@ -928,6 +1185,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * @param error The error message * + * Requirements: 1.1, 2.1, 2.2 */ override fun onTaskFailed(error: String) { runOnUiThread { @@ -943,6 +1201,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * Note: Floating window hide is handled by ScreenshotService. * + * Requirements: 1.1, 2.1, 2.2 */ override fun onScreenshotStarted() { // Floating window hide is handled by ScreenshotService @@ -953,6 +1212,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * * Note: Floating window show is handled by ScreenshotService. * + * Requirements: 1.1, 2.1, 2.2 */ override fun onScreenshotCompleted() { // Floating window show is handled by ScreenshotService @@ -972,7 +1232,22 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { // endregion - // region Shizuku Methods + // region Device Service Status Methods + + /** + * Updates the device service status display based on current control mode. + * + * For Shizuku mode: shows Shizuku connection status. + * For Accessibility mode: shows Accessibility service status. + */ + private fun updateDeviceServiceStatus() { + val controlMode = componentManager.settingsManager.getDeviceControlMode() + + when (controlMode) { + DeviceControlMode.SHIZUKU -> updateShizukuStatusDisplay() + DeviceControlMode.ACCESSIBILITY -> updateAccessibilityStatusDisplay() + } + } /** * Updates the Shizuku connection status display. @@ -980,7 +1255,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { * Checks Shizuku binder status, permission, and service connection, * then updates the UI to reflect the current state. */ - private fun updateShizukuStatus() { + private fun updateShizukuStatusDisplay() { val isBinderAlive = try { Shizuku.pingBinder() } catch (e: Exception) { @@ -996,7 +1271,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { !serviceConnected -> getString(R.string.shizuku_status_connecting) else -> getString(R.string.shizuku_status_connected) } - + val statusColor = when { !isBinderAlive -> R.color.status_failed !hasPermission -> R.color.status_waiting @@ -1007,7 +1282,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { runOnUiThread { statusText.text = statusMessage shizukuStatusIndicator.background.setTint(getColor(statusColor)) - + // Show buttons based on Shizuku state if (serviceConnected) { // Connected - hide buttons row @@ -1021,7 +1296,40 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { requestPermissionBtn.visibility = View.VISIBLE requestPermissionBtn.isEnabled = isBinderAlive } - + + updateTaskButtonStates() + } + } + + /** + * Updates the Accessibility service status display. + * + * Shows the accessibility service connection status in the same location + * where Shizuku status is normally displayed. + */ + private fun updateAccessibilityStatusDisplay() { + val isAccessibilityEnabled = AutoGLMAccessibilityService.isEnabled() + val isAndroidSupported = Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU + + val statusMessage = when { + !isAndroidSupported -> getString(R.string.accessibility_not_supported) + !isAccessibilityEnabled -> getString(R.string.accessibility_permission_denied) + else -> getString(R.string.accessibility_permission_granted) + } + + val statusColor = when { + !isAndroidSupported -> R.color.status_failed + !isAccessibilityEnabled -> R.color.status_waiting + else -> R.color.status_running + } + + runOnUiThread { + statusText.text = statusMessage + shizukuStatusIndicator.background.setTint(getColor(statusColor)) + + // Hide Shizuku buttons in accessibility mode + shizukuButtonsRow.visibility = View.GONE + updateTaskButtonStates() } } @@ -1098,9 +1406,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { private fun formatAction(action: AgentAction): String = action.formatForDisplay() // endregion - + // region Task Templates - + /** * Shows a dialog to select a task template. * @@ -1109,7 +1417,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { */ private fun showTemplateSelectionDialog() { val templates = componentManager.settingsManager.getTaskTemplates() - + if (templates.isEmpty()) { Toast.makeText(this, R.string.settings_no_templates, Toast.LENGTH_SHORT).show() // Offer to go to settings to add templates @@ -1123,9 +1431,9 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { .show() return } - + val templateNames = templates.map { it.name }.toTypedArray() - + AlertDialog.Builder(this) .setTitle(R.string.task_select_template) .setItems(templateNames) { _, which -> @@ -1136,7 +1444,7 @@ class MainActivity : AppCompatActivity(), PhoneAgentListener { .setNegativeButton(R.string.dialog_cancel, null) .show() } - + // endregion companion object { diff --git a/app/src/main/java/com/kevinluo/autoglm/UserService.kt b/app/src/main/java/com/kevinluo/autoglm/UserService.kt index 909022f..9bac2ee 100644 --- a/app/src/main/java/com/kevinluo/autoglm/UserService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/UserService.kt @@ -9,6 +9,8 @@ import java.io.InputStreamReader * This service runs in a separate process with Shizuku permissions. */ class UserService : IUserService.Stub() { + // Internal buffer to store base64-encoded screenshot data + private var screenshotBase64: String? = null /** * Destroys the service and exits the process. @@ -57,6 +59,63 @@ class UserService : IUserService.Stub() { } } + /** + * Captures a screenshot and stores base64 data in memory. + * Returns the base64 string length, or -1 on failure. + */ + override fun captureScreenshotAndGetSize(): Int { + return try { + val process = Runtime.getRuntime().exec(arrayOf("sh", "-c", "screencap -p | base64")) + val reader = BufferedReader(InputStreamReader(process.inputStream)) + val errorReader = BufferedReader(InputStreamReader(process.errorStream)) + + val sb = StringBuilder() + var line: String? + // Read without preserving newlines to avoid wrapping issues + while (reader.readLine().also { line = it } != null) { + sb.append(line) + } + + val errorOutput = StringBuilder() + while (errorReader.readLine().also { line = it } != null) { + errorOutput.append(line).append('\n') + } + + val exitCode = process.waitFor() + reader.close() + errorReader.close() + + if (exitCode != 0 || errorOutput.isNotEmpty()) { + // Failure, clear buffer + screenshotBase64 = null + return -1 + } + + screenshotBase64 = sb.toString() + screenshotBase64?.length ?: -1 + } catch (e: Exception) { + screenshotBase64 = null + -1 + } + } + + /** + * Reads a chunk from the in-memory base64 screenshot. + */ + override fun readScreenshotChunk(offset: Int, size: Int): String { + val data = screenshotBase64 ?: return "" + if (offset < 0 || size <= 0 || offset >= data.length) return "" + val end = kotlin.math.min(offset + size, data.length) + return data.substring(offset, end) + } + + /** + * Clears the in-memory screenshot buffer. + */ + override fun clearScreenshotBuffer() { + screenshotBase64 = null + } + companion object { private const val TAG = "UserService" } diff --git a/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt new file mode 100644 index 0000000..f776a68 --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/accessibility/AutoGLMAccessibilityService.kt @@ -0,0 +1,345 @@ +package com.kevinluo.autoglm.accessibility + +import android.accessibilityservice.AccessibilityService +import android.accessibilityservice.GestureDescription +import android.content.Intent +import android.graphics.Bitmap +import android.graphics.Path +import android.os.Build +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo + +import com.kevinluo.autoglm.util.Logger +import kotlinx.coroutines.suspendCancellableCoroutine +import java.util.concurrent.atomic.AtomicReference +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +/** + * Accessibility Service for device control without Shizuku. + * + * This service provides: + * - Screen capture using takeScreenshot() API (Android 13+) + * - Touch operations via dispatchGesture() + * - Text input via AccessibilityNodeInfo + * - Key press simulation + * - App launching and info retrieval + * + * Requirements: Android 13+ (API 33) for screenshot functionality + */ +class AutoGLMAccessibilityService : AccessibilityService() { + + companion object { + private const val TAG = "AutoGLMAccessibilityService" + + // Gesture timing constants + private const val TAP_DURATION_MS = 100L + private const val DOUBLE_TAP_INTERVAL_MS = 100L + + // Android KeyEvent keycodes + const val KEYCODE_BACK = 4 + const val KEYCODE_HOME = 3 + const val KEYCODE_RECENTS = 187 + const val KEYCODE_NOTIFICATIONS = 4 + const val KEYCODE_QUICK_SETTINGS = 5 + + @Volatile + private var instance: AutoGLMAccessibilityService? = null + + /** + * Gets the singleton instance of the accessibility service. + * + * @return The service instance, or null if not connected + */ + fun getInstance(): AutoGLMAccessibilityService? = instance + + /** + * Checks if the accessibility service is enabled. + * + * @return true if the service is running, false otherwise + */ + fun isEnabled(): Boolean = instance != null + } + + private val serviceState = AtomicReference(ServiceState.IDLE) + + private enum class ServiceState { + IDLE, CONNECTING, CONNECTED, DISCONNECTED + } + + override fun onServiceConnected() { + super.onServiceConnected() + Logger.i(TAG, "AccessibilityService connected") + instance = this + serviceState.set(ServiceState.CONNECTED) + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + // Handle accessibility events if needed + // Currently not used, but can be used for detecting UI changes + } + + override fun onInterrupt() { + Logger.w(TAG, "AccessibilityService interrupted") + } + + override fun onDestroy() { + super.onDestroy() + Logger.i(TAG, "AccessibilityService destroyed") + instance = null + serviceState.set(ServiceState.DISCONNECTED) + } + + // ==================== Screenshot ==================== + + /** + * Captures the current screen content. + * + * Uses AccessibilityService.takeScreenshot() API available on Android 13+. + * + * @return Bitmap of the screen, or null if capture failed + */ + suspend fun captureScreen(): Bitmap? = suspendCancellableCoroutine { continuation -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + try { + Logger.d(TAG, "Starting screenshot capture via AccessibilityService.takeScreenshot()") + takeScreenshot(/* displayId= */ 0, /* executor= */ mainExecutor, + object : TakeScreenshotCallback { + override fun onSuccess(screenshot: ScreenshotResult) { + try { + Logger.d(TAG, "Screenshot callback: onSuccess") + val hardwareBuffer = screenshot.hardwareBuffer + val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, screenshot.colorSpace) + if (bitmap != null) { + Logger.d(TAG, "Successfully wrapped HardwareBuffer to Bitmap: ${bitmap.width}x${bitmap.height}") + continuation.resume(bitmap) + } else { + Logger.w(TAG, "Failed to wrap HardwareBuffer to Bitmap") + continuation.resume(null) + } + hardwareBuffer.close() + } catch (e: Exception) { + Logger.e(TAG, "Failed to process screenshot result", e) + continuation.resume(null) + } + } + + override fun onFailure(error: Int) { + Logger.e(TAG, "Screenshot callback: onFailure with error code: $error") + continuation.resume(null) + } + }) + } catch (e: SecurityException) { + Logger.e(TAG, "SecurityException: Service doesn't have screenshot capability. " + + "Please ensure: 1) Accessibility service is enabled 2) accessibility_service_config.xml has canTakeScreenshot=true 3) Device is Android 13+", e) + continuation.resume(null) + } catch (e: Exception) { + Logger.e(TAG, "Failed to call takeScreenshot", e) + continuation.resume(null) + } + } else { + Logger.w(TAG, "Screenshot not supported on Android < 13 (current: ${Build.VERSION.SDK_INT})") + continuation.resume(null) + } + } + + // ==================== Touch Operations ==================== + + /** + * Performs a tap gesture at the specified coordinates. + * + * @param x X coordinate in pixels + * @param y Y coordinate in pixels + * @return true if gesture was dispatched successfully + */ + suspend fun tap(x: Int, y: Int): Boolean { + val path = Path().apply { + moveTo(x.toFloat(), y.toFloat()) + } + + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, TAP_DURATION_MS)) + .build() + + return dispatchGesture(gesture, null, null) + } + + /** + * Performs a double tap gesture at the specified coordinates. + * + * @param x X coordinate in pixels + * @param y Y coordinate in pixels + * @return true if both taps were dispatched successfully + */ + suspend fun doubleTap(x: Int, y: Int): Boolean { + val firstTap = tap(x, y) + kotlinx.coroutines.delay(DOUBLE_TAP_INTERVAL_MS) + val secondTap = tap(x, y) + return firstTap && secondTap + } + + /** + * Performs a long press gesture at the specified coordinates. + * + * @param x X coordinate in pixels + * @param y Y coordinate in pixels + * @param durationMs Duration of the long press in milliseconds + * @return true if gesture was dispatched successfully + */ + suspend fun longPress(x: Int, y: Int, durationMs: Int): Boolean { + val path = Path().apply { + moveTo(x.toFloat(), y.toFloat()) + } + + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs.toLong())) + .build() + + return dispatchGesture(gesture, null, null) + } + + /** + * Performs a swipe gesture from start to end coordinates. + * + * @param startX Start X coordinate in pixels + * @param startY Start Y coordinate in pixels + * @param endX End X coordinate in pixels + * @param endY End Y coordinate in pixels + * @param durationMs Duration of the swipe in milliseconds + * @return true if gesture was dispatched successfully + */ + suspend fun swipe(startX: Int, startY: Int, endX: Int, endY: Int, durationMs: Int): Boolean { + val path = Path().apply { + moveTo(startX.toFloat(), startY.toFloat()) + lineTo(endX.toFloat(), endY.toFloat()) + } + + val gesture = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs.toLong())) + .build() + + return dispatchGesture(gesture, null, null) + } + + // ==================== Key Press ==================== + + /** + * Performs a key press event. + * + * Uses performGlobalAction for system keys and fallback to dispatchKeyEvent. + * + * @param keyCode Android KeyEvent keycode + * @return true if key press was successful + */ + fun pressKey(keyCode: Int): Boolean { + return when (keyCode) { + KEYCODE_BACK -> performGlobalAction(GLOBAL_ACTION_BACK) + KEYCODE_HOME -> performGlobalAction(GLOBAL_ACTION_HOME) + KEYCODE_RECENTS -> performGlobalAction(GLOBAL_ACTION_RECENTS) + KEYCODE_NOTIFICATIONS -> performGlobalAction(GLOBAL_ACTION_NOTIFICATIONS) + KEYCODE_QUICK_SETTINGS -> performGlobalAction(GLOBAL_ACTION_QUICK_SETTINGS) + else -> { + Logger.w(TAG, "Key code $keyCode not supported via accessibility") + false + } + } + } + + // ==================== Text Input ==================== + + /** + * Inputs text into the currently focused editable field. + * + * @param text The text to input + * @return true if text was entered successfully + */ + fun inputText(text: String): Boolean { + val rootNode = rootInActiveWindow ?: run { + Logger.w(TAG, "No active window root node") + return false + } + + val focusedNode = findFocusedEditableNode(rootNode) ?: run { + Logger.w(TAG, "No focused editable node found") + return false + } + + return try { + // Clear existing text + focusedNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, createSetTextBundle("")) + + // Set new text + focusedNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, createSetTextBundle(text)) + + Logger.d(TAG, "Text input successful: '${text.take(30)}...'") + true + } catch (e: Exception) { + Logger.e(TAG, "Failed to input text", e) + false + } + } + + /** + * Finds the currently focused editable node in the node tree. + * + * @param node The root node to search from + * @return The focused editable node, or null if not found + */ + private fun findFocusedEditableNode(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { + // Check if this node is focused and editable + if (node.isFocused && node.isEditable) { + return node + } + + // Recursively search children + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + val result = findFocusedEditableNode(child) + if (result != null) { + return result + } + // Note: AccessibilityNodeInfo.recycle() is deprecated and no longer needed + // The garbage collector will handle cleanup automatically + } + + return null + } + + /** + * Creates a bundle for setting text in an editable node. + * + * @param text The text to set + * @return Bundle with text argument + */ + private fun createSetTextBundle(text: String): android.os.Bundle { + return android.os.Bundle().apply { + putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) + } + } + + // ==================== App Operations ==================== + + /** + * Gets the current foreground app's package name. + * + * @return Package name of the current app, or empty string if not found + */ + fun getCurrentApp(): String { + val rootNode = rootInActiveWindow ?: return "" + + // Try to get package name from window info + val packageName = rootNode.packageName?.toString() + + return packageName ?: "" + } + + /** + * Opens the accessibility settings for this app. + * + * @param intent The intent to start the accessibility settings + */ + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + // Handle any commands sent to the service + return START_STICKY + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt b/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt index 100cda2..6ee5c5b 100644 --- a/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt +++ b/app/src/main/java/com/kevinluo/autoglm/action/ActionHandler.kt @@ -1,8 +1,7 @@ package com.kevinluo.autoglm.action import com.kevinluo.autoglm.app.AppResolver -import com.kevinluo.autoglm.device.DeviceExecutor -import com.kevinluo.autoglm.input.TextInputManager +import com.kevinluo.autoglm.device.IDeviceController import com.kevinluo.autoglm.screenshot.FloatingWindowController import com.kevinluo.autoglm.util.CoordinateConverter import com.kevinluo.autoglm.util.ErrorHandler @@ -11,25 +10,23 @@ import com.kevinluo.autoglm.util.Logger import kotlinx.coroutines.delay /** - * Handles execution of agent actions by coordinating with DeviceExecutor, - * AppResolver, HumanizedSwipeGenerator, and TextInputManager. + * Handles execution of agent actions by coordinating with IDeviceController, + * AppResolver, and HumanizedSwipeGenerator. * * This class is responsible for translating high-level [AgentAction] commands * into device-level operations. It manages floating window visibility during * touch operations to prevent interference. * - * @param deviceExecutor Executor for device-level operations (tap, swipe, etc.) + * @param deviceController Device controller for device-level operations (tap, swipe, text input, etc.) * @param appResolver Resolver for app name to package name mapping * @param swipeGenerator Generator for humanized swipe paths - * @param textInputManager Manager for text input operations * @param floatingWindowProvider Optional provider for floating window controller * */ class ActionHandler( - private val deviceExecutor: DeviceExecutor, + private val deviceController: IDeviceController, private val appResolver: AppResolver, private val swipeGenerator: HumanizedSwipeGenerator, - private val textInputManager: TextInputManager, private val floatingWindowProvider: (() -> FloatingWindowController?)? = null ) { @@ -97,6 +94,26 @@ class ActionHandler( floatingWindowProvider?.invoke()?.show() } + /** + * Handles text input via the device controller. + * + * @param text The text to input + * @return ActionResult with success status and message + */ + private suspend fun inputText(text: String): ActionResult { + return try { + val result = deviceController.inputText(text) + if (result.contains("Error", ignoreCase = true)) { + ActionResult(false, false, result) + } else { + ActionResult(true, false, result) + } + } catch (e: Exception) { + Logger.e(TAG, "Text input failed", e) + ActionResult(false, false, "文本输入失败: ${e.message}") + } + } + /** * Executes an agent action on the device. * @@ -180,7 +197,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.tap(absX, absY) + val result = deviceController.tap(absX, absY) if (isDeviceExecutorError(result)) { Logger.w(TAG, "Tap command failed: $result") ActionResult(false, false, "点击失败: $result") @@ -224,7 +241,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.swipe(swipePath.points, swipePath.durationMs) + val result = deviceController.swipe(swipePath.points, swipePath.durationMs) // Wait for swipe animation to complete before showing floating window // The swipe command returns immediately, but the gesture takes time @@ -259,9 +276,8 @@ class ActionHandler( return try { // Small delay to let the system settle focus delay(200) - - val result = textInputManager.typeText(action.text) - ActionResult(result.success, false, result.message) + + inputText(action.text) } finally { // Always show floating window after typing, even if typing fails showFloatingWindow() @@ -279,8 +295,8 @@ class ActionHandler( return try { // Small delay to let the system settle focus delay(200) - - val result = textInputManager.typeText(action.text) + + val result = inputText(action.text) ActionResult(result.success, false, "输入名称: ${action.text}") } finally { // Always show floating window after typing, even if typing fails @@ -310,7 +326,7 @@ class ActionHandler( return if (packageName != null) { Logger.i(TAG, "Launching package: $packageName") - val launchResult = deviceExecutor.launchApp(packageName) + val launchResult = deviceController.launchApp(packageName) // Check if launch was successful by examining the result val isError = launchResult.contains("Error", ignoreCase = true) || @@ -321,7 +337,7 @@ class ActionHandler( if (isError) { Logger.w(TAG, "Launch failed for $packageName: $launchResult") // Launch failed - instruct model to find app icon on screen - deviceExecutor.pressKey(DeviceExecutor.KEYCODE_HOME) + deviceController.pressKey(IDeviceController.KEYCODE_HOME) ActionResult( success = true, // Operation itself succeeded, just app not found shouldFinish = false, @@ -334,7 +350,7 @@ class ActionHandler( // Package not found - instruct model to find app icon on screen Logger.i(TAG, "Package not found for '${action.app}', instructing model to find app icon on screen") // Press Home first to go to home screen - deviceExecutor.pressKey(DeviceExecutor.KEYCODE_HOME) + deviceController.pressKey(IDeviceController.KEYCODE_HOME) ActionResult( success = true, shouldFinish = false, @@ -378,11 +394,11 @@ class ActionHandler( private suspend fun executeBack(): ActionResult { // First, dismiss keyboard with ESCAPE key to ensure Back actually navigates // If keyboard is shown, the first Back would just close it - deviceExecutor.pressKey(111) // KEYCODE_ESCAPE + deviceController.pressKey(111) // KEYCODE_ESCAPE delay(100) // Now press Back to navigate - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_BACK) + val result = deviceController.pressKey(IDeviceController.KEYCODE_BACK) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Back key press failed: $result") ActionResult(false, false, "返回键失败: $result") @@ -395,7 +411,7 @@ class ActionHandler( * Executes a Home action. */ private suspend fun executeHome(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_HOME) + val result = deviceController.pressKey(IDeviceController.KEYCODE_HOME) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Home key press failed: $result") ActionResult(false, false, "主页键失败: $result") @@ -408,7 +424,7 @@ class ActionHandler( * Executes a VolumeUp action. */ private suspend fun executeVolumeUp(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_VOLUME_UP) + val result = deviceController.pressKey(IDeviceController.KEYCODE_VOLUME_UP) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Volume up key press failed: $result") ActionResult(false, false, "音量+键失败: $result") @@ -421,7 +437,7 @@ class ActionHandler( * Executes a VolumeDown action. */ private suspend fun executeVolumeDown(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_VOLUME_DOWN) + val result = deviceController.pressKey(IDeviceController.KEYCODE_VOLUME_DOWN) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Volume down key press failed: $result") ActionResult(false, false, "音量-键失败: $result") @@ -434,7 +450,7 @@ class ActionHandler( * Executes a Power action. */ private suspend fun executePower(): ActionResult { - val result = deviceExecutor.pressKey(DeviceExecutor.KEYCODE_POWER) + val result = deviceController.pressKey(IDeviceController.KEYCODE_POWER) return if (isDeviceExecutorError(result)) { Logger.w(TAG, "Power key press failed: $result") ActionResult(false, false, "电源键失败: $result") @@ -462,7 +478,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.longPress(absX, absY, action.durationMs) + val result = deviceController.longPress(absX, absY, action.durationMs) // Wait for long press to complete before showing floating window // The command returns immediately, but the gesture takes time @@ -498,7 +514,7 @@ class ActionHandler( hideFloatingWindow() return try { - val result = deviceExecutor.doubleTap(absX, absY) + val result = deviceController.doubleTap(absX, absY) if (isDeviceExecutorError(result)) { Logger.w(TAG, "Double tap command failed: $result") ActionResult(false, false, "双击失败: $result") diff --git a/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt new file mode 100644 index 0000000..c65fb59 --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/device/AccessibilityDeviceController.kt @@ -0,0 +1,283 @@ +package com.kevinluo.autoglm.device + +import android.content.Context +import android.content.Intent +import android.graphics.Bitmap +import android.os.Build +import android.provider.Settings +import com.kevinluo.autoglm.accessibility.AutoGLMAccessibilityService +import com.kevinluo.autoglm.screenshot.Screenshot +import com.kevinluo.autoglm.util.Logger +import com.kevinluo.autoglm.util.Point +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import android.util.Base64 + +/** + * Device controller implementation using Android Accessibility Service. + * + * This controller uses the AccessibilityService API for device control. + * It requires the accessibility service to be enabled in system settings. + * + * @param context Android context + * + * Requirements: Android 13+ (API 33) for screenshot functionality + */ +class AccessibilityDeviceController( + private val context: Context +) : IDeviceController { + + override fun getMode(): DeviceControlMode = DeviceControlMode.ACCESSIBILITY + + override fun checkPermission(): Boolean { + return AutoGLMAccessibilityService.isEnabled() + } + + override fun requestPermission(context: Context): Boolean { + try { + val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + context.startActivity(intent) + return true + } catch (e: Exception) { + Logger.e(TAG, "Failed to open accessibility settings", e) + return false + } + } + + override suspend fun tap(x: Int, y: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.tap(x, y) + if (result) { + "Tap at ($x, $y) succeeded" + } else { + "Tap at ($x, $y) failed" + } + } + + override suspend fun doubleTap(x: Int, y: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.doubleTap(x, y) + if (result) { + "Double tap at ($x, $y) succeeded" + } else { + "Double tap at ($x, $y) failed" + } + } + + override suspend fun longPress(x: Int, y: Int, durationMs: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.longPress(x, y, durationMs) + if (result) { + "Long press at ($x, $y) for ${durationMs}ms succeeded" + } else { + "Long press at ($x, $y) failed" + } + } + + override suspend fun swipe(points: List, durationMs: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + if (points.size < 2) { + return@withContext "Error: Swipe requires at least 2 points" + } + + // Perform swipe between each consecutive point + var allSuccess = true + for (i in 0 until points.size - 1) { + val start = points[i] + val end = points[i + 1] + val segmentDuration = durationMs / (points.size - 1) + + val result = service.swipe(start.x, start.y, end.x, end.y, segmentDuration) + if (!result) { + allSuccess = false + } + + // Small delay between segments for smooth multi-point swipe + if (i < points.size - 2) { + kotlinx.coroutines.delay(20) + } + } + + if (allSuccess) { + "Swipe with ${points.size} points succeeded" + } else { + "Swipe completed with some segments failed" + } + } + + override suspend fun pressKey(keyCode: Int): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.pressKey(keyCode) + if (result) { + "Key press $keyCode succeeded" + } else { + "Key press $keyCode failed" + } + } + + override suspend fun launchApp(packageName: String): String = withContext(Dispatchers.Main) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + // For accessibility mode, we can't directly launch apps via shell + // We need to use a different approach or return an error + // One option is to use PackageManager to get the launch intent + try { + val pm = context.packageManager + val intent = pm.getLaunchIntentForPackage(packageName) + if (intent != null) { + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + context.startActivity(intent) + "Launched app: $packageName" + } else { + "Error: No launch intent found for $packageName" + } + } catch (e: Exception) { + Logger.e(TAG, "Failed to launch app: $packageName", e) + "Error: Failed to launch $packageName: ${e.message}" + } + } else { + "Error: App launch not supported on Android < 5.0" + } + } + + override suspend fun getCurrentApp(): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "" + } + + service.getCurrentApp() + } + + override suspend fun inputText(text: String): String = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + return@withContext "Error: Accessibility service not connected" + } + + val result = service.inputText(text) + if (result) { + "Text input successful: '${text.take(30)}...'" + } else { + "Text input failed: No focused editable field" + } + } + + override suspend fun captureScreen(): Screenshot = withContext(Dispatchers.Main) { + val service = AutoGLMAccessibilityService.getInstance() + if (service == null) { + Logger.e(TAG, "Accessibility service not connected") + return@withContext createFallbackScreenshot() + } + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) { + Logger.w(TAG, "Screenshot not supported on Android < 13 (API 33)") + return@withContext createFallbackScreenshot() + } + + try { + Logger.d(TAG, "Attempting to capture screenshot via AccessibilityService") + val bitmap = service.captureScreen() + if (bitmap != null) { + val width = bitmap.width + val height = bitmap.height + Logger.d(TAG, "Screenshot captured successfully: ${width}x${height}") + + // Convert bitmap to Screenshot + val base64Data = encodeBitmapToBase64(bitmap) + bitmap.recycle() + + Screenshot( + base64Data = base64Data, + width = width, + height = height, + originalWidth = width, + originalHeight = height, + isSensitive = false + ) + } else { + Logger.w(TAG, "Screenshot capture returned null bitmap") + createFallbackScreenshot() + } + } catch (e: SecurityException) { + Logger.e(TAG, "Security exception when capturing screenshot - service may not have screenshot capability", e) + createFallbackScreenshot() + } catch (e: Exception) { + Logger.e(TAG, "Failed to capture screenshot via accessibility service", e) + createFallbackScreenshot() + } + } + + /** + * Creates a fallback black screenshot when capture fails. + */ + private fun createFallbackScreenshot(): Screenshot { + val bitmap = Bitmap.createBitmap(1080, 1920, Bitmap.Config.ARGB_8888) + bitmap.eraseColor(android.graphics.Color.BLACK) + + val base64Data = encodeBitmapToBase64(bitmap) + bitmap.recycle() + + return Screenshot( + base64Data = base64Data, + width = 1080, + height = 1920, + isSensitive = true + ) + } + + /** + * Encodes a Bitmap to a base64 string. + * + * @param bitmap The bitmap to encode + * @param format Compression format (default: WEBP_LOSSY) + * @param quality Compression quality 0-100 (default: 80) + * @return Base64-encoded string of the compressed image + */ + private fun encodeBitmapToBase64( + bitmap: Bitmap, + format: Bitmap.CompressFormat? = null, + quality: Int = 80 + ): String { + val compressFormat = format ?: if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + + val outputStream = ByteArrayOutputStream() + bitmap.compress(compressFormat, quality, outputStream) + return Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP) + } + + companion object { + private const val TAG = "AccessibilityDeviceController" + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt new file mode 100644 index 0000000..66cd59e --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/device/IDeviceController.kt @@ -0,0 +1,171 @@ +package com.kevinluo.autoglm.device + +import android.content.Context +import com.kevinluo.autoglm.screenshot.Screenshot +import com.kevinluo.autoglm.util.Point + +/** + * Device control mode enumeration. + * + * Defines the available methods for controlling the device. + * + * @property SHIZUKU Uses Shizuku shell commands (requires Shizuku app) + * @property ACCESSIBILITY Uses Android Accessibility Service + */ +enum class DeviceControlMode(val displayName: String) { + SHIZUKU("Shizuku"), + ACCESSIBILITY("无障碍服务") +} + +/** + * Abstract interface for device control operations. + * + * This interface defines the contract for device manipulation regardless of the + * underlying implementation (Shizuku shell commands or Accessibility Service). + * + * Implementations must handle: + * - Touch operations (tap, swipe, long press, double tap) + * - Key press events + * - App launching and info retrieval + * - Text input (method varies by implementation) + * - Screen capture + * - Permission management + * + * Requirements: Integration with both Shizuku and Accessibility modes + */ +interface IDeviceController { + + /** + * Gets the control mode of this device controller. + * + * @return The device control mode (SHIZUKU or ACCESSIBILITY) + */ + fun getMode(): DeviceControlMode + + /** + * Checks if the required permissions are granted. + * + * For Shizuku mode: Checks if Shizuku service is connected + * For Accessibility mode: Checks if accessibility service is enabled + * + * @return true if all required permissions are granted, false otherwise + */ + fun checkPermission(): Boolean + + /** + * Requests the required permissions from the user. + * + * For Shizuku mode: Launches Shizuku app or shows setup instructions + * For Accessibility mode: Opens system accessibility settings + * + * @param context Android context for starting intents + * @return true if the request was initiated successfully, false otherwise + */ + fun requestPermission(context: Context): Boolean + + // ==================== Touch Operations ==================== + + /** + * Performs a tap at the specified absolute coordinates. + * + * @param x Absolute X coordinate in pixels + * @param y Absolute Y coordinate in pixels + * @return Result of the operation + */ + suspend fun tap(x: Int, y: Int): String + + /** + * Performs a double tap at the specified absolute coordinates. + * + * @param x Absolute X coordinate in pixels + * @param y Absolute Y coordinate in pixels + * @return Result of the operation + */ + suspend fun doubleTap(x: Int, y: Int): String + + /** + * Performs a long press at the specified absolute coordinates. + * + * @param x Absolute X coordinate in pixels + * @param y Absolute Y coordinate in pixels + * @param durationMs Duration of the long press in milliseconds + * @return Result of the operation + */ + suspend fun longPress(x: Int, y: Int, durationMs: Int = 3000): String + + /** + * Performs a swipe gesture using a list of points. + * + * @param points List of points defining the swipe path, must contain at least 2 points + * @param durationMs Total duration of the swipe in milliseconds + * @return Result of the operation + */ + suspend fun swipe(points: List, durationMs: Int): String + + // ==================== Key Press Operations ==================== + + /** + * Presses a key by its keycode. + * + * @param keyCode The Android KeyEvent keycode (e.g., KEYCODE_BACK, KEYCODE_HOME) + * @return Result of the operation + */ + suspend fun pressKey(keyCode: Int): String + + // ==================== App Operations ==================== + + /** + * Launches an app by its package name. + * + * @param packageName The package name of the app to launch + * @return Result of the operation + */ + suspend fun launchApp(packageName: String): String + + /** + * Gets the current foreground app's package name. + * + * @return The package name of the current foreground app, or empty string if not found + */ + suspend fun getCurrentApp(): String + + // ==================== Text Input ==================== + + /** + * Inputs text into the currently focused input field. + * + * Implementation varies by mode: + * - Shizuku: Uses keyboard switching via TextInputManager + * - Accessibility: Directly injects text via AccessibilityNodeInfo + * + * @param text The text to input + * @return Result of the operation + */ + suspend fun inputText(text: String): String + + // ==================== Screenshot ==================== + + /** + * Captures the current screen content. + * + * Implementation varies by mode: + * - Shizuku: Uses screencap shell command + * - Accessibility: Uses AccessibilityService.takeScreenshot() (Android 13+) + * + * @return Screenshot object containing the captured image data and metadata + */ + suspend fun captureScreen(): Screenshot + + // ==================== Constants ==================== + + companion object { + // Android KeyEvent keycodes + const val KEYCODE_BACK = 4 + const val KEYCODE_HOME = 3 + const val KEYCODE_VOLUME_UP = 24 + const val KEYCODE_VOLUME_DOWN = 25 + const val KEYCODE_POWER = 26 + const val KEYCODE_ENTER = 66 + const val KEYCODE_DEL = 67 + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt b/app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt new file mode 100644 index 0000000..8ff5d5d --- /dev/null +++ b/app/src/main/java/com/kevinluo/autoglm/device/ShizukuDeviceController.kt @@ -0,0 +1,101 @@ +package com.kevinluo.autoglm.device + +import android.content.Context +import com.kevinluo.autoglm.IUserService +import com.kevinluo.autoglm.input.TextInputManager +import com.kevinluo.autoglm.screenshot.Screenshot +import com.kevinluo.autoglm.screenshot.ScreenshotService +import com.kevinluo.autoglm.util.Logger + +/** + * Device controller implementation using Shizuku shell commands. + * + * This controller uses the Shizuku framework to execute shell commands for device control. + * It requires the Shizuku app to be installed and the user service to be connected. + * + * @param userService Shizuku user service for executing shell commands + * @param textInputManager Manager for text input operations (keyboard switching) + * @param screenshotProvider Provider function for screenshot service + * + * Requirements: Shizuku-based device control + */ +class ShizukuDeviceController( + private val userService: IUserService, + private val textInputManager: TextInputManager, + private val screenshotProvider: () -> ScreenshotService +) : IDeviceController { + + private val executor = DeviceExecutor(userService) + + override fun getMode(): DeviceControlMode = DeviceControlMode.SHIZUKU + + override fun checkPermission(): Boolean { + return try { + // Try to execute a simple command to check if Shizuku is connected + val result = userService.executeCommand("echo test") + result.contains("test") + } catch (e: Exception) { + Logger.w(TAG, "Shizuku permission check failed: ${e.message}") + false + } + } + + override fun requestPermission(context: Context): Boolean { + try { + // Launch Shizuku app + val intent = context.packageManager.getLaunchIntentForPackage("moe.shizuku.privileged.api") + if (intent != null) { + context.startActivity(intent) + return true + } + } catch (e: Exception) { + Logger.e(TAG, "Failed to launch Shizuku app", e) + } + return false + } + + override suspend fun tap(x: Int, y: Int): String { + return executor.tap(x, y) + } + + override suspend fun doubleTap(x: Int, y: Int): String { + return executor.doubleTap(x, y) + } + + override suspend fun longPress(x: Int, y: Int, durationMs: Int): String { + return executor.longPress(x, y, durationMs) + } + + override suspend fun swipe(points: List, durationMs: Int): String { + return executor.swipe(points, durationMs) + } + + override suspend fun pressKey(keyCode: Int): String { + return executor.pressKey(keyCode) + } + + override suspend fun launchApp(packageName: String): String { + return executor.launchApp(packageName) + } + + override suspend fun getCurrentApp(): String { + return executor.getCurrentApp() + } + + override suspend fun inputText(text: String): String { + val result = textInputManager.typeText(text) + return if (result.success) { + "Text input successful: ${text.take(30)}..." + } else { + "Text input failed: ${result.message}" + } + } + + override suspend fun captureScreen(): Screenshot { + return screenshotProvider().capture() + } + + companion object { + private const val TAG = "ShizukuDeviceController" + } +} diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt index 0dafc97..b15f616 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryActivity.kt @@ -10,6 +10,7 @@ import android.widget.ImageButton import android.widget.ImageView import android.widget.LinearLayout import android.widget.TextView +import androidx.activity.OnBackPressedCallback import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AppCompatActivity import androidx.core.content.ContextCompat @@ -39,29 +40,42 @@ import java.util.Locale * */ class HistoryActivity : AppCompatActivity() { - + private lateinit var historyManager: HistoryManager private lateinit var recyclerView: RecyclerView private lateinit var emptyState: LinearLayout private lateinit var adapter: HistoryAdapter - + // Multi-select mode private lateinit var normalToolbar: LinearLayout private lateinit var selectionToolbar: LinearLayout private lateinit var selectionCountText: TextView private var isSelectionMode = false - + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_history) - + historyManager = HistoryManager.getInstance(this) - + Logger.d(TAG, "HistoryActivity created") setupViews() observeHistory() + + // Handle back press using modern OnBackPressedDispatcher + val backCallback = object : OnBackPressedCallback(true) { + override fun handleOnBackPressed() { + if (isSelectionMode) { + exitSelectionMode() + } else { + isEnabled = false + onBackPressedDispatcher.onBackPressed() + } + } + } + onBackPressedDispatcher.addCallback(this, backCallback) } - + /** * Sets up all view references and click listeners. */ @@ -69,31 +83,31 @@ class HistoryActivity : AppCompatActivity() { normalToolbar = findViewById(R.id.normalToolbar) selectionToolbar = findViewById(R.id.selectionToolbar) selectionCountText = findViewById(R.id.selectionCountText) - + findViewById(R.id.backBtn).setOnClickListener { finish() } - + findViewById(R.id.clearAllBtn).setOnClickListener { showClearAllDialog() } - + // Selection toolbar buttons findViewById(R.id.cancelSelectionBtn).setOnClickListener { exitSelectionMode() } - + findViewById(R.id.selectAllBtn).setOnClickListener { adapter.selectAll() } - + findViewById(R.id.deleteSelectedBtn).setOnClickListener { showDeleteSelectedDialog() } - + recyclerView = findViewById(R.id.historyRecyclerView) emptyState = findViewById(R.id.emptyState) - + adapter = HistoryAdapter( onItemClick = { task -> if (isSelectionMode) { @@ -115,11 +129,11 @@ class HistoryActivity : AppCompatActivity() { } } ) - + recyclerView.layoutManager = LinearLayoutManager(this) recyclerView.adapter = adapter } - + /** * Observes the history list and updates the UI accordingly. */ @@ -129,7 +143,7 @@ class HistoryActivity : AppCompatActivity() { adapter.submitList(list) emptyState.visibility = if (list.isEmpty()) View.VISIBLE else View.GONE recyclerView.visibility = if (list.isEmpty()) View.GONE else View.VISIBLE - + // Exit selection mode if list becomes empty if (list.isEmpty() && isSelectionMode) { exitSelectionMode() @@ -137,7 +151,7 @@ class HistoryActivity : AppCompatActivity() { } } } - + /** * Enters multi-select mode for batch operations. */ @@ -148,7 +162,7 @@ class HistoryActivity : AppCompatActivity() { selectionToolbar.visibility = View.VISIBLE Logger.d(TAG, "Entered selection mode") } - + /** * Exits multi-select mode and clears selection. */ @@ -160,7 +174,7 @@ class HistoryActivity : AppCompatActivity() { selectionToolbar.visibility = View.GONE Logger.d(TAG, "Exited selection mode") } - + /** * Updates the selection count display in the toolbar. * @@ -169,7 +183,7 @@ class HistoryActivity : AppCompatActivity() { private fun updateSelectionCount(count: Int) { selectionCountText.text = getString(R.string.history_selected_count, count) } - + /** * Opens the task detail activity for the given task. * @@ -181,14 +195,14 @@ class HistoryActivity : AppCompatActivity() { intent.putExtra(HistoryDetailActivity.EXTRA_TASK_ID, task.id) startActivity(intent) } - + /** * Shows a confirmation dialog for deleting selected tasks. */ private fun showDeleteSelectedDialog() { val selectedIds = adapter.getSelectedIds() if (selectedIds.isEmpty()) return - + AlertDialog.Builder(this) .setTitle(R.string.history_delete_selected) .setMessage(getString(R.string.history_delete_selected_confirm, selectedIds.size)) @@ -201,7 +215,7 @@ class HistoryActivity : AppCompatActivity() { .setNegativeButton(R.string.dialog_cancel, null) .show() } - + /** * Shows a confirmation dialog for clearing all history. */ @@ -218,16 +232,7 @@ class HistoryActivity : AppCompatActivity() { .setNegativeButton(R.string.dialog_cancel, null) .show() } - - @Deprecated("Deprecated in Java") - override fun onBackPressed() { - if (isSelectionMode) { - exitSelectionMode() - } else { - super.onBackPressed() - } - } - + companion object { private const val TAG = "HistoryActivity" } @@ -250,12 +255,12 @@ class HistoryAdapter( private val onItemLongClick: (TaskHistory) -> Unit, private val onSelectionChanged: (Int) -> Unit ) : RecyclerView.Adapter() { - + private var items: List = emptyList() private val selectedIds = mutableSetOf() private var isSelectionMode = false private val dateFormat = SimpleDateFormat("MM-dd HH:mm", Locale.getDefault()) - + /** * Submits a new list of items to display. * @@ -267,7 +272,7 @@ class HistoryAdapter( selectedIds.retainAll(list.map { it.id }.toSet()) notifyDataSetChanged() } - + /** * Enables or disables selection mode. * @@ -277,7 +282,7 @@ class HistoryAdapter( isSelectionMode = enabled notifyDataSetChanged() } - + /** * Toggles the selection state of a task. * @@ -292,7 +297,7 @@ class HistoryAdapter( notifyDataSetChanged() onSelectionChanged(selectedIds.size) } - + /** * Selects all items in the list. */ @@ -302,7 +307,7 @@ class HistoryAdapter( notifyDataSetChanged() onSelectionChanged(selectedIds.size) } - + /** * Clears all selections. */ @@ -311,26 +316,26 @@ class HistoryAdapter( notifyDataSetChanged() onSelectionChanged(0) } - + /** * Gets the set of currently selected task IDs. * * @return Immutable copy of selected task IDs */ fun getSelectedIds(): Set = selectedIds.toSet() - + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { val view = LayoutInflater.from(parent.context) .inflate(R.layout.item_history_task, parent, false) return ViewHolder(view) } - + override fun onBindViewHolder(holder: ViewHolder, position: Int) { holder.bind(items[position]) } - + override fun getItemCount(): Int = items.size - + /** * ViewHolder for history list items. * @@ -343,7 +348,7 @@ class HistoryAdapter( private val timeText: TextView = itemView.findViewById(R.id.timeText) private val stepsText: TextView = itemView.findViewById(R.id.stepsText) private val durationText: TextView = itemView.findViewById(R.id.durationText) - + /** * Binds task data to the view. * @@ -357,7 +362,7 @@ class HistoryAdapter( R.string.history_duration_format, formatDuration(task.duration) ) - + if (task.success) { statusIcon.setImageResource(R.drawable.ic_check_circle) statusIcon.setColorFilter( @@ -369,22 +374,22 @@ class HistoryAdapter( ContextCompat.getColor(itemView.context, R.color.status_error) ) } - + // Handle selection mode checkBox.visibility = if (isSelectionMode) View.VISIBLE else View.GONE checkBox.isChecked = selectedIds.contains(task.id) - + checkBox.setOnClickListener { toggleSelection(task.id) } - + itemView.setOnClickListener { onItemClick(task) } itemView.setOnLongClickListener { onItemLongClick(task) true } } - + /** * Formats duration in milliseconds to a human-readable string. * diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt index a2deb84..7c5e41e 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryDetailActivity.kt @@ -50,28 +50,28 @@ import java.util.Locale * */ class HistoryDetailActivity : AppCompatActivity() { - + private lateinit var historyManager: HistoryManager private var taskId: String? = null private var task: TaskHistory? = null - + private lateinit var contentRecyclerView: RecyclerView private lateinit var detailAdapter: HistoryDetailAdapter - + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()) - + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_history_detail) - + historyManager = HistoryManager.getInstance(this) taskId = intent.getStringExtra(EXTRA_TASK_ID) - + Logger.d(TAG, "HistoryDetailActivity created for task: $taskId") setupViews() loadTask() } - + /** * Sets up all view references and click listeners. */ @@ -79,23 +79,23 @@ class HistoryDetailActivity : AppCompatActivity() { findViewById(R.id.backBtn).setOnClickListener { finish() } - + findViewById(R.id.copyPromptBtn).setOnClickListener { copyPromptToClipboard() } - + findViewById(R.id.saveImageBtn).setOnClickListener { saveAsImage() } - + findViewById(R.id.shareBtn).setOnClickListener { shareAsImage() } - + findViewById(R.id.deleteBtn).setOnClickListener { showDeleteDialog() } - + // Setup RecyclerView with true recycling contentRecyclerView = findViewById(R.id.contentRecyclerView) detailAdapter = HistoryDetailAdapter(historyManager, lifecycleScope) @@ -107,27 +107,27 @@ class HistoryDetailActivity : AppCompatActivity() { setItemViewCacheSize(3) } } - + /** * Loads the task from history manager. */ private fun loadTask() { val id = taskId ?: return - + lifecycleScope.launch { task = historyManager.getTask(id) - task?.let { + task?.let { Logger.d(TAG, "Loaded task with ${it.stepCount} steps") - detailAdapter.setTask(it) + detailAdapter.setTask(it) } } } - + override fun onDestroy() { super.onDestroy() detailAdapter.cleanup() } - + /** * Shows a confirmation dialog for deleting the task. */ @@ -140,22 +140,22 @@ class HistoryDetailActivity : AppCompatActivity() { .setNegativeButton(R.string.dialog_cancel, null) .show() } - + /** * Copies the task prompt/description to clipboard. * */ private fun copyPromptToClipboard() { val currentTask = task ?: return - + val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager val clip = ClipData.newPlainText("AutoGLM Prompt", currentTask.taskDescription) clipboard.setPrimaryClip(clip) - + Logger.d(TAG, "Copied prompt to clipboard") Toast.makeText(this, R.string.history_prompt_copied, Toast.LENGTH_SHORT).show() } - + /** * Deletes the current task from history. */ @@ -167,7 +167,7 @@ class HistoryDetailActivity : AppCompatActivity() { finish() } } - + /** * Formats duration in milliseconds to a human-readable string. * @@ -182,7 +182,7 @@ class HistoryDetailActivity : AppCompatActivity() { else -> "${seconds / 3600}时${(seconds % 3600) / 60}分" } } - + /** * Saves the task history as an image to gallery. * @@ -191,22 +191,22 @@ class HistoryDetailActivity : AppCompatActivity() { */ private fun saveAsImage() { val currentTask = task ?: return - + Logger.d(TAG, "Saving task as image") Toast.makeText(this, R.string.history_generating_image, Toast.LENGTH_SHORT).show() - + lifecycleScope.launch { try { val bitmap = withContext(Dispatchers.Default) { generateShareImage(currentTask) } - + val saved = withContext(Dispatchers.IO) { saveBitmapToGallery(bitmap) } - + bitmap.recycle() - + if (saved) { Logger.d(TAG, "Image saved to gallery") Toast.makeText(this@HistoryDetailActivity, R.string.history_save_success, Toast.LENGTH_SHORT).show() @@ -220,7 +220,7 @@ class HistoryDetailActivity : AppCompatActivity() { } } } - + /** * Shares the task history as an image. * @@ -229,24 +229,24 @@ class HistoryDetailActivity : AppCompatActivity() { */ private fun shareAsImage() { val currentTask = task ?: return - + Logger.d(TAG, "Sharing task as image") Toast.makeText(this, R.string.history_generating_image, Toast.LENGTH_SHORT).show() - + lifecycleScope.launch { try { val bitmap = withContext(Dispatchers.Default) { generateShareImage(currentTask) } - + // Save bitmap to cache directory val file = withContext(Dispatchers.IO) { saveBitmapToCache(bitmap) } - + // Share the image shareImageFile(file) - + // Recycle bitmap bitmap.recycle() } catch (e: Exception) { @@ -255,7 +255,7 @@ class HistoryDetailActivity : AppCompatActivity() { } } } - + /** * Generates a share image from task history. * @@ -270,16 +270,16 @@ class HistoryDetailActivity : AppCompatActivity() { val contentWidth = width - padding * 2 val stepSpacing = 40 // Spacing between steps val screenshotWidthRatio = 0.8f // Screenshot width = 80% of image width - + // Calculate heights val headerHeight = 200 val stepBaseHeight = 120 val thinkingLineHeight = 50 - + // First pass: calculate total height (need to load screenshots to get their heights) var totalHeight = headerHeight + padding * 2 val screenshotHeights = mutableMapOf() - + for ((index, step) in task.steps.withIndex()) { totalHeight += stepBaseHeight if (step.thinking.isNotBlank()) { @@ -301,52 +301,52 @@ class HistoryDetailActivity : AppCompatActivity() { totalHeight += stepSpacing } totalHeight += 80 // footer - + // Create bitmap val bitmap = Bitmap.createBitmap(width, totalHeight, Bitmap.Config.ARGB_8888) val canvas = Canvas(bitmap) - + // Background canvas.drawColor(Color.parseColor("#1A1A1A")) - + // Paints val titlePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.WHITE textSize = 48f typeface = Typeface.DEFAULT_BOLD } - + val subtitlePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#AAAAAA") textSize = 32f } - + val stepNumberPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#4CAF50") textSize = 36f typeface = Typeface.DEFAULT_BOLD } - + val actionPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.WHITE textSize = 34f } - + val thinkingPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#888888") textSize = 28f } - + val cardPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.parseColor("#2A2A2A") } - + var y = padding.toFloat() - + // Draw header canvas.drawText("AutoGLM 任务记录", padding.toFloat(), y + 50, titlePaint) y += 70 - + // Task description val descLines = wrapText(task.taskDescription, actionPaint, contentWidth.toFloat()) for (line in descLines) { @@ -354,18 +354,18 @@ class HistoryDetailActivity : AppCompatActivity() { y += 45 } y += 20 - + // Status and info val statusColor = if (task.success) Color.parseColor("#4CAF50") else Color.parseColor("#F44336") val statusPaint = Paint(subtitlePaint).apply { color = statusColor } val statusText = if (task.success) "✓ 成功" else "✗ 失败" canvas.drawText(statusText, padding.toFloat(), y + 35, statusPaint) - + val duration = formatDuration(task.duration) val infoStr = "${dateFormat.format(Date(task.startTime))} · ${task.stepCount}步 · $duration" canvas.drawText(infoStr, padding + 150f, y + 35, subtitlePaint) y += 60 - + // Draw steps for ((index, step) in task.steps.withIndex()) { // Add spacing before each step (except first one uses smaller spacing) @@ -374,7 +374,7 @@ class HistoryDetailActivity : AppCompatActivity() { } else { y += stepSpacing } - + // Step card background val cardTop = y var cardHeight = stepBaseHeight.toFloat() @@ -385,32 +385,32 @@ class HistoryDetailActivity : AppCompatActivity() { screenshotHeights[index]?.let { h -> cardHeight += h + 40 } - + canvas.drawRoundRect( padding.toFloat(), cardTop, (width - padding).toFloat(), cardTop + cardHeight, 20f, 20f, cardPaint ) - + y += 15 - + // Step number canvas.drawText("步骤 ${step.stepNumber}", padding + 20f, y + 40, stepNumberPaint) - + // Status indicator val stepStatusPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = if (step.success) Color.parseColor("#4CAF50") else Color.parseColor("#F44336") } canvas.drawCircle(width - padding - 30f, y + 25, 10f, stepStatusPaint) y += 50 - + // Action description val actionLines = wrapText(step.actionDescription, actionPaint, contentWidth - 40f) for (line in actionLines) { canvas.drawText(line, padding + 20f, y + 30, actionPaint) y += 40 } - + // Thinking if (step.thinking.isNotBlank()) { y += 10 @@ -420,7 +420,7 @@ class HistoryDetailActivity : AppCompatActivity() { y += 35 } } - + // Screenshot val screenshotPath = step.annotatedScreenshotPath ?: step.screenshotPath if (screenshotPath != null) { @@ -432,29 +432,29 @@ class HistoryDetailActivity : AppCompatActivity() { val scale = targetWidth.toFloat() / screenshotBitmap.width val scaledWidth = targetWidth val scaledHeight = (screenshotBitmap.height * scale).toInt() - + val scaledBitmap = Bitmap.createScaledBitmap(screenshotBitmap, scaledWidth, scaledHeight, true) val left = (width - scaledWidth) / 2 // Center horizontally canvas.drawBitmap(scaledBitmap, left.toFloat(), y, null) - + y += scaledHeight scaledBitmap.recycle() screenshotBitmap.recycle() } } - + // Move y to end of card (cardTop + cardHeight) y = cardTop + cardHeight } - + // Footer y += 30 val footerPaint = Paint(subtitlePaint).apply { textSize = 24f } canvas.drawText("由 AutoGLM For Android 生成", padding.toFloat(), y + 30, footerPaint) - + return bitmap } - + /** * Wraps text to fit within a given width. * @@ -466,11 +466,11 @@ class HistoryDetailActivity : AppCompatActivity() { private fun wrapText(text: String, paint: Paint, maxWidth: Float): List { val lines = mutableListOf() var remaining = text - + while (remaining.isNotEmpty()) { val count = paint.breakText(remaining, true, maxWidth, null) if (count == 0) break - + // Try to break at word boundary var breakAt = count if (count < remaining.length) { @@ -479,14 +479,14 @@ class HistoryDetailActivity : AppCompatActivity() { breakAt = lastSpace + 1 } } - + lines.add(remaining.substring(0, breakAt).trim()) remaining = remaining.substring(breakAt).trim() } - + return lines.ifEmpty { listOf("") } } - + /** * Saves bitmap to cache directory. * @@ -496,14 +496,21 @@ class HistoryDetailActivity : AppCompatActivity() { private fun saveBitmapToCache(bitmap: Bitmap): File { val cacheDir = File(cacheDir, "share") if (!cacheDir.exists()) cacheDir.mkdirs() - + + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + val file = File(cacheDir, "autoglm_task_${System.currentTimeMillis()}.webp") FileOutputStream(file).use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) + bitmap.compress(compressFormat, 90, out) } return file } - + /** * Shares the image file using system share sheet. * @@ -515,16 +522,16 @@ class HistoryDetailActivity : AppCompatActivity() { "${packageName}.fileprovider", file ) - + val intent = Intent(Intent.ACTION_SEND).apply { type = "image/webp" putExtra(Intent.EXTRA_STREAM, uri) addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) } - + startActivity(Intent.createChooser(intent, getString(R.string.history_share_title))) } - + /** * Saves bitmap to device gallery. * @@ -535,7 +542,7 @@ class HistoryDetailActivity : AppCompatActivity() { */ private fun saveBitmapToGallery(bitmap: Bitmap): Boolean { val filename = "AutoGLM_${System.currentTimeMillis()}.webp" - + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { // Android 10+ use MediaStore val contentValues = ContentValues().apply { @@ -544,15 +551,21 @@ class HistoryDetailActivity : AppCompatActivity() { put(MediaStore.Images.Media.RELATIVE_PATH, Environment.DIRECTORY_PICTURES + "/AutoGLM") put(MediaStore.Images.Media.IS_PENDING, 1) } - + val resolver = contentResolver val uri = resolver.insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, contentValues) - + uri?.let { resolver.openOutputStream(it)?.use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, 90, out) } - + contentValues.clear() contentValues.put(MediaStore.Images.Media.IS_PENDING, 0) resolver.update(it, contentValues, null, null) @@ -564,24 +577,34 @@ class HistoryDetailActivity : AppCompatActivity() { val picturesDir = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES) val autoglmDir = File(picturesDir, "AutoGLM") if (!autoglmDir.exists()) autoglmDir.mkdirs() - + val file = File(autoglmDir, filename) FileOutputStream(file).use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 90, out) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, 90, out) } - - // Notify gallery - val intent = Intent(Intent.ACTION_MEDIA_SCANNER_SCAN_FILE) - intent.data = android.net.Uri.fromFile(file) - sendBroadcast(intent) - + + // Notify gallery using MediaScannerConnection (recommended approach) + android.media.MediaScannerConnection.scanFile( + this, + arrayOf(file.absolutePath), + arrayOf("image/webp") + ) { path, uri -> + Logger.d(TAG, "Scanned $path: $uri") + } + true } } - + companion object { private const val TAG = "HistoryDetailActivity" - + /** Intent extra key for task ID. */ const val EXTRA_TASK_ID = "task_id" } diff --git a/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt b/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt index 0ca4298..101bb53 100644 --- a/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/history/HistoryManager.kt @@ -3,6 +3,7 @@ package com.kevinluo.autoglm.history import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.os.Build import android.util.Base64 import com.kevinluo.autoglm.action.AgentAction import com.kevinluo.autoglm.util.Logger @@ -38,33 +39,33 @@ import java.util.Locale * */ class HistoryManager private constructor(private val context: Context) { - + /** Directory for storing task history files. */ private val historyDir: File by lazy { File(context.filesDir, HISTORY_DIR).also { it.mkdirs() } } - + /** Currently recording task, null if no task is being recorded. */ private var currentTask: TaskHistory? = null - + /** Base64-encoded screenshot data for the current step. */ private var currentScreenshotBase64: String? = null - + /** Width of the current screenshot in pixels. */ private var currentScreenshotWidth: Int = 0 - + /** Height of the current screenshot in pixels. */ private var currentScreenshotHeight: Int = 0 - + private val _historyList = MutableStateFlow>(emptyList()) - + /** Observable list of all task histories, sorted by most recent first. */ val historyList: StateFlow> = _historyList.asStateFlow() - + init { loadHistoryIndex() } - + /** * Starts recording a new task. * @@ -81,7 +82,7 @@ class HistoryManager private constructor(private val context: Context) { Logger.d(TAG, "Started recording task: ${task.id}") return task } - + /** * Sets the current screenshot for the next step. * @@ -123,19 +124,19 @@ class HistoryManager private constructor(private val context: Context) { message: String? = null ) = withContext(Dispatchers.IO) { val task = currentTask ?: return@withContext - + var screenshotPath: String? = null var annotatedPath: String? = null - + // Save screenshot if available currentScreenshotBase64?.let { base64 -> try { // Decode base64 to raw bytes (already WebP format) val webpBytes = Base64.decode(base64, Base64.DEFAULT) - + // Save original screenshot directly without re-compression screenshotPath = saveScreenshotBytes(task.id, stepNumber, webpBytes, false) - + // Create and save annotated screenshot if action has visual annotation if (action != null) { val annotation = ScreenshotAnnotator.createAnnotation( @@ -163,7 +164,7 @@ class HistoryManager private constructor(private val context: Context) { Logger.e(TAG, "Failed to save screenshot for step $stepNumber", e) } } - + val step = HistoryStep( stepNumber = stepNumber, thinking = thinking, @@ -174,14 +175,14 @@ class HistoryManager private constructor(private val context: Context) { success = success, message = message ) - + task.steps.add(step) Logger.d(TAG, "Recorded step $stepNumber for task ${task.id}") - + // Clear current screenshot currentScreenshotBase64 = null } - + /** * Completes the current task recording. * @@ -195,38 +196,38 @@ class HistoryManager private constructor(private val context: Context) { */ suspend fun completeTask(success: Boolean, message: String?) = withContext(Dispatchers.IO) { val task = currentTask ?: return@withContext - + // Don't save empty tasks (no steps recorded) if (task.steps.isEmpty()) { Logger.d(TAG, "Skipping empty task ${task.id}") currentTask = null return@withContext } - + task.endTime = System.currentTimeMillis() task.success = success task.completionMessage = message - + // Save task to disk saveTask(task) - + // Update history list val updatedList = _historyList.value.toMutableList() updatedList.add(0, task) - + // Trim old history if needed while (updatedList.size > MAX_HISTORY_COUNT) { val removed = updatedList.removeAt(updatedList.size - 1) deleteTaskFiles(removed.id) } - + _historyList.value = updatedList saveHistoryIndex() - + Logger.d(TAG, "Completed task ${task.id}, success=$success") currentTask = null } - + /** * Gets a task history by ID. * @@ -239,7 +240,7 @@ class HistoryManager private constructor(private val context: Context) { suspend fun getTask(taskId: String): TaskHistory? = withContext(Dispatchers.IO) { loadTask(taskId) } - + /** * Deletes a task history. * @@ -253,7 +254,7 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value = _historyList.value.filter { it.id != taskId } saveHistoryIndex() } - + /** * Deletes multiple task histories. * @@ -269,7 +270,7 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value = _historyList.value.filter { it.id !in taskIds } saveHistoryIndex() } - + /** * Clears all history. * @@ -281,7 +282,7 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value = emptyList() saveHistoryIndex() } - + /** * Gets the screenshot bitmap for a step. * @@ -297,9 +298,9 @@ class HistoryManager private constructor(private val context: Context) { if (!file.exists()) return null return BitmapFactory.decodeFile(path) } - + // Private helper methods - + /** * Saves raw WebP bytes directly to file (no re-compression). * @@ -318,14 +319,14 @@ class HistoryManager private constructor(private val context: Context) { val taskDir = File(historyDir, taskId).also { it.mkdirs() } val suffix = if (annotated) "_annotated" else "" val file = File(taskDir, "step_${stepNumber}${suffix}.webp") - + FileOutputStream(file).use { out -> out.write(webpBytes) } - + return file.absolutePath } - + /** * Saves bitmap as WebP (used for annotated screenshots). * @@ -344,14 +345,21 @@ class HistoryManager private constructor(private val context: Context) { val taskDir = File(historyDir, taskId).also { it.mkdirs() } val suffix = if (annotated) "_annotated" else "" val file = File(taskDir, "step_${stepNumber}${suffix}.webp") - + + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + FileOutputStream(file).use { out -> - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, 85, out) + bitmap.compress(compressFormat, 85, out) } - + return file.absolutePath } - + /** * Decodes a Base64 string to a Bitmap. * @@ -367,7 +375,7 @@ class HistoryManager private constructor(private val context: Context) { null } } - + /** * Saves a task's metadata to JSON file. * @@ -376,7 +384,7 @@ class HistoryManager private constructor(private val context: Context) { private fun saveTask(task: TaskHistory) { val taskDir = File(historyDir, task.id).also { it.mkdirs() } val metaFile = File(taskDir, "meta.json") - + val json = JSONObject().apply { put("id", task.id) put("taskDescription", task.taskDescription) @@ -384,7 +392,7 @@ class HistoryManager private constructor(private val context: Context) { put("endTime", task.endTime) put("success", task.success) put("completionMessage", task.completionMessage) - + val stepsArray = JSONArray() task.steps.forEach { step -> stepsArray.put(JSONObject().apply { @@ -400,10 +408,10 @@ class HistoryManager private constructor(private val context: Context) { } put("steps", stepsArray) } - + metaFile.writeText(json.toString(2)) } - + /** * Loads a task from its JSON metadata file. * @@ -413,11 +421,11 @@ class HistoryManager private constructor(private val context: Context) { private fun loadTask(taskId: String): TaskHistory? { val metaFile = File(historyDir, "$taskId/meta.json") if (!metaFile.exists()) return null - + return try { val json = JSONObject(metaFile.readText()) val steps = mutableListOf() - + val stepsArray = json.optJSONArray("steps") if (stepsArray != null) { for (i in 0 until stepsArray.length()) { @@ -436,7 +444,7 @@ class HistoryManager private constructor(private val context: Context) { )) } } - + TaskHistory( id = json.getString("id"), taskDescription = json.getString("taskDescription"), @@ -451,7 +459,7 @@ class HistoryManager private constructor(private val context: Context) { null } } - + /** * Deletes all files associated with a task. * @@ -460,7 +468,7 @@ class HistoryManager private constructor(private val context: Context) { private fun deleteTaskFiles(taskId: String) { File(historyDir, taskId).deleteRecursively() } - + /** * Loads the history index from persistent storage. * @@ -469,22 +477,22 @@ class HistoryManager private constructor(private val context: Context) { private fun loadHistoryIndex() { val indexFile = File(historyDir, INDEX_FILE) if (!indexFile.exists()) return - + try { val json = JSONArray(indexFile.readText()) val list = mutableListOf() - + for (i in 0 until json.length()) { val taskId = json.getString(i) loadTask(taskId)?.let { list.add(it) } } - + _historyList.value = list } catch (e: Exception) { Logger.e(TAG, "Failed to load history index", e) } } - + /** * Saves the history index to persistent storage. * @@ -496,16 +504,16 @@ class HistoryManager private constructor(private val context: Context) { _historyList.value.forEach { json.put(it.id) } indexFile.writeText(json.toString()) } - + companion object { private const val TAG = "HistoryManager" private const val HISTORY_DIR = "task_history" private const val INDEX_FILE = "history_index.json" private const val MAX_HISTORY_COUNT = 50 - + @Volatile private var instance: HistoryManager? = null - + /** * Gets the singleton instance of HistoryManager. * diff --git a/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt b/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt index 484a688..7dbf7ab 100644 --- a/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/input/AutoGLMKeyboardService.kt @@ -47,7 +47,7 @@ class AutoGLMKeyboardService : InputMethodService() { private val inputReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context, intent: Intent) { Logger.d(TAG, "Received broadcast: ${intent.action}") - + when (intent.action) { ACTION_INPUT_TEXT, ACTION_INPUT_B64 -> { handleInputText(intent) @@ -90,13 +90,13 @@ class AutoGLMKeyboardService : InputMethodService() { */ override fun onCreateInputView(): View { Logger.d(TAG, "onCreateInputView called") - + // Register receiver when input view is created registerInputReceiver() - + // Create a minimal status view val view = layoutInflater.inflate(R.layout.keyboard_autoglm, null) - + return view } @@ -122,7 +122,7 @@ class AutoGLMKeyboardService : InputMethodService() { override fun onStartInputView(info: EditorInfo?, restarting: Boolean) { super.onStartInputView(info, restarting) Logger.d(TAG, "onStartInputView: restarting=$restarting") - + // Ensure receiver is registered registerInputReceiver() } @@ -165,7 +165,7 @@ class AutoGLMKeyboardService : InputMethodService() { try { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - registerReceiver(inputReceiver, filter, Context.RECEIVER_EXPORTED) + registerReceiver(inputReceiver, filter, RECEIVER_EXPORTED) } else { @Suppress("UnspecifiedRegisterReceiverFlag") registerReceiver(inputReceiver, filter) @@ -211,7 +211,7 @@ class AutoGLMKeyboardService : InputMethodService() { val decodedBytes = Base64.decode(encodedText, Base64.DEFAULT) val text = String(decodedBytes, Charsets.UTF_8) Logger.d(TAG, "Decoded text: '${text.take(50)}${if (text.length > 50) "..." else ""}'") - + commitText(text) } catch (e: Exception) { Logger.e(TAG, "Failed to decode Base64 text", e) @@ -240,7 +240,7 @@ class AutoGLMKeyboardService : InputMethodService() { */ private fun handleClearText() { Logger.d(TAG, "Clearing text") - + val ic = currentInputConnection ?: run { Logger.w(TAG, "No input connection for clear text") return @@ -249,10 +249,10 @@ class AutoGLMKeyboardService : InputMethodService() { try { // Perform select all ic.performContextMenuAction(android.R.id.selectAll) - + // Delete selected text by committing empty string ic.commitText("", 0) - + Logger.d(TAG, "Text cleared successfully") } catch (e: Exception) { Logger.e(TAG, "Failed to clear text", e) diff --git a/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt b/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt index 9a754ee..816c20e 100644 --- a/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt +++ b/app/src/main/java/com/kevinluo/autoglm/input/KeyboardHelper.kt @@ -4,7 +4,9 @@ import android.content.Context import android.content.Intent import android.provider.Settings import android.view.inputmethod.InputMethodManager +import com.kevinluo.autoglm.BuildConfig import com.kevinluo.autoglm.util.Logger +import kotlin.coroutines.coroutineContext /** * Helper class for keyboard-related operations. @@ -16,17 +18,24 @@ import com.kevinluo.autoglm.util.Logger object KeyboardHelper { private const val TAG = "KeyboardHelper" - + /** AutoGLM package name. */ - private const val PACKAGE_NAME = "com.kevinluo.autoglm" - + private const val PACKAGE_NAME = BuildConfig.APPLICATION_ID + /** AutoGLM Keyboard IME ID (Android system format). */ - const val IME_ID = "$PACKAGE_NAME/.input.AutoGLMKeyboardService" - + + val IME_ID = "${BuildConfig.APPLICATION_ID}/${AutoGLMKeyboardService::class.java.name}" + + fun getImeId(context: Context): String { + val pkg = context.packageName + val service = AutoGLMKeyboardService::class.java.name + return "$pkg/$service" + } + /** * Checks if the given IME ID belongs to AutoGLM Keyboard. */ - fun isAutoGLMKeyboard(imeId: String): Boolean = + fun isAutoGLMKeyboard(imeId: String): Boolean = imeId.startsWith("$PACKAGE_NAME/") /** @@ -53,7 +62,7 @@ object KeyboardHelper { for (ime in enabledInputMethods) { Logger.d(TAG, "Found IME: package=${ime.packageName}, service=${ime.serviceName}") - if (ime.packageName == PACKAGE_NAME && + if (ime.packageName.contains(PACKAGE_NAME) && ime.serviceName.endsWith(".AutoGLMKeyboardService")) { Logger.d(TAG, "AutoGLM Keyboard is enabled") return KeyboardStatus.ENABLED diff --git a/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt b/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt index c9cf994..802ca83 100644 --- a/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/input/TextInputManager.kt @@ -1,6 +1,7 @@ package com.kevinluo.autoglm.input import android.util.Base64 +import com.kevinluo.autoglm.BuildConfig import com.kevinluo.autoglm.IUserService import com.kevinluo.autoglm.util.Logger import kotlinx.coroutines.Dispatchers @@ -36,7 +37,7 @@ class TextInputManager(private val userService: IUserService) { /** Cached original IME for restoration after text input. */ private var originalIme: String? = null - + /** * Types text into the currently focused input field. * @@ -112,11 +113,11 @@ class TextInputManager(private val userService: IUserService) { // Save original IME originalIme = currentIme Logger.d(TAG, "Saved original IME: $originalIme") - + // List all enabled IMEs to debug val enabledImes = shell("ime list -s") Logger.d(TAG, "Enabled IMEs:\n$enabledImes") - + // Get the IME ID val imeId = KeyboardHelper.IME_ID Logger.d(TAG, "AutoGLM Keyboard IME ID: $imeId") @@ -153,7 +154,7 @@ class TextInputManager(private val userService: IUserService) { val newIme = getCurrentIme() return KeyboardHelper.isAutoGLMKeyboard(newIme) } - + /** * Gets the current default input method. * @@ -173,7 +174,7 @@ class TextInputManager(private val userService: IUserService) { */ private fun clearText(): String { Logger.d(TAG, "Clearing text") - return shell("am broadcast -a $ACTION_CLEAR_TEXT -p $PACKAGE_NAME") + return shell("am broadcast -a $ACTION_CLEAR_TEXT -p $PACKAGE_NAME --user current") } /** @@ -192,7 +193,7 @@ class TextInputManager(private val userService: IUserService) { private fun inputTextViaB64(text: String): String { val encoded = Base64.encodeToString(text.toByteArray(Charsets.UTF_8), Base64.NO_WRAP) Logger.d(TAG, "Input text via B64: '$text' -> '$encoded'") - return shell("am broadcast -a $ACTION_INPUT_B64 -p $PACKAGE_NAME --es msg '$encoded'") + return shell("am broadcast -a $ACTION_INPUT_B64 -p $PACKAGE_NAME --es msg '$encoded' --user current") } /** @@ -232,9 +233,9 @@ class TextInputManager(private val userService: IUserService) { // Broadcast actions private const val ACTION_INPUT_B64 = "ADB_INPUT_B64" private const val ACTION_CLEAR_TEXT = "ADB_CLEAR_TEXT" - + // Package name - private const val PACKAGE_NAME = "com.kevinluo.autoglm" + private const val PACKAGE_NAME = BuildConfig.APPLICATION_ID // Timing constants (increased for stability) // Wait after switching keyboard to ensure it's fully active diff --git a/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt b/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt index 40f04f7..17b9285 100644 --- a/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/screenshot/ScreenshotService.kt @@ -2,6 +2,7 @@ package com.kevinluo.autoglm.screenshot import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.os.Build import android.util.Base64 import com.kevinluo.autoglm.IUserService import com.kevinluo.autoglm.util.ErrorHandler @@ -78,29 +79,39 @@ interface FloatingWindowController { * */ class ScreenshotService( - private val userService: IUserService, + userService: IUserService? = null, + private val screenshotProvider: (suspend () -> Screenshot)? = null, private val floatingWindowControllerProvider: () -> FloatingWindowController? = { null } ) { - + // Store userService for internal use (only used for Shizuku mode via shell commands) + private val userService: IUserService? = userService + + init { + // At least one capture method must be available + require(userService != null || screenshotProvider != null) { + "Either userService or screenshotProvider must be provided" + } + } + companion object { private const val TAG = "ScreenshotService" private const val HIDE_DELAY_MS = 200L private const val SHOW_DELAY_MS = 100L private const val FALLBACK_WIDTH = 1080 private const val FALLBACK_HEIGHT = 1920 - + // Screenshot compression settings - optimized for API upload private const val WEBP_QUALITY = 65 // Reduced from 70 for better compression - + // Screenshot scaling settings - use max dimensions instead of fixed scale factor // This ensures consistent output size regardless of device resolution private const val MAX_WIDTH = 720 // Max width after scaling private const val MAX_HEIGHT = 1280 // Max height after scaling - + // Base64 output chunk size for reading (safe for Binder) private const val BASE64_CHUNK_SIZE = 500000 } - + /** * Captures the current screen content. * @@ -115,7 +126,7 @@ class ScreenshotService( val floatingWindowController = floatingWindowControllerProvider() val hasFloatingWindow = floatingWindowController != null Logger.d(TAG, "Starting screenshot capture, window visible: ${floatingWindowController?.isVisible()}") - + // Hide floating window before capture if (hasFloatingWindow) { Logger.d(TAG, "Hiding floating window") @@ -124,7 +135,7 @@ class ScreenshotService( } delay(HIDE_DELAY_MS) } - + try { // Capture screenshot val result = captureScreen() @@ -145,7 +156,7 @@ class ScreenshotService( } } } - + /** * Captures the screen using Shizuku shell command. * @@ -156,31 +167,43 @@ class ScreenshotService( * */ private suspend fun captureScreen(): Screenshot = withContext(Dispatchers.IO) { + // Use screenshotProvider if available (Accessibility mode) + if (screenshotProvider != null) { + Logger.d(TAG, "Using screenshot provider (Accessibility mode)") + return@withContext try { + screenshotProvider.invoke() + } catch (e: Exception) { + Logger.e(TAG, "Screenshot provider failed, using fallback", e) + createFallbackScreenshot() + } + } + + // Otherwise use shell command method (Shizuku mode) try { - Logger.d(TAG, "Executing screencap command") - + Logger.d(TAG, "Executing screencap command (Shizuku mode)") + val pngData = executeScreencapToBytes() - + if (pngData == null || pngData.isEmpty()) { Logger.w(TAG, "Failed to capture screenshot, returning fallback") return@withContext createFallbackScreenshot() } - + Logger.d(TAG, "PNG data captured: ${pngData.size} bytes") - + // Decode PNG to bitmap var bitmap = BitmapFactory.decodeByteArray(pngData, 0, pngData.size) if (bitmap == null) { Logger.w(TAG, "Failed to decode PNG, returning fallback") return@withContext createFallbackScreenshot() } - + val originalWidth = bitmap.width val originalHeight = bitmap.height - + // Calculate scaled dimensions based on max size constraints val (scaledWidth, scaledHeight) = calculateOptimalDimensions(originalWidth, originalHeight) - + // Scale bitmap if needed if (scaledWidth != originalWidth || scaledHeight != originalHeight) { val scaledBitmap = Bitmap.createScaledBitmap(bitmap, scaledWidth, scaledHeight, true) @@ -188,19 +211,25 @@ class ScreenshotService( bitmap = scaledBitmap Logger.d(TAG, "Scaled from ${originalWidth}x${originalHeight} to ${scaledWidth}x${scaledHeight}") } - + // Convert to WebP for better compression val webpStream = ByteArrayOutputStream() - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, WEBP_QUALITY, webpStream) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, WEBP_QUALITY, webpStream) bitmap.recycle() - + val webpData = webpStream.toByteArray() val compressionRatio = if (pngData.isNotEmpty()) 100 * webpData.size / pngData.size else 0 Logger.d(TAG, "Converted to WebP: ${webpData.size} bytes ($compressionRatio% of PNG)") - + val base64Data = encodeToBase64(webpData) Logger.d(TAG, "Screenshot captured: ${scaledWidth}x${scaledHeight}, base64 length: ${base64Data.length}") - + Screenshot( base64Data = base64Data, width = scaledWidth, @@ -214,7 +243,7 @@ class ScreenshotService( createFallbackScreenshot() } } - + /** * Calculates optimal dimensions based on max size constraints. * @@ -232,21 +261,21 @@ class ScreenshotService( Logger.d(TAG, "Image already within limits: ${originalWidth}x${originalHeight}") return originalWidth to originalHeight } - + // Calculate scale ratios for both dimensions val widthRatio = MAX_WIDTH.toFloat() / originalWidth val heightRatio = MAX_HEIGHT.toFloat() / originalHeight - + // Use the smaller ratio to ensure both dimensions fit within limits val ratio = minOf(widthRatio, heightRatio) - + val scaledWidth = (originalWidth * ratio).toInt() val scaledHeight = (originalHeight * ratio).toInt() - + Logger.d(TAG, "Scaling with ratio $ratio: ${originalWidth}x${originalHeight} -> ${scaledWidth}x${scaledHeight}") return scaledWidth to scaledHeight } - + /** * Executes screencap command and returns raw bytes. * @@ -257,106 +286,64 @@ class ScreenshotService( * */ private suspend fun executeScreencapToBytes(): ByteArray? = coroutineScope { - val timestamp = System.currentTimeMillis() - val pngFile = "/data/local/tmp/screenshot_$timestamp.png" - val base64File = "$pngFile.b64" - + val service = userService ?: run { + Logger.w(TAG, "UserService not available for shell command screenshot") + return@coroutineScope null + } + try { - Logger.d(TAG, "Attempting screenshot capture") + Logger.d(TAG, "Attempting screenshot capture (streamed, no temp files)") val startTime = System.currentTimeMillis() - - // Capture screenshot and pipe to base64 - val captureResult = userService.executeCommand( - "screencap -p | base64 > $base64File && stat -c %s $base64File" - ) - - val captureTime = System.currentTimeMillis() - startTime - Logger.d(TAG, "Screenshot capture took ${captureTime}ms") - - // Check for errors - if (captureResult.contains("Error") || captureResult.contains("permission denied", ignoreCase = true)) { - Logger.w(TAG, "Screenshot capture failed: $captureResult") - return@coroutineScope null - } - - // Parse base64 file size from output - val base64Size = captureResult.lines() - .firstOrNull { it.trim().all { c -> c.isDigit() } } - ?.trim()?.toLongOrNull() ?: 0L - - if (base64Size == 0L) { - Logger.w(TAG, "Base64 file not created or empty") + + // Trigger capture in the service and get total base64 size + val totalSize = service.captureScreenshotAndGetSize() + if (totalSize <= 0) { + Logger.w(TAG, "Screenshot capture failed or returned empty data") return@coroutineScope null } - - Logger.d(TAG, "Base64 file size: $base64Size bytes") - - // Read base64 file + val readStartTime = System.currentTimeMillis() - val base64Data: String - - if (base64Size <= BASE64_CHUNK_SIZE) { - // Small file - read in one go - val result = userService.executeCommand("cat $base64File") - base64Data = result.lines() - .filter { line -> - !line.startsWith("[") && - line.isNotBlank() && - !line.contains("exit code", ignoreCase = true) - } - .joinToString("") - } else { - // Large file - read chunks sequentially to avoid Binder buffer overflow - val chunkCount = ((base64Size + BASE64_CHUNK_SIZE - 1) / BASE64_CHUNK_SIZE).toInt() - Logger.d(TAG, "Reading $chunkCount chunks sequentially") - - val chunks = mutableListOf() - - for (index in 0 until chunkCount) { - val offset = index.toLong() * BASE64_CHUNK_SIZE - val remaining = base64Size - offset - val currentChunkSize = minOf(BASE64_CHUNK_SIZE.toLong(), remaining) - - val chunkResult = userService.executeCommand( - "tail -c +${offset + 1} $base64File | head -c $currentChunkSize" - ) - - val chunkData = chunkResult.lines() - .filter { line -> - !line.startsWith("[") && - line.isNotBlank() && - !line.contains("exit code", ignoreCase = true) - } - .joinToString("") - - chunks.add(chunkData) + val chunkCount = ((totalSize + BASE64_CHUNK_SIZE - 1) / BASE64_CHUNK_SIZE) + Logger.d(TAG, "Reading $chunkCount chunks sequentially, total size: $totalSize") + + val chunks = StringBuilder(totalSize) + var offset = 0 + for (i in 0 until chunkCount) { + val currentChunkSize = kotlin.math.min(BASE64_CHUNK_SIZE, totalSize - offset) + val chunk = service.readScreenshotChunk(offset, currentChunkSize) + if (chunk.isEmpty()) { + Logger.w(TAG, "Empty chunk at index $i, offset=$offset size=$currentChunkSize") + // Bail out and return null to fallback + return@coroutineScope null } - - base64Data = chunks.joinToString("") + chunks.append(chunk) + offset += currentChunkSize } - + + val base64Data = chunks.toString() val readTime = System.currentTimeMillis() - readStartTime - Logger.d(TAG, "Base64 read took ${readTime}ms, total length: ${base64Data.length}") - + val captureTime = System.currentTimeMillis() - startTime + Logger.d(TAG, "Base64 read took ${readTime}ms, capture total ${captureTime}ms, length=${base64Data.length}") + if (base64Data.isBlank()) { Logger.w(TAG, "No base64 data read") return@coroutineScope null } - + Base64.decode(base64Data, Base64.DEFAULT) } catch (e: Exception) { - Logger.e(TAG, "Failed to capture screenshot to bytes", e) + Logger.e(TAG, "Failed to capture screenshot to bytes (stream)", e) null } finally { - // Clean up temp files + // Always clear internal buffer after reading try { - userService.executeCommand("rm -f $pngFile $base64File") + service.clearScreenshotBuffer() } catch (_: Exception) { // Ignore cleanup errors } } } - + /** * Creates a fallback black screenshot in WebP format. * @@ -368,13 +355,19 @@ class ScreenshotService( */ private fun createFallbackScreenshot(): Screenshot { val bitmap = Bitmap.createBitmap(FALLBACK_WIDTH, FALLBACK_HEIGHT, Bitmap.Config.ARGB_8888) - + val outputStream = ByteArrayOutputStream() - bitmap.compress(Bitmap.CompressFormat.WEBP_LOSSY, WEBP_QUALITY, outputStream) + val compressFormat = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } + bitmap.compress(compressFormat, WEBP_QUALITY, outputStream) bitmap.recycle() - + val base64Data = Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP) - + return Screenshot( base64Data = base64Data, width = FALLBACK_WIDTH, @@ -382,7 +375,7 @@ class ScreenshotService( isSensitive = true ) } - + /** * Encodes byte array to base64 string. * @@ -393,7 +386,7 @@ class ScreenshotService( fun encodeToBase64(data: ByteArray): String { return Base64.encodeToString(data, Base64.NO_WRAP) } - + /** * Decodes base64 string to byte array. * @@ -404,7 +397,7 @@ class ScreenshotService( fun decodeFromBase64(base64Data: String): ByteArray { return Base64.decode(base64Data, Base64.DEFAULT) } - + /** * Decodes base64 screenshot data to a Bitmap. * @@ -420,7 +413,7 @@ class ScreenshotService( null } } - + /** * Encodes a Bitmap to base64 string. * @@ -432,14 +425,20 @@ class ScreenshotService( */ fun encodeBitmapToBase64( bitmap: Bitmap, - format: Bitmap.CompressFormat = Bitmap.CompressFormat.WEBP_LOSSY, + format: Bitmap.CompressFormat? = null, quality: Int = WEBP_QUALITY ): String { + val compressFormat = format ?: if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Bitmap.CompressFormat.WEBP_LOSSY + } else { + @Suppress("DEPRECATION") + Bitmap.CompressFormat.WEBP + } val outputStream = ByteArrayOutputStream() - bitmap.compress(format, quality, outputStream) + bitmap.compress(compressFormat, quality, outputStream) return encodeToBase64(outputStream.toByteArray()) } - + /** * Creates a Screenshot object from a Bitmap. * diff --git a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt index 1f14a08..7d171ef 100644 --- a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt +++ b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsActivity.kt @@ -22,6 +22,7 @@ import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView import com.kevinluo.autoglm.R import com.kevinluo.autoglm.agent.AgentConfig +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.model.ModelClient import com.kevinluo.autoglm.model.ModelConfig import com.kevinluo.autoglm.util.LogFileManager @@ -41,7 +42,12 @@ import kotlinx.coroutines.launch class SettingsActivity : AppCompatActivity() { private lateinit var settingsManager: SettingsManager - + + // Device control mode views + private lateinit var controlModeRadioGroup: RadioGroup + private lateinit var controlModeShizuku: RadioButton + private lateinit var controlModeAccessibility: RadioButton + // Profile selector views private lateinit var profileSelectorLayout: TextInputLayout private lateinit var profileSelector: AutoCompleteTextView @@ -115,6 +121,11 @@ class SettingsActivity : AppCompatActivity() { * Initializes all view references. */ private fun initViews() { + // Device control mode + controlModeRadioGroup = findViewById(R.id.controlModeRadioGroup) + controlModeShizuku = findViewById(R.id.controlModeShizuku) + controlModeAccessibility = findViewById(R.id.controlModeAccessibility) + // Profile selector profileSelectorLayout = findViewById(R.id.profileSelectorLayout) profileSelector = findViewById(R.id.profileSelector) @@ -177,7 +188,14 @@ class SettingsActivity : AppCompatActivity() { Logger.d(TAG, "Loading current settings") val modelConfig = settingsManager.getModelConfig() val agentConfig = settingsManager.getAgentConfig() - + val controlMode = settingsManager.getDeviceControlMode() + + // Load device control mode + when (controlMode) { + DeviceControlMode.SHIZUKU -> controlModeShizuku.isChecked = true + DeviceControlMode.ACCESSIBILITY -> controlModeAccessibility.isChecked = true + } + // Load saved profiles loadSavedProfiles() @@ -316,6 +334,16 @@ class SettingsActivity : AppCompatActivity() { modelNameInput.setOnFocusChangeListener { _, _ -> modelNameLayout.error = null } maxStepsInput.setOnFocusChangeListener { _, _ -> maxStepsLayout.error = null } screenshotDelayInput.setOnFocusChangeListener { _, _ -> screenshotDelayLayout.error = null } + + // Device control mode change listener + controlModeRadioGroup.setOnCheckedChangeListener { _, checkedId -> + val newMode = when (checkedId) { + R.id.controlModeShizuku -> DeviceControlMode.SHIZUKU + R.id.controlModeAccessibility -> DeviceControlMode.ACCESSIBILITY + else -> return@setOnCheckedChangeListener + } + Logger.d(TAG, "Device control mode changed to: ${newMode.name}") + } } /** @@ -585,14 +613,22 @@ class SettingsActivity : AppCompatActivity() { Logger.i(TAG, "Saving settings") val baseUrl = baseUrlInput.text?.toString()?.trim() ?: "" val modelName = modelNameInput.text?.toString()?.trim() ?: "" - val apiKey = apiKeyInput.text?.toString()?.trim().let { - if (it.isNullOrEmpty()) "EMPTY" else it + val apiKey = apiKeyInput.text?.toString()?.trim().let { + if (it.isNullOrEmpty()) "EMPTY" else it } val maxSteps = maxStepsInput.text?.toString()?.trim()?.toIntOrNull() ?: 100 val screenshotDelaySeconds = screenshotDelayInput.text?.toString()?.trim()?.toDoubleOrNull() ?: 2.0 val screenshotDelayMs = (screenshotDelaySeconds * 1000).toLong() val language = if (languageEnglish.isChecked) "en" else "cn" - + + // Save device control mode + val controlMode = when (controlModeRadioGroup.checkedRadioButtonId) { + R.id.controlModeShizuku -> DeviceControlMode.SHIZUKU + R.id.controlModeAccessibility -> DeviceControlMode.ACCESSIBILITY + else -> DeviceControlMode.SHIZUKU + } + settingsManager.saveDeviceControlMode(controlMode) + // Create and save model config val modelConfig = ModelConfig( baseUrl = baseUrl, @@ -600,7 +636,7 @@ class SettingsActivity : AppCompatActivity() { modelName = modelName ) settingsManager.saveModelConfig(modelConfig) - + // Create and save agent config val agentConfig = AgentConfig( maxSteps = maxSteps, @@ -608,7 +644,7 @@ class SettingsActivity : AppCompatActivity() { screenshotDelayMs = screenshotDelayMs ) settingsManager.saveAgentConfig(agentConfig) - + Toast.makeText(this, R.string.settings_saved, Toast.LENGTH_SHORT).show() finish() } diff --git a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt index 9c8dc5e..545ffd0 100644 --- a/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt +++ b/app/src/main/java/com/kevinluo/autoglm/settings/SettingsManager.kt @@ -6,6 +6,7 @@ import android.os.Build import androidx.security.crypto.EncryptedSharedPreferences import androidx.security.crypto.MasterKey import com.kevinluo.autoglm.agent.AgentConfig +import com.kevinluo.autoglm.device.DeviceControlMode import com.kevinluo.autoglm.model.ModelConfig import com.kevinluo.autoglm.util.Logger import org.json.JSONArray @@ -95,7 +96,10 @@ class SettingsManager(private val context: Context) { // Dev profiles import key private const val KEY_DEV_PROFILES_IMPORTED = "dev_profiles_imported" - + + // Device control mode key + private const val KEY_DEVICE_CONTROL_MODE = "device_control_mode" + // Default values private val DEFAULT_MODEL_CONFIG = ModelConfig() private val DEFAULT_AGENT_CONFIG = AgentConfig() @@ -104,6 +108,7 @@ class SettingsManager(private val context: Context) { // Cache for detecting config changes private var lastModelConfig: ModelConfig? = null private var lastAgentConfig: AgentConfig? = null + private var lastDeviceControlMode: DeviceControlMode? = null // Regular preferences for non-sensitive data private val prefs: SharedPreferences by lazy { @@ -285,13 +290,17 @@ class SettingsManager(private val context: Context) { fun hasConfigChanged(): Boolean { val currentModelConfig = getModelConfig() val currentAgentConfig = getAgentConfig() - - val changed = lastModelConfig != currentModelConfig || lastAgentConfig != currentAgentConfig - + val currentDeviceControlMode = getDeviceControlMode() + + val changed = lastModelConfig != currentModelConfig || + lastAgentConfig != currentAgentConfig || + lastDeviceControlMode != currentDeviceControlMode + // Update cache lastModelConfig = currentModelConfig lastAgentConfig = currentAgentConfig - + lastDeviceControlMode = currentDeviceControlMode + return changed } @@ -689,4 +698,42 @@ class SettingsManager(private val context: Context) { -1 } } + + // ==================== Device Control Mode ==================== + + /** + * Gets the current device control mode. + * + * Returns SHIZUKU as default if not previously set. + * + * @return The current device control mode + */ + fun getDeviceControlMode(): DeviceControlMode { + val modeName = prefs.getString(KEY_DEVICE_CONTROL_MODE, DeviceControlMode.SHIZUKU.name) + return try { + DeviceControlMode.valueOf(modeName ?: DeviceControlMode.SHIZUKU.name) + } catch (e: Exception) { + Logger.w(TAG, "Invalid device control mode: $modeName, using SHIZUKU") + DeviceControlMode.SHIZUKU + } + } + + /** + * Saves the device control mode. + * + * @param mode The device control mode to save + */ + fun saveDeviceControlMode(mode: DeviceControlMode) { + Logger.d(TAG, "Saving device control mode: ${mode.name}") + prefs.edit().putString(KEY_DEVICE_CONTROL_MODE, mode.name).apply() + } + + /** + * Gets the display name of the current device control mode. + * + * @return The display name (e.g., "Shizuku" or "无障碍服务") + */ + fun getDeviceControlModeDisplayName(): String { + return getDeviceControlMode().displayName + } } diff --git a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt index 2f1fd1d..c236879 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowService.kt @@ -775,7 +775,9 @@ class FloatingWindowService : Service(), FloatingWindowController { gravity = Gravity.TOP or Gravity.START x = 0 y = 0 - // Allow keyboard input + // Allow keyboard input - Note: SOFT_INPUT_ADJUST_RESIZE is deprecated but still + // required for floating windows to resize when keyboard is shown + @Suppress("DEPRECATION") softInputMode = WindowManager.LayoutParams.SOFT_INPUT_ADJUST_RESIZE } } diff --git a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt index 8cf79ec..1521690 100644 --- a/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt +++ b/app/src/main/java/com/kevinluo/autoglm/ui/FloatingWindowTileService.kt @@ -47,7 +47,7 @@ class FloatingWindowTileService : TileService() { } startActivityAndCollapseCompat(intent) } - + /** * Compatibility wrapper for startActivityAndCollapse. * API 34+ requires PendingIntent, older versions use Intent directly. @@ -64,7 +64,7 @@ class FloatingWindowTileService : TileService() { startActivityAndCollapse(pendingIntent) } else { // API < 34 - @Suppress("DEPRECATION") + @Suppress("StartActivityAndCollapseDeprecated") startActivityAndCollapse(intent) } } @@ -88,14 +88,14 @@ class FloatingWindowTileService : TileService() { private fun updateTileState() { val tile = qsTile ?: return - + val service = FloatingWindowService.getInstance() val isVisible = service?.isVisible() == true - + tile.state = if (isVisible) Tile.STATE_ACTIVE else Tile.STATE_INACTIVE tile.label = getString(com.kevinluo.autoglm.R.string.tile_floating_window) tile.contentDescription = getString(com.kevinluo.autoglm.R.string.tile_floating_window_desc) - + tile.updateTile() } diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index ec9eaff..6ffee13 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -59,14 +59,20 @@ android:paddingHorizontal="16dp" android:paddingBottom="32dp"> - - + + android:layout_marginBottom="12dp"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AutoGLM For Android - + Shizuku 状态 未知 @@ -10,26 +10,26 @@ 已连接 授权 打开 Shizuku - + 悬浮窗权限 已授权 未授权 授权 - + 输入法 已启用 未启用 去启用 需要启用 AutoGLM Keyboard 才能输入文字 - + 任务输入 描述你想要执行的任务... 启动 停止 - + 状态 空闲 @@ -42,12 +42,12 @@ 步骤: %d 暂停 继续 - + 执行日志 查看任务执行的详细日志 日志将显示在这里... - + 请先启动 Shizuku Shizuku 版本过低 @@ -61,7 +61,8 @@ 任务已开始 任务已取消 需要悬浮窗权限 - + 后台服务需要通知权限 + 设置 模型配置 @@ -108,7 +109,7 @@ 连接失败: %s 连接超时: %s 请先填写完整的配置信息 - + 任务模板 保存常用任务，快速填入 @@ -124,7 +125,7 @@ 确定要删除此模板吗？ 暂无模板 选择模板 - + 高级设置 自定义系统提示词 @@ -139,7 +140,7 @@ 已重置为默认提示词 已自定义 默认 - + 确认操作 确定要执行此操作吗？ @@ -147,13 +148,13 @@ 取消 选择选项 请选择以下选项之一 - + 需要手动操作 请完成以下操作 操作说明将显示在这里 已完成，继续 - + 思考 操作 @@ -164,15 +165,15 @@ 等待确认 打开悬浮窗 新任务 - + 悬浮窗 历史记录 - + AutoGLM 显示/隐藏悬浮窗 - + 历史记录 任务详情 @@ -204,7 +205,7 @@ 提示词已复制到剪贴板 加载更多 加载更多 (剩余 %d 步) - + 调试日志 导出日志用于问题排查 @@ -215,11 +216,28 @@ 导出失败 暂无日志 确定要清空所有日志吗？ - + AutoGLM Keyboard 就绪 文本输入由 AutoGLM 控制 AutoGLM 键盘 已启用 + + + AutoGLM 无障碍服务，用于辅助残障人士进行设备自动化控制。 + 无障碍权限 + 已开启 + 未开启 + 去开启 + 无障碍模式不支持 Android 13 以下版本 + + + 设备控制模式 + 选择控制设备的方式 + Shizuku（推荐） + 无障碍服务 + 需要安装 Shizuku 应用，功能完整，兼容性好 + 无需安装额外应用，需要 Android 13+ + 控制模式已更改，请重启应用以生效 diff --git a/app/src/main/res/xml/accessibility_service_config.xml b/app/src/main/res/xml/accessibility_service_config.xml new file mode 100644 index 0000000..8e4efdb --- /dev/null +++ b/app/src/main/res/xml/accessibility_service_config.xml @@ -0,0 +1,26 @@ + + + + diff --git a/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt b/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt index cb1e11a..f55e6b0 100644 --- a/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt +++ b/app/src/test/java/com/kevinluo/autoglm/app/AppResolverPropertyTest.kt @@ -7,42 +7,42 @@ import io.kotest.property.forAll /** * Property-based tests for AppResolver. - * + * * **Feature: autoglm-phone-agent, Property 20: App name resolution consistency** * **Validates: Requirements 9.1** - * + * * These tests verify the core similarity calculation and matching logic * of the AppResolver without requiring Android framework dependencies. */ class AppResolverPropertyTest : StringSpec({ - + /** * Property 20: App name resolution consistency - * + * * For any installed app with a known display name, querying the AppResolver * with that exact name should return the correct package name. - * + * * Since we can't use PackageManager in unit tests, we test the underlying * similarity calculation logic which is the core of the resolution algorithm. - * + * * **Validates: Requirements 9.1** */ - + // Test that exact matches always return similarity of 1.0 "exact match should always return similarity of 1.0" { val stringArb = Arb.string(1..50, Codepoint.alphanumeric()) - + forAll(100, stringArb) { name -> val resolver = createTestableAppResolver() val similarity = resolver.calculateSimilarity(name.lowercase(), name.lowercase()) similarity == 1.0 } } - + // Test that similarity is symmetric for Levenshtein distance "levenshtein distance should be symmetric" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb) { s1, s2 -> val resolver = createTestableAppResolver() val dist1 = resolver.levenshteinDistance(s1, s2) @@ -50,33 +50,33 @@ class AppResolverPropertyTest : StringSpec({ dist1 == dist2 } } - + // Test that levenshtein distance is non-negative "levenshtein distance should be non-negative" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb) { s1, s2 -> val resolver = createTestableAppResolver() val distance = resolver.levenshteinDistance(s1, s2) distance >= 0 } } - + // Test that levenshtein distance is zero only for identical strings "levenshtein distance should be zero only for identical strings" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb) { s -> val resolver = createTestableAppResolver() val distance = resolver.levenshteinDistance(s, s) distance == 0 } } - + // Test triangle inequality for levenshtein distance "levenshtein distance should satisfy triangle inequality" { val stringArb = Arb.string(0..20, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb, stringArb) { s1, s2, s3 -> val resolver = createTestableAppResolver() val d12 = resolver.levenshteinDistance(s1, s2) @@ -85,46 +85,49 @@ class AppResolverPropertyTest : StringSpec({ d13 <= d12 + d23 } } - + // Test that similarity is always in range [0, 1] "similarity should always be in range 0 to 1" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb, stringArb) { query, target -> val resolver = createTestableAppResolver() val similarity = resolver.calculateSimilarity(query.lowercase(), target.lowercase()) similarity >= 0.0 && similarity <= 1.0 } } - + // Test that contains match has higher similarity than fuzzy match "contains match should have higher similarity than non-contains fuzzy match" { val prefixArb = Arb.string(1..10, Codepoint.alphanumeric()) val suffixArb = Arb.string(1..10, Codepoint.alphanumeric()) val queryArb = Arb.string(3..15, Codepoint.alphanumeric()) - + forAll(100, prefixArb, queryArb, suffixArb) { prefix, query, suffix -> val resolver = createTestableAppResolver() val normalizedQuery = query.lowercase() - + // Target that contains the query val containsTarget = (prefix + query + suffix).lowercase() - + // Target that doesn't contain the query (completely different) val differentTarget = "zzz${prefix}zzz".lowercase() - + + // Skip valid match cases in the "different" target + if (differentTarget.contains(normalizedQuery)) return@forAll true + val containsSimilarity = resolver.calculateSimilarity(normalizedQuery, containsTarget) val differentSimilarity = resolver.calculateSimilarity(normalizedQuery, differentTarget) - + // Contains match should score higher (unless different target happens to be similar) containsSimilarity >= differentSimilarity || differentSimilarity < AppResolver.MIN_SIMILARITY_THRESHOLD } } - + // Test that levenshtein distance to empty string equals string length "levenshtein distance to empty string should equal string length" { val stringArb = Arb.string(0..30, Codepoint.alphanumeric()) - + forAll(100, stringArb) { s -> val resolver = createTestableAppResolver() val distanceToEmpty = resolver.levenshteinDistance(s, "") @@ -132,19 +135,19 @@ class AppResolverPropertyTest : StringSpec({ distanceToEmpty == s.length && distanceFromEmpty == s.length } } - + // Test that startsWith match has high similarity "startsWith match should have high similarity" { val queryArb = Arb.string(3..15, Codepoint.alphanumeric()) val suffixArb = Arb.string(1..10, Codepoint.alphanumeric()) - + forAll(100, queryArb, suffixArb) { query, suffix -> val resolver = createTestableAppResolver() val normalizedQuery = query.lowercase() val target = (query + suffix).lowercase() - + val similarity = resolver.calculateSimilarity(normalizedQuery, target) - + // StartsWith should have similarity >= 0.75 similarity >= 0.75 } @@ -153,7 +156,7 @@ class AppResolverPropertyTest : StringSpec({ /** * Creates a testable AppResolver instance. - * + * * Since AppResolver requires PackageManager which is an Android system service, * we create a minimal mock that allows us to test the core similarity logic. */ @@ -164,12 +167,12 @@ private fun createTestableAppResolver(): TestableAppResolver { /** * A testable version of AppResolver that exposes the internal similarity * calculation methods for property-based testing. - * + * * This class duplicates the core logic from AppResolver to enable testing * without Android framework dependencies. */ class TestableAppResolver { - + /** * Calculates the similarity between two strings using a combination of: * 1. Exact match (highest priority) @@ -182,57 +185,57 @@ class TestableAppResolver { if (query == target) { return 1.0 } - + // Target contains query exactly if (target.contains(query)) { val coverageScore = query.length.toDouble() / target.length return 0.8 + (coverageScore * 0.15) } - + // Target starts with query if (target.startsWith(query)) { val coverageScore = query.length.toDouble() / target.length return 0.75 + (coverageScore * 0.15) } - + // Query starts with target if (query.startsWith(target)) { val coverageScore = target.length.toDouble() / query.length return 0.7 + (coverageScore * 0.15) } - + // Fuzzy matching using Levenshtein distance val distance = levenshteinDistance(query, target) val maxLength = maxOf(query.length, target.length) - + if (maxLength == 0) { return 0.0 } - + val similarity = 1.0 - (distance.toDouble() / maxLength) return similarity * 0.7 } - + /** * Calculates the Levenshtein distance between two strings. */ fun levenshteinDistance(s1: String, s2: String): Int { val m = s1.length val n = s2.length - + if (m == 0) return n if (n == 0) return m - + val dp = Array(m + 1) { IntArray(n + 1) } - + for (i in 0..m) { dp[i][0] = i } - + for (j in 0..n) { dp[0][j] = j } - + for (i in 1..m) { for (j in 1..n) { val cost = if (s1[i - 1] == s2[j - 1]) 0 else 1 @@ -243,10 +246,10 @@ class TestableAppResolver { ) } } - + return dp[m][n] } - + companion object { const val MIN_SIMILARITY_THRESHOLD = 0.3 }