初始化 AutoAgent 代码与文档

2026-09-04 15:42:16 +08:00
commit 6458419730
50 changed files with 2747 additions and 0 deletions
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <uses-permission android:name="android.permission.INTERNET" />
+
+    <application
+        android:allowBackup="false"
+        android:label="@string/app_name"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.AutoAgent">
+        <activity
+            android:name=".app.MainActivity"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+        <service
+            android:name=".core.accessibility.AutoAccessibilityService"
+            android:exported="true"
+            android:label="@string/accessibility_service_name"
+            android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
+            <intent-filter>
+                <action android:name="android.accessibilityservice.AccessibilityService" />
+            </intent-filter>
+            <meta-data
+                android:name="android.accessibilityservice"
+                android:resource="@xml/accessibility_service_config" />
+        </service>
+    </application>
+</manifest>
@@ -0,0 +1,37 @@
+package cn.auto.agent.app
+
+import android.content.Intent
+import android.os.Bundle
+import android.provider.Settings
+import android.widget.Button
+import android.widget.LinearLayout
+import android.widget.TextView
+import androidx.appcompat.app.AppCompatActivity
+import cn.auto.agent.R
+import cn.auto.agent.core.accessibility.AutoAccessibilityService
+
+class MainActivity : AppCompatActivity() {
+    private lateinit var state: TextView
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        title = getString(R.string.app_name)
+        state = TextView(this).apply { textSize = 18f }
+        val button = Button(this).apply {
+            text = "打开无障碍设置"
+            setOnClickListener { startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)) }
+        }
+        setContentView(LinearLayout(this).apply {
+            orientation = LinearLayout.VERTICAL
+            val padding = (24 * resources.displayMetrics.density).toInt()
+            setPadding(padding, padding, padding, padding)
+            addView(state)
+            addView(button)
+        })
+    }
+
+    override fun onResume() {
+        super.onResume()
+        state.text = if (AutoAccessibilityService.instance == null) "autoagent 未连接" else "autoagent 已就绪"
+    }
+}
@@ -0,0 +1,13 @@
+package cn.auto.agent.core.accessibility
+
+internal class ActivityTracker {
+    private val activities = mutableMapOf<String, String>()
+
+    @Synchronized
+    fun observe(packageName: String?, className: String?) {
+        if (!packageName.isNullOrBlank() && !className.isNullOrBlank()) activities[packageName] = className
+    }
+
+    @Synchronized
+    fun current(packageName: String): String? = activities[packageName]
+}
@@ -0,0 +1,73 @@
+package cn.auto.agent.core.accessibility
+
+import android.accessibilityservice.AccessibilityService
+import android.os.Bundle
+import android.view.accessibility.AccessibilityEvent
+import android.view.accessibility.AccessibilityNodeInfo
+import cn.auto.agent.core.executor.ActionDriver
+import cn.auto.agent.core.executor.UiNodeRef
+import cn.auto.agent.core.model.NodeSelector
+
+class AutoAccessibilityService : AccessibilityService(), ActionDriver {
+    private val activityTracker = ActivityTracker()
+
+    override fun onServiceConnected() { instance = this }
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent?) {
+        if (event?.eventType == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) {
+            activityTracker.observe(event.packageName?.toString(), event.className?.toString())
+        }
+    }
+
+    override fun onInterrupt() = Unit
+
+    override fun onDestroy() {
+        if (instance === this) instance = null
+        super.onDestroy()
+    }
+
+    override fun currentPackage(): String? = rootInActiveWindow?.packageName?.toString()
+    override fun currentActivity(): String? = currentPackage()?.let(activityTracker::current)
+
+    override fun find(selector: NodeSelector): List<UiNodeRef> {
+        val root = rootInActiveWindow ?: return emptyList()
+        val result = mutableListOf<UiNodeRef>()
+        walk(root) { node ->
+            if (node.matches(selector)) result += UiNodeRef(node, (node.text ?: node.contentDescription)?.toString())
+        }
+        return result
+    }
+
+    override fun click(node: UiNodeRef): Boolean {
+        var target = node.opaqueId as? AccessibilityNodeInfo ?: return false
+        while (!target.isClickable) target = target.parent ?: return false
+        return target.performAction(AccessibilityNodeInfo.ACTION_CLICK)
+    }
+
+    override fun input(node: UiNodeRef, value: String): Boolean {
+        val arguments = Bundle().apply {
+            putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, value)
+        }
+        return (node.opaqueId as? AccessibilityNodeInfo)
+            ?.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) == true
+    }
+
+    override fun back(): Boolean = performGlobalAction(GLOBAL_ACTION_BACK)
+
+    private fun walk(node: AccessibilityNodeInfo, visit: (AccessibilityNodeInfo) -> Unit) {
+        visit(node)
+        for (index in 0 until node.childCount) node.getChild(index)?.let { walk(it, visit) }
+    }
+
+    private fun AccessibilityNodeInfo.matches(selector: NodeSelector): Boolean =
+        (selector.resourceId == null || viewIdResourceName == selector.resourceId) &&
+            (selector.text == null || text?.toString() == selector.text) &&
+            (selector.contentDescription == null || contentDescription?.toString() == selector.contentDescription) &&
+            (selector.className == null || className?.toString() == selector.className) &&
+            (selector.clickable == null || isClickable == selector.clickable)
+
+    companion object {
+        @Volatile var instance: AutoAccessibilityService? = null
+            private set
+    }
+}
@@ -0,0 +1,15 @@
+package cn.auto.agent.core.executor
+
+import java.util.concurrent.atomic.AtomicReference
+
+class TaskExecutionMutex {
+    private val activeTaskId = AtomicReference<String?>(null)
+
+    fun tryAcquire(taskId: String): Boolean {
+        require(taskId.isNotBlank()) { "taskId 不能为空" }
+        return activeTaskId.compareAndSet(null, taskId) || activeTaskId.get() == taskId
+    }
+
+    fun currentTaskId(): String? = activeTaskId.get()
+    fun release(taskId: String): Boolean = activeTaskId.compareAndSet(taskId, null)
+}
@@ -0,0 +1,70 @@
+package cn.auto.agent.core.executor
+
+import cn.auto.agent.core.model.ActionType
+import cn.auto.agent.core.model.AgentTask
+import cn.auto.agent.core.model.NodeSelector
+import cn.auto.agent.core.model.TaskExecutionResult
+
+data class UiNodeRef(val opaqueId: Any, val text: String?)
+
+interface ActionDriver {
+    fun currentPackage(): String?
+    fun currentActivity(): String?
+    fun find(selector: NodeSelector): List<UiNodeRef>
+    fun click(node: UiNodeRef): Boolean
+    fun input(node: UiNodeRef, value: String): Boolean
+    fun back(): Boolean
+}
+
+class TaskExecutor(
+    private val driver: ActionDriver,
+    private val now: () -> Long = System::currentTimeMillis,
+    private val pause: (Long) -> Unit = Thread::sleep,
+) {
+    fun execute(task: AgentTask): TaskExecutionResult {
+        val output = linkedMapOf<String, MutableList<String>>()
+        task.steps.forEach { step ->
+            val deadline = now() + step.timeoutMs
+            var lastFailure = "控件未出现"
+            while (now() <= deadline) {
+                if (driver.currentPackage() != step.packageName) {
+                    lastFailure = "当前应用与步骤不匹配"
+                } else if (step.activityName != null && driver.currentActivity() != step.activityName) {
+                    lastFailure = "当前 Activity 与步骤不匹配"
+                } else if (step.action == ActionType.BACK) {
+                    if (driver.back()) break
+                    lastFailure = "返回操作失败"
+                } else {
+                    val matches = driver.find(requireNotNull(step.selector))
+                    when {
+                        matches.size > 1 -> return failure(step.id, "TASK_AMBIGUOUS", "控件匹配到多个节点", output)
+                        matches.isEmpty() -> lastFailure = "控件未出现"
+                        else -> {
+                            val node = matches.single()
+                            val successful = when (step.action) {
+                                ActionType.WAIT -> true
+                                ActionType.CLICK -> driver.click(node)
+                                ActionType.INPUT -> driver.input(node, requireNotNull(step.value))
+                                ActionType.EXTRACT_TEXT -> node.text?.trim()?.takeIf(String::isNotEmpty)?.also {
+                                    output.getOrPut(requireNotNull(step.outputField)) { mutableListOf() }.add(it)
+                                } != null
+                                ActionType.BACK -> error("handled above")
+                            }
+                            if (successful) break
+                            lastFailure = "操作执行失败"
+                        }
+                    }
+                }
+                if (now() >= deadline) {
+                    if (step.optional) break
+                    return failure(step.id, "TASK_NOT_MATCHED", lastFailure, output)
+                }
+                pause(minOf(100, (deadline - now()).coerceAtLeast(1)))
+            }
+        }
+        return TaskExecutionResult(true, "OK", "任务执行完成", output.mapValues { it.value.toList() })
+    }
+
+    private fun failure(id: String, code: String, detail: String, output: Map<String, List<String>>) =
+        TaskExecutionResult(false, code, "步骤 $id 失败：$detail", output)
+}
@@ -0,0 +1,42 @@
+package cn.auto.agent.core.model
+
+enum class TaskStatus { RECEIVED, RUNNING, SUCCEEDED, FAILED }
+enum class ActionType { WAIT, CLICK, INPUT, BACK, EXTRACT_TEXT }
+
+data class NodeSelector(
+    val resourceId: String? = null,
+    val text: String? = null,
+    val contentDescription: String? = null,
+    val className: String? = null,
+    val clickable: Boolean? = null,
+) {
+    fun isEmpty(): Boolean = resourceId == null && text == null && contentDescription == null &&
+        className == null && clickable == null
+}
+
+data class TaskStep(
+    val id: String,
+    val action: ActionType,
+    val packageName: String,
+    val activityName: String? = null,
+    val selector: NodeSelector? = null,
+    val value: String? = null,
+    val outputField: String? = null,
+    val timeoutMs: Long = 5_000,
+    val optional: Boolean = false,
+)
+
+data class AgentTask(
+    val taskId: String,
+    val revision: Long,
+    val assignedAgentName: String,
+    val steps: List<TaskStep>,
+    val rawPayload: String,
+)
+
+data class TaskExecutionResult(
+    val successful: Boolean,
+    val code: String,
+    val message: String,
+    val extracted: Map<String, List<String>> = emptyMap(),
+)
@@ -0,0 +1,66 @@
+package cn.auto.agent.core.persistence
+
+import android.content.ContentValues
+import android.content.Context
+import android.database.sqlite.SQLiteDatabase
+import android.database.sqlite.SQLiteOpenHelper
+import cn.auto.agent.core.model.AgentTask
+import cn.auto.agent.core.model.TaskStatus
+import cn.auto.agent.core.protocol.TaskParser
+
+class SqliteTaskInbox(context: Context) : SQLiteOpenHelper(context, DATABASE_NAME, null, DATABASE_VERSION), TaskInbox {
+    override fun onCreate(db: SQLiteDatabase) {
+        db.execSQL(
+            """
+            CREATE TABLE task_inbox (
+                task_id TEXT NOT NULL,
+                revision INTEGER NOT NULL,
+                assigned_agent_name TEXT NOT NULL,
+                payload_json TEXT NOT NULL,
+                status TEXT NOT NULL,
+                received_at INTEGER NOT NULL,
+                updated_at INTEGER NOT NULL,
+                PRIMARY KEY (task_id, revision)
+            )
+            """.trimIndent(),
+        )
+    }
+
+    override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) = Unit
+
+    override fun save(task: AgentTask) {
+        val now = System.currentTimeMillis()
+        writableDatabase.insertWithOnConflict(
+            "task_inbox",
+            null,
+            ContentValues().apply {
+                put("task_id", task.taskId)
+                put("revision", task.revision)
+                put("assigned_agent_name", task.assignedAgentName)
+                put("payload_json", task.rawPayload)
+                put("status", TaskStatus.RECEIVED.name.lowercase())
+                put("received_at", now)
+                put("updated_at", now)
+            },
+            SQLiteDatabase.CONFLICT_IGNORE,
+        )
+    }
+
+    override fun find(taskId: String, revision: Long): AgentTask? = readableDatabase.query(
+        "task_inbox",
+        arrayOf("payload_json"),
+        "task_id = ? AND revision = ?",
+        arrayOf(taskId, revision.toString()),
+        null,
+        null,
+        null,
+        "1",
+    ).use { cursor ->
+        if (!cursor.moveToFirst()) null else TaskParser.parse(cursor.getString(0))
+    }
+
+    private companion object {
+        const val DATABASE_NAME = "auto_agent.db"
+        const val DATABASE_VERSION = 1
+    }
+}
@@ -0,0 +1,8 @@
+package cn.auto.agent.core.persistence
+
+import cn.auto.agent.core.model.AgentTask
+
+interface TaskInbox {
+    fun save(task: AgentTask)
+    fun find(taskId: String, revision: Long): AgentTask?
+}
@@ -0,0 +1,68 @@
+package cn.auto.agent.core.protocol
+
+import cn.auto.agent.core.model.ActionType
+import cn.auto.agent.core.model.AgentTask
+import cn.auto.agent.core.model.NodeSelector
+import cn.auto.agent.core.model.TaskStep
+import org.json.JSONObject
+
+class TaskProtocolException(message: String) : IllegalArgumentException(message)
+
+object TaskParser {
+    fun parse(raw: String): AgentTask {
+        val root = runCatching { JSONObject(raw) }
+            .getOrElse { throw TaskProtocolException("任务不是有效的 JSON 对象") }
+        if (root.optInt("schemaVersion", -1) != 1) throw TaskProtocolException("仅支持 schemaVersion=1")
+        val taskId = root.requiredString("taskId")
+        val revision = root.optLong("revision", 1).also {
+            if (it <= 0) throw TaskProtocolException("revision 必须为正整数")
+        }
+        val assignedAgentName = root.requiredString("assignedAgentName")
+        val items = root.optJSONArray("steps") ?: throw TaskProtocolException("steps 必须是数组")
+        if (items.length() !in 1..200) throw TaskProtocolException("steps 必须包含 1..200 项")
+        val ids = mutableSetOf<String>()
+        val steps = (0 until items.length()).map { index ->
+            val item = items.optJSONObject(index) ?: throw TaskProtocolException("steps[$index] 必须是对象")
+            val id = item.requiredString("id")
+            if (!ids.add(id)) throw TaskProtocolException("步骤 id 不能重复")
+            val action = runCatching { ActionType.valueOf(item.requiredString("action").uppercase()) }
+                .getOrElse { throw TaskProtocolException("步骤 $id 的 action 不受支持") }
+            val selector = item.optJSONObject("selector")?.let {
+                NodeSelector(
+                    resourceId = it.optionalString("resourceId"),
+                    text = it.optionalString("text"),
+                    contentDescription = it.optionalString("contentDescription"),
+                    className = it.optionalString("className"),
+                    clickable = if (it.has("clickable")) it.getBoolean("clickable") else null,
+                )
+            }
+            if (action !in setOf(ActionType.BACK) && (selector == null || selector.isEmpty())) {
+                throw TaskProtocolException("步骤 $id 缺少 selector")
+            }
+            val value = item.optionalString("value")
+            val outputField = item.optionalString("outputField")
+            if (action == ActionType.INPUT && value == null) throw TaskProtocolException("输入步骤 $id 缺少 value")
+            if (action == ActionType.EXTRACT_TEXT && outputField == null) throw TaskProtocolException("读取步骤 $id 缺少 outputField")
+            TaskStep(
+                id = id,
+                action = action,
+                packageName = item.requiredString("packageName"),
+                activityName = item.optionalString("activityName"),
+                selector = selector,
+                value = value,
+                outputField = outputField,
+                timeoutMs = item.optLong("timeoutMs", 5_000).also {
+                    if (it !in 100..30_000) throw TaskProtocolException("步骤 $id 的 timeoutMs 必须为 100..30000")
+                },
+                optional = item.optBoolean("optional", false),
+            )
+        }
+        return AgentTask(taskId, revision, assignedAgentName, steps, raw)
+    }
+
+    private fun JSONObject.requiredString(name: String): String = optionalString(name)
+        ?: throw TaskProtocolException("$name 必填")
+
+    private fun JSONObject.optionalString(name: String): String? =
+        optString(name).trim().takeIf { it.isNotEmpty() }
+}
@@ -0,0 +1,13 @@
+package cn.auto.agent.core.scheduler
+
+/** External service adapter. Implement this interface after the server API is confirmed. */
+interface TaskGateway {
+    fun fetchTaskPayloads(): List<String>
+}
+
+data class TaskSyncSummary(
+    val fetched: Int,
+    val accepted: Int,
+    val discarded: Int,
+    val invalid: Int,
+)
@@ -0,0 +1,31 @@
+package cn.auto.agent.core.scheduler
+
+import cn.auto.agent.core.persistence.TaskInbox
+import cn.auto.agent.core.protocol.TaskParser
+
+class TaskSynchronizer(
+    private val gateway: TaskGateway,
+    private val inbox: TaskInbox,
+) {
+    fun sync(localAgentName: String): TaskSyncSummary {
+        require(localAgentName.isNotBlank()) { "设备名不能为空" }
+        val payloads = gateway.fetchTaskPayloads()
+        var accepted = 0
+        var discarded = 0
+        var invalid = 0
+        payloads.forEach { raw ->
+            val task = runCatching { TaskParser.parse(raw) }.getOrElse {
+                invalid++
+                return@forEach
+            }
+            // Exact, case-sensitive comparison is intentional. Non-target tasks are never persisted.
+            if (task.assignedAgentName != localAgentName) {
+                discarded++
+                return@forEach
+            }
+            inbox.save(task)
+            accepted++
+        }
+        return TaskSyncSummary(payloads.size, accepted, discarded, invalid)
+    }
+}
@@ -0,0 +1,5 @@
+<resources>
+    <string name="app_name">Auto Agent</string>
+    <string name="accessibility_service_name">autoagent</string>
+    <string name="accessibility_service_description">根据已授权的任务步骤执行点击、输入、返回、滑动和信息读取操作。</string>
+</resources>
@@ -0,0 +1,7 @@
+<resources>
+    <style name="Theme.AutoAgent" parent="Theme.MaterialComponents.DayNight.DarkActionBar">
+        <item name="colorPrimary">#2563EB</item>
+        <item name="colorPrimaryVariant">#1D4ED8</item>
+        <item name="colorSecondary">#22C55E</item>
+    </style>
+</resources>
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
+    android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged"
+    android:accessibilityFeedbackType="feedbackGeneric"
+    android:accessibilityFlags="flagReportViewIds|flagIncludeNotImportantViews|flagRetrieveInteractiveWindows"
+    android:canPerformGestures="true"
+    android:canRetrieveWindowContent="true"
+    android:description="@string/accessibility_service_description"
+    android:notificationTimeout="100" />
@@ -0,0 +1,42 @@
+package cn.auto.agent
+
+import cn.auto.agent.core.executor.ActionDriver
+import cn.auto.agent.core.executor.TaskExecutor
+import cn.auto.agent.core.executor.UiNodeRef
+import cn.auto.agent.core.model.NodeSelector
+import cn.auto.agent.core.protocol.TaskParser
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class TaskExecutorTest {
+    @Test
+    fun executesUniqueClick() {
+        var clicked = false
+        val driver = object : ActionDriver {
+            override fun currentPackage() = "com.example.target"
+            override fun currentActivity(): String? = null
+            override fun find(selector: NodeSelector) = listOf(UiNodeRef("node", "登录"))
+            override fun click(node: UiNodeRef): Boolean { clicked = true; return true }
+            override fun input(node: UiNodeRef, value: String) = false
+            override fun back() = false
+        }
+        val result = TaskExecutor(driver).execute(TaskParser.parse(TaskParserTest.taskJson("phone-01")))
+        assertTrue(result.successful)
+        assertTrue(clicked)
+    }
+
+    @Test
+    fun rejectsAmbiguousNodes() {
+        val driver = object : ActionDriver {
+            override fun currentPackage() = "com.example.target"
+            override fun currentActivity(): String? = null
+            override fun find(selector: NodeSelector) = listOf(UiNodeRef(1, null), UiNodeRef(2, null))
+            override fun click(node: UiNodeRef) = true
+            override fun input(node: UiNodeRef, value: String) = true
+            override fun back() = true
+        }
+        val result = TaskExecutor(driver).execute(TaskParser.parse(TaskParserTest.taskJson("phone-01")))
+        assertEquals("TASK_AMBIGUOUS", result.code)
+    }
+}
@@ -0,0 +1,40 @@
+package cn.auto.agent
+
+import cn.auto.agent.core.model.ActionType
+import cn.auto.agent.core.protocol.TaskParser
+import cn.auto.agent.core.protocol.TaskProtocolException
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class TaskParserTest {
+    @Test
+    fun parsesTypedTask() {
+        val task = TaskParser.parse(taskJson("phone-01"))
+        assertEquals("task-1", task.taskId)
+        assertEquals("phone-01", task.assignedAgentName)
+        assertEquals(ActionType.CLICK, task.steps.single().action)
+    }
+
+    @Test(expected = TaskProtocolException::class)
+    fun rejectsUnsupportedAction() {
+        TaskParser.parse(taskJson("phone-01").replace("CLICK", "SHELL"))
+    }
+
+    companion object {
+        fun taskJson(agentName: String) = """
+            {
+              "schemaVersion": 1,
+              "taskId": "task-1",
+              "revision": 1,
+              "assignedAgentName": "$agentName",
+              "steps": [{
+                "id": "click-login",
+                "action": "CLICK",
+                "packageName": "com.example.target",
+                "selector": {"resourceId": "com.example.target:id/login"},
+                "timeoutMs": 1000
+              }]
+            }
+        """.trimIndent()
+    }
+}
@@ -0,0 +1,37 @@
+package cn.auto.agent
+
+import cn.auto.agent.core.model.AgentTask
+import cn.auto.agent.core.persistence.TaskInbox
+import cn.auto.agent.core.scheduler.TaskGateway
+import cn.auto.agent.core.scheduler.TaskSynchronizer
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class TaskSynchronizerTest {
+    @Test
+    fun onlyPersistsTasksAssignedToExactLocalName() {
+        val inbox = MemoryInbox()
+        val gateway = object : TaskGateway {
+            override fun fetchTaskPayloads() = listOf(
+                TaskParserTest.taskJson("phone-01"),
+                TaskParserTest.taskJson("Phone-01").replace("task-1", "task-2"),
+                TaskParserTest.taskJson("phone-02").replace("task-1", "task-3"),
+                "invalid",
+            )
+        }
+
+        val result = TaskSynchronizer(gateway, inbox).sync("phone-01")
+
+        assertEquals(4, result.fetched)
+        assertEquals(1, result.accepted)
+        assertEquals(2, result.discarded)
+        assertEquals(1, result.invalid)
+        assertEquals(listOf("task-1"), inbox.tasks.map { it.taskId })
+    }
+
+    private class MemoryInbox : TaskInbox {
+        val tasks = mutableListOf<AgentTask>()
+        override fun save(task: AgentTask) { tasks += task }
+        override fun find(taskId: String, revision: Long) = tasks.find { it.taskId == taskId && it.revision == revision }
+    }
+}