diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index a3ef51f..9200838 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -11,8 +11,8 @@ android { applicationId = "cn.ilapage.goauto.agent" minSdk = 23 targetSdk = 34 - versionCode = 64 - versionName = "0.9.51" + versionCode = 65 + versionName = "0.9.52" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/GoAutoAccessibilityService.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/GoAutoAccessibilityService.kt index 477285a..fbd763a 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/GoAutoAccessibilityService.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/GoAutoAccessibilityService.kt @@ -330,6 +330,42 @@ class GoAutoAccessibilityService : AccessibilityService(), UiDriver, PddCollecto } } + override fun clickAddressEntryFresh(target: SnapshotNode): FreshClickOutcome { + val root = rootInActiveWindow ?: return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.ROOT_UNAVAILABLE) + val indexes = target.path.split('/').mapNotNull(String::toIntOrNull) + if (indexes.isEmpty() || indexes.first() != 0 || indexes.size != target.path.split('/').size) { + return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND) + } + var node = root + for (index in indexes.drop(1)) { + node = node.getChild(index) + ?: return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND) + } + val bounds = Rect().also(node::getBoundsInScreen) + if (!node.isVisibleToUser || !node.isEnabled || + node.preferredOrDescendantLabel() != target.label || + node.className?.toString() != target.className || + kotlin.math.abs(bounds.centerX() - target.bounds.centerX) > 32 || + kotlin.math.abs(bounds.centerY() - target.bounds.centerY) > 32 + ) { + return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND) + } + var ancestorDepth = 0 + while (!node.isClickable) { + node = node.parent ?: return FreshClickOutcome( + FreshActionResult.FAILED, + FreshClickReason.NO_CLICKABLE_ANCESTOR, + clickableAncestorDepth = ancestorDepth, + ) + ancestorDepth++ + } + return if (node.performAction(AccessibilityNodeInfo.ACTION_CLICK)) { + FreshClickOutcome(FreshActionResult.SUCCESS, FreshClickReason.SUCCESS, 1, ancestorDepth) + } else { + FreshClickOutcome(FreshActionResult.FAILED, FreshClickReason.ACTION_CLICK_FALSE, 1, ancestorDepth) + } + } + override fun tapPurchaseFresh(target: SnapshotNode): FreshActionResult { val root = rootInActiveWindow ?: return FreshActionResult.NOT_FOUND val candidates = mutableListOf() diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt index 8cd35db..b68517a 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt @@ -113,7 +113,7 @@ class PurchaseLiveAutomation( } val target = stableResolution.target val result = when (target.activation) { - AddressEntryActivation.CLICK -> driver.clickFreshDetailed(target.node).result + AddressEntryActivation.CLICK -> driver.clickAddressEntryFresh(target.node).result AddressEntryActivation.TAP -> driver.tapPurchaseFresh(target.node) } when (result) { @@ -153,6 +153,12 @@ class PurchaseLiveAutomation( val cardKey: String, ) + private data class AddressCardCandidate( + val phone: SnapshotNode, + val clickableCard: SnapshotNode?, + val semanticKey: String?, + ) + private data class AddressEntryResolution( val phoneNodeCount: Int, val addressCardCount: Int, @@ -183,22 +189,33 @@ class PurchaseLiveAutomation( if (phones.isEmpty()) return AddressEntryResolution(0, 0, 0, null) val byPath = snapshot.nodes.associateBy(SnapshotNode::path) - val withClickableCard = phones.mapNotNull { phone -> - nearestClickableAncestor(phone, byPath)?.let { ancestor -> ancestor.path to phone } + val candidates = phones.map { phone -> + val card = nearestClickableAncestor(phone, byPath) + AddressCardCandidate(phone, card, card?.let { addressCardSemanticKey(it, snapshot) }) + } + val groups = mutableListOf>() + candidates.forEach { candidate -> + val matching = groups.filter { group -> group.any { sameLogicalAddress(it, candidate) } } + if (matching.isEmpty()) { + groups += mutableListOf(candidate) + } else { + val primary = matching.first() + primary += candidate + matching.drop(1).forEach { duplicate -> + primary += duplicate + groups.remove(duplicate) + } + } } - val clickableGroups = withClickableCard.groupBy({ it.first }, { it.second }) - val assignedPaths = withClickableCard.map { it.second.path }.toSet() - val targets = clickableGroups.map { (cardPath, nodes) -> - AddressEntryTarget(nodes.minBy(::nodeArea), AddressEntryActivation.CLICK, "click:$cardPath") - }.toMutableList() - val spatialGroups = mutableListOf>() - phones.filterNot { it.path in assignedPaths }.forEach { phone -> - val group = spatialGroups.firstOrNull { existing -> existing.any { sameAddressRegion(it, phone) } } - if (group == null) spatialGroups += mutableListOf(phone) else group += phone - } - spatialGroups.forEachIndexed { index, nodes -> - targets += AddressEntryTarget(nodes.minBy(::nodeArea), AddressEntryActivation.TAP, "tap:$index") + val targets = groups.mapIndexed { index, group -> + val clickable = group.filter { it.clickableCard != null } + .minWithOrNull(compareBy({ nodeArea(requireNotNull(it.clickableCard)) }, { it.phone.path })) + if (clickable != null) { + AddressEntryTarget(clickable.phone, AddressEntryActivation.CLICK, logicalAddressKey(group, index)) + } else { + AddressEntryTarget(group.minBy { nodeArea(it.phone) }.phone, AddressEntryActivation.TAP, logicalAddressKey(group, index)) + } } val actionable = targets.filter { it.node.bounds.width >= 2 && it.node.bounds.height >= 2 } @@ -222,6 +239,47 @@ class PurchaseLiveAutomation( return null } + private fun addressCardSemanticKey(card: SnapshotNode, snapshot: UiSnapshot): String? { + val labels = snapshot.nodes.asSequence() + .filter { it.visible && (it.path == card.path || it.path.startsWith("${card.path}/")) } + .map { normalizeAddressSemantic(it.label) } + .filter { it.isNotBlank() && maskedPhone(it) == null && it !in ADDRESS_CARD_GENERIC_LABELS } + .distinct() + .sorted() + .toList() + if (labels.none { it.length >= ADDRESS_SEMANTIC_MIN_LENGTH }) return null + return labels.joinToString("|") + } + + private fun normalizeAddressSemantic(value: String): String = value + .replace(Regex("\\s+"), "") + .replace("，", ",") + .replace("：", ":") + .lowercase() + + private fun sameLogicalAddress(first: AddressCardCandidate, second: AddressCardCandidate): Boolean { + if (maskedPhone(first.phone.label) != maskedPhone(second.phone.label)) return false + if (first.clickableCard?.path != null && first.clickableCard.path == second.clickableCard?.path) return true + if (sameAddressRegion(first.phone, second.phone)) return true + val firstCard = first.clickableCard + val secondCard = second.clickableCard + if (firstCard != null && secondCard != null && boundsRepresentSameRegion(firstCard.bounds, secondCard.bounds)) return true + return first.semanticKey != null && first.semanticKey == second.semanticKey + } + + private fun boundsRepresentSameRegion(first: NodeBounds, second: NodeBounds): Boolean { + val overlapWidth = (minOf(first.right, second.right) - maxOf(first.left, second.left)).coerceAtLeast(0) + val overlapHeight = (minOf(first.bottom, second.bottom) - maxOf(first.top, second.top)).coerceAtLeast(0) + val overlap = overlapWidth.toLong() * overlapHeight + val smaller = minOf(first.width.toLong() * first.height, second.width.toLong() * second.height) + return smaller > 0 && overlap * 100 >= smaller * ADDRESS_CARD_OVERLAP_PERCENT + } + + private fun logicalAddressKey(group: List, index: Int): String { + val semantic = group.mapNotNull(AddressCardCandidate::semanticKey).distinct().singleOrNull() + return semantic?.let { "semantic:$it" } ?: "geometry:$index:${group.minOf { it.phone.bounds.top }}" + } + private fun sameAddressRegion(first: SnapshotNode, second: SnapshotNode): Boolean { if (maskedPhone(first.label) != maskedPhone(second.label)) return false val overlapWidth = (minOf(first.bounds.right, second.bounds.right) - maxOf(first.bounds.left, second.bounds.left)).coerceAtLeast(0) @@ -711,6 +769,9 @@ class PurchaseLiveAutomation( const val SPEC_CONFIRMATION_MAX_SAMPLES = 20 const val SPEC_CONFIRMATION_SAMPLE_INTERVAL_MS = 100L const val ADDRESS_ENTRY_STABLE_INTERVAL_MS = 200L + const val ADDRESS_CARD_OVERLAP_PERCENT = 70 + const val ADDRESS_SEMANTIC_MIN_LENGTH = 4 + val ADDRESS_CARD_GENERIC_LABELS = setOf("收货地址", "修改", "默认地址", "默认") const val ADDRESS_CONFIRMATION_SCROLL_LIMIT = 5 const val ADDRESS_CONFIRMATION_SCROLL_INTERVAL_MS = 500L } diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseRehearsalExecutor.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseRehearsalExecutor.kt index 7035dbf..ab456ed 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseRehearsalExecutor.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseRehearsalExecutor.kt @@ -10,6 +10,13 @@ interface PurchaseUiDriver { result = clickFresh(target), reason = FreshClickReason.UNKNOWN, ) + /** + * Reacquires one address-entry node by its accessibility path and validates + * its immutable snapshot traits before clicking its nearest clickable + * ancestor. This is intentionally limited to the reversible transition + * from order confirmation into address management. + */ + fun clickAddressEntryFresh(target: SnapshotNode): FreshClickOutcome = clickFreshDetailed(target) fun tapPurchaseFresh(target: SnapshotNode): FreshActionResult /** * Reacquires and center-taps a target that the purchase parser has already diff --git a/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseLiveAutomationTest.kt b/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseLiveAutomationTest.kt index 26b6470..7b6a5b7 100644 --- a/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseLiveAutomationTest.kt +++ b/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseLiveAutomationTest.kt @@ -1,6 +1,8 @@ package cn.ilapage.goauto.agent import cn.ilapage.goauto.agent.automation.FreshActionResult +import cn.ilapage.goauto.agent.automation.FreshClickOutcome +import cn.ilapage.goauto.agent.automation.FreshClickReason import cn.ilapage.goauto.agent.automation.NodeBounds import cn.ilapage.goauto.agent.automation.PurchaseExecutionInput import cn.ilapage.goauto.agent.automation.PurchaseLiveAutomation @@ -509,6 +511,17 @@ class PurchaseLiveAutomationTest { assertEquals(0, driver.addressTaps) } + @Test + fun `duplicate semantic address cards in separate accessibility trees are activated once`() { + val driver = LiveDriver(duplicateSemanticAddressCards = true) + + PurchaseLiveAutomation(driver, pause = {}).updateShippingAddress("_cg94") + + assertEquals(1, driver.addressPathClicks) + assertEquals(1, driver.clicked.count { it == "138****5678" }) + assertEquals(0, driver.addressTaps) + } + @Test fun `two independent address cards remain ambiguous without any click`() { val driver = LiveDriver(duplicateAddressCards = true) @@ -522,6 +535,18 @@ class PurchaseLiveAutomationTest { assertEquals(0, driver.addressTaps) } + @Test + fun `address entry path drift fails without fallback tap`() { + val driver = LiveDriver(duplicatePhoneNodesSameCard = true, addressEntryPathUnavailable = true) + val error = runCatching { PurchaseLiveAutomation(driver, pause = {}).updateShippingAddress("_cg94") } + .exceptionOrNull() as PurchaseLiveException + + assertEquals("PURCHASE_ADDRESS_ENTRY_NOT_READY", error.code) + assertTrue(driver.clicked.isEmpty()) + assertEquals(1, driver.addressPathClicks) + assertEquals(0, driver.addressTaps) + } + private fun input() = PurchaseExecutionInput( taskId = 11, executionMode = "live", @@ -622,7 +647,9 @@ class PurchaseLiveAutomationTest { private val saveReturnsToSpecPanel: Boolean = false, private val savedSpecPanelRecoveryStuck: Boolean = false, private val duplicatePhoneNodesSameCard: Boolean = false, + private val duplicateSemanticAddressCards: Boolean = false, private val duplicateAddressCards: Boolean = false, + private val addressEntryPathUnavailable: Boolean = false, private val wechatLoginAfterSubmit: Boolean = false, private val wechatRestoreStuck: Boolean = false, private val orderEvidenceBelowFold: Boolean = false, @@ -641,6 +668,7 @@ class PurchaseLiveAutomationTest { var scopedSwipes = 0 var genericSwipes = 0 var addressTaps = 0 + var addressPathClicks = 0 var lastInputTargetPath: String? = null var backCount = 0 var postSubmitBackCount = 0 @@ -727,11 +755,21 @@ class PurchaseLiveAutomationTest { if (duplicatePanels) nodes += node("panel2", "", scrollable = true, bounds = NodeBounds(0, 500, 1080, 2000)) if (addressVisible) { when { + duplicateSemanticAddressCards -> { + nodes += node("address-layer-a", "", clickable = true, bounds = NodeBounds(0, 620, 1080, 840)) + nodes += node("address-layer-a/phone", "138****5678", parentPath = "address-layer-a", bounds = NodeBounds(20, 650, 400, 710)) + nodes += node("address-layer-a/detail", "广东省广州市天园街道骏景花园", parentPath = "address-layer-a", bounds = NodeBounds(20, 720, 900, 790)) + nodes += node("address-layer-b", "", clickable = true, bounds = NodeBounds(0, 900, 1080, 1120)) + nodes += node("address-layer-b/phone", "138****5678", parentPath = "address-layer-b", bounds = NodeBounds(20, 930, 400, 990)) + nodes += node("address-layer-b/detail", "广东省广州市天园街道骏景花园", parentPath = "address-layer-b", bounds = NodeBounds(20, 1000, 900, 1070)) + } duplicateAddressCards -> { nodes += node("address-card-a", "", clickable = true, bounds = NodeBounds(0, 620, 1080, 820)) nodes += node("address-card-a/phone", "138****5678", parentPath = "address-card-a", bounds = NodeBounds(20, 650, 400, 710)) + nodes += node("address-card-a/detail", "广东省广州市天园街道一号", parentPath = "address-card-a", bounds = NodeBounds(20, 720, 900, 780)) nodes += node("address-card-b", "", clickable = true, bounds = NodeBounds(0, 840, 1080, 1040)) nodes += node("address-card-b/phone", "138****5678", parentPath = "address-card-b", bounds = NodeBounds(20, 870, 400, 930)) + nodes += node("address-card-b/detail", "广东省广州市天园街道二号", parentPath = "address-card-b", bounds = NodeBounds(20, 940, 900, 1000)) } duplicatePhoneNodesSameCard -> { nodes += node("address-card", "", clickable = true, bounds = NodeBounds(0, 620, 1080, 900)) @@ -788,6 +826,14 @@ class PurchaseLiveAutomationTest { return FreshActionResult.SUCCESS } + override fun clickAddressEntryFresh(target: SnapshotNode): FreshClickOutcome { + addressPathClicks++ + if (addressEntryPathUnavailable) { + return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND) + } + return FreshClickOutcome(clickFresh(target), FreshClickReason.SUCCESS, 1, 1) + } + override fun tapPurchaseFresh(target: SnapshotNode): FreshActionResult { addressTaps++ clicked += target.label