fix(agent): resolve duplicated semantic address cards (#229)

2026-09-05 18:28:47 +08:00
parent c9aaade6e9
commit 1f40a7fcb1
5 changed files with 167 additions and 17 deletions
@@ -11,8 +11,8 @@ android {
        applicationId = "cn.ilapage.goauto.agent"
        minSdk = 23
        targetSdk = 34
-        versionCode = 64
-        versionName = "0.9.51"
+        versionCode = 65
+        versionName = "0.9.52"

        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"

@@ -330,6 +330,42 @@ class GoAutoAccessibilityService : AccessibilityService(), UiDriver, PddCollecto
        }
    }

+    override fun clickAddressEntryFresh(target: SnapshotNode): FreshClickOutcome {
+        val root = rootInActiveWindow ?: return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.ROOT_UNAVAILABLE)
+        val indexes = target.path.split('/').mapNotNull(String::toIntOrNull)
+        if (indexes.isEmpty() || indexes.first() != 0 || indexes.size != target.path.split('/').size) {
+            return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND)
+        }
+        var node = root
+        for (index in indexes.drop(1)) {
+            node = node.getChild(index)
+                ?: return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND)
+        }
+        val bounds = Rect().also(node::getBoundsInScreen)
+        if (!node.isVisibleToUser || !node.isEnabled ||
+            node.preferredOrDescendantLabel() != target.label ||
+            node.className?.toString() != target.className ||
+            kotlin.math.abs(bounds.centerX() - target.bounds.centerX) > 32 ||
+            kotlin.math.abs(bounds.centerY() - target.bounds.centerY) > 32
+        ) {
+            return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND)
+        }
+        var ancestorDepth = 0
+        while (!node.isClickable) {
+            node = node.parent ?: return FreshClickOutcome(
+                FreshActionResult.FAILED,
+                FreshClickReason.NO_CLICKABLE_ANCESTOR,
+                clickableAncestorDepth = ancestorDepth,
+            )
+            ancestorDepth++
+        }
+        return if (node.performAction(AccessibilityNodeInfo.ACTION_CLICK)) {
+            FreshClickOutcome(FreshActionResult.SUCCESS, FreshClickReason.SUCCESS, 1, ancestorDepth)
+        } else {
+            FreshClickOutcome(FreshActionResult.FAILED, FreshClickReason.ACTION_CLICK_FALSE, 1, ancestorDepth)
+        }
+    }
+
    override fun tapPurchaseFresh(target: SnapshotNode): FreshActionResult {
        val root = rootInActiveWindow ?: return FreshActionResult.NOT_FOUND
        val candidates = mutableListOf<AccessibilityNodeInfo>()
@@ -113,7 +113,7 @@ class PurchaseLiveAutomation(
                }
                val target = stableResolution.target
                val result = when (target.activation) {
-                    AddressEntryActivation.CLICK -> driver.clickFreshDetailed(target.node).result
+                    AddressEntryActivation.CLICK -> driver.clickAddressEntryFresh(target.node).result
                    AddressEntryActivation.TAP -> driver.tapPurchaseFresh(target.node)
                }
                when (result) {
@@ -153,6 +153,12 @@ class PurchaseLiveAutomation(
        val cardKey: String,
    )

+    private data class AddressCardCandidate(
+        val phone: SnapshotNode,
+        val clickableCard: SnapshotNode?,
+        val semanticKey: String?,
+    )
+
    private data class AddressEntryResolution(
        val phoneNodeCount: Int,
        val addressCardCount: Int,
@@ -183,22 +189,33 @@ class PurchaseLiveAutomation(
        if (phones.isEmpty()) return AddressEntryResolution(0, 0, 0, null)

        val byPath = snapshot.nodes.associateBy(SnapshotNode::path)
-        val withClickableCard = phones.mapNotNull { phone ->
-            nearestClickableAncestor(phone, byPath)?.let { ancestor -> ancestor.path to phone }
+        val candidates = phones.map { phone ->
+            val card = nearestClickableAncestor(phone, byPath)
+            AddressCardCandidate(phone, card, card?.let { addressCardSemanticKey(it, snapshot) })
+        }
+        val groups = mutableListOf<MutableList<AddressCardCandidate>>()
+        candidates.forEach { candidate ->
+            val matching = groups.filter { group -> group.any { sameLogicalAddress(it, candidate) } }
+            if (matching.isEmpty()) {
+                groups += mutableListOf(candidate)
+            } else {
+                val primary = matching.first()
+                primary += candidate
+                matching.drop(1).forEach { duplicate ->
+                    primary += duplicate
+                    groups.remove(duplicate)
+                }
+            }
        }
-        val clickableGroups = withClickableCard.groupBy({ it.first }, { it.second })
-        val assignedPaths = withClickableCard.map { it.second.path }.toSet()
-        val targets = clickableGroups.map { (cardPath, nodes) ->
-            AddressEntryTarget(nodes.minBy(::nodeArea), AddressEntryActivation.CLICK, "click:$cardPath")
-        }.toMutableList()

-        val spatialGroups = mutableListOf<MutableList<SnapshotNode>>()
-        phones.filterNot { it.path in assignedPaths }.forEach { phone ->
-            val group = spatialGroups.firstOrNull { existing -> existing.any { sameAddressRegion(it, phone) } }
-            if (group == null) spatialGroups += mutableListOf(phone) else group += phone
-        }
-        spatialGroups.forEachIndexed { index, nodes ->
-            targets += AddressEntryTarget(nodes.minBy(::nodeArea), AddressEntryActivation.TAP, "tap:$index")
+        val targets = groups.mapIndexed { index, group ->
+            val clickable = group.filter { it.clickableCard != null }
+                .minWithOrNull(compareBy<AddressCardCandidate>({ nodeArea(requireNotNull(it.clickableCard)) }, { it.phone.path }))
+            if (clickable != null) {
+                AddressEntryTarget(clickable.phone, AddressEntryActivation.CLICK, logicalAddressKey(group, index))
+            } else {
+                AddressEntryTarget(group.minBy { nodeArea(it.phone) }.phone, AddressEntryActivation.TAP, logicalAddressKey(group, index))
+            }
        }

        val actionable = targets.filter { it.node.bounds.width >= 2 && it.node.bounds.height >= 2 }
@@ -222,6 +239,47 @@ class PurchaseLiveAutomation(
        return null
    }

+    private fun addressCardSemanticKey(card: SnapshotNode, snapshot: UiSnapshot): String? {
+        val labels = snapshot.nodes.asSequence()
+            .filter { it.visible && (it.path == card.path || it.path.startsWith("${card.path}/")) }
+            .map { normalizeAddressSemantic(it.label) }
+            .filter { it.isNotBlank() && maskedPhone(it) == null && it !in ADDRESS_CARD_GENERIC_LABELS }
+            .distinct()
+            .sorted()
+            .toList()
+        if (labels.none { it.length >= ADDRESS_SEMANTIC_MIN_LENGTH }) return null
+        return labels.joinToString("|")
+    }
+
+    private fun normalizeAddressSemantic(value: String): String = value
+        .replace(Regex("\\s+"), "")
+        .replace("，", ",")
+        .replace("：", ":")
+        .lowercase()
+
+    private fun sameLogicalAddress(first: AddressCardCandidate, second: AddressCardCandidate): Boolean {
+        if (maskedPhone(first.phone.label) != maskedPhone(second.phone.label)) return false
+        if (first.clickableCard?.path != null && first.clickableCard.path == second.clickableCard?.path) return true
+        if (sameAddressRegion(first.phone, second.phone)) return true
+        val firstCard = first.clickableCard
+        val secondCard = second.clickableCard
+        if (firstCard != null && secondCard != null && boundsRepresentSameRegion(firstCard.bounds, secondCard.bounds)) return true
+        return first.semanticKey != null && first.semanticKey == second.semanticKey
+    }
+
+    private fun boundsRepresentSameRegion(first: NodeBounds, second: NodeBounds): Boolean {
+        val overlapWidth = (minOf(first.right, second.right) - maxOf(first.left, second.left)).coerceAtLeast(0)
+        val overlapHeight = (minOf(first.bottom, second.bottom) - maxOf(first.top, second.top)).coerceAtLeast(0)
+        val overlap = overlapWidth.toLong() * overlapHeight
+        val smaller = minOf(first.width.toLong() * first.height, second.width.toLong() * second.height)
+        return smaller > 0 && overlap * 100 >= smaller * ADDRESS_CARD_OVERLAP_PERCENT
+    }
+
+    private fun logicalAddressKey(group: List<AddressCardCandidate>, index: Int): String {
+        val semantic = group.mapNotNull(AddressCardCandidate::semanticKey).distinct().singleOrNull()
+        return semantic?.let { "semantic:$it" } ?: "geometry:$index:${group.minOf { it.phone.bounds.top }}"
+    }
+
    private fun sameAddressRegion(first: SnapshotNode, second: SnapshotNode): Boolean {
        if (maskedPhone(first.label) != maskedPhone(second.label)) return false
        val overlapWidth = (minOf(first.bounds.right, second.bounds.right) - maxOf(first.bounds.left, second.bounds.left)).coerceAtLeast(0)
@@ -711,6 +769,9 @@ class PurchaseLiveAutomation(
        const val SPEC_CONFIRMATION_MAX_SAMPLES = 20
        const val SPEC_CONFIRMATION_SAMPLE_INTERVAL_MS = 100L
        const val ADDRESS_ENTRY_STABLE_INTERVAL_MS = 200L
+        const val ADDRESS_CARD_OVERLAP_PERCENT = 70
+        const val ADDRESS_SEMANTIC_MIN_LENGTH = 4
+        val ADDRESS_CARD_GENERIC_LABELS = setOf("收货地址", "修改", "默认地址", "默认")
        const val ADDRESS_CONFIRMATION_SCROLL_LIMIT = 5
        const val ADDRESS_CONFIRMATION_SCROLL_INTERVAL_MS = 500L
    }
@@ -10,6 +10,13 @@ interface PurchaseUiDriver {
        result = clickFresh(target),
        reason = FreshClickReason.UNKNOWN,
    )
+    /**
+     * Reacquires one address-entry node by its accessibility path and validates
+     * its immutable snapshot traits before clicking its nearest clickable
+     * ancestor. This is intentionally limited to the reversible transition
+     * from order confirmation into address management.
+     */
+    fun clickAddressEntryFresh(target: SnapshotNode): FreshClickOutcome = clickFreshDetailed(target)
    fun tapPurchaseFresh(target: SnapshotNode): FreshActionResult
    /**
     * Reacquires and center-taps a target that the purchase parser has already
@@ -1,6 +1,8 @@
 package cn.ilapage.goauto.agent

 import cn.ilapage.goauto.agent.automation.FreshActionResult
+import cn.ilapage.goauto.agent.automation.FreshClickOutcome
+import cn.ilapage.goauto.agent.automation.FreshClickReason
 import cn.ilapage.goauto.agent.automation.NodeBounds
 import cn.ilapage.goauto.agent.automation.PurchaseExecutionInput
 import cn.ilapage.goauto.agent.automation.PurchaseLiveAutomation
@@ -509,6 +511,17 @@ class PurchaseLiveAutomationTest {
        assertEquals(0, driver.addressTaps)
    }

+    @Test
+    fun `duplicate semantic address cards in separate accessibility trees are activated once`() {
+        val driver = LiveDriver(duplicateSemanticAddressCards = true)
+
+        PurchaseLiveAutomation(driver, pause = {}).updateShippingAddress("_cg94")
+
+        assertEquals(1, driver.addressPathClicks)
+        assertEquals(1, driver.clicked.count { it == "138****5678" })
+        assertEquals(0, driver.addressTaps)
+    }
+
    @Test
    fun `two independent address cards remain ambiguous without any click`() {
        val driver = LiveDriver(duplicateAddressCards = true)
@@ -522,6 +535,18 @@ class PurchaseLiveAutomationTest {
        assertEquals(0, driver.addressTaps)
    }

+    @Test
+    fun `address entry path drift fails without fallback tap`() {
+        val driver = LiveDriver(duplicatePhoneNodesSameCard = true, addressEntryPathUnavailable = true)
+        val error = runCatching { PurchaseLiveAutomation(driver, pause = {}).updateShippingAddress("_cg94") }
+            .exceptionOrNull() as PurchaseLiveException
+
+        assertEquals("PURCHASE_ADDRESS_ENTRY_NOT_READY", error.code)
+        assertTrue(driver.clicked.isEmpty())
+        assertEquals(1, driver.addressPathClicks)
+        assertEquals(0, driver.addressTaps)
+    }
+
    private fun input() = PurchaseExecutionInput(
        taskId = 11,
        executionMode = "live",
@@ -622,7 +647,9 @@ class PurchaseLiveAutomationTest {
        private val saveReturnsToSpecPanel: Boolean = false,
        private val savedSpecPanelRecoveryStuck: Boolean = false,
        private val duplicatePhoneNodesSameCard: Boolean = false,
+        private val duplicateSemanticAddressCards: Boolean = false,
        private val duplicateAddressCards: Boolean = false,
+        private val addressEntryPathUnavailable: Boolean = false,
        private val wechatLoginAfterSubmit: Boolean = false,
        private val wechatRestoreStuck: Boolean = false,
        private val orderEvidenceBelowFold: Boolean = false,
@@ -641,6 +668,7 @@ class PurchaseLiveAutomationTest {
        var scopedSwipes = 0
        var genericSwipes = 0
        var addressTaps = 0
+        var addressPathClicks = 0
        var lastInputTargetPath: String? = null
        var backCount = 0
        var postSubmitBackCount = 0
@@ -727,11 +755,21 @@ class PurchaseLiveAutomationTest {
                if (duplicatePanels) nodes += node("panel2", "", scrollable = true, bounds = NodeBounds(0, 500, 1080, 2000))
                if (addressVisible) {
                    when {
+                        duplicateSemanticAddressCards -> {
+                            nodes += node("address-layer-a", "", clickable = true, bounds = NodeBounds(0, 620, 1080, 840))
+                            nodes += node("address-layer-a/phone", "138****5678", parentPath = "address-layer-a", bounds = NodeBounds(20, 650, 400, 710))
+                            nodes += node("address-layer-a/detail", "广东省广州市天园街道骏景花园", parentPath = "address-layer-a", bounds = NodeBounds(20, 720, 900, 790))
+                            nodes += node("address-layer-b", "", clickable = true, bounds = NodeBounds(0, 900, 1080, 1120))
+                            nodes += node("address-layer-b/phone", "138****5678", parentPath = "address-layer-b", bounds = NodeBounds(20, 930, 400, 990))
+                            nodes += node("address-layer-b/detail", "广东省广州市天园街道骏景花园", parentPath = "address-layer-b", bounds = NodeBounds(20, 1000, 900, 1070))
+                        }
                        duplicateAddressCards -> {
                            nodes += node("address-card-a", "", clickable = true, bounds = NodeBounds(0, 620, 1080, 820))
                            nodes += node("address-card-a/phone", "138****5678", parentPath = "address-card-a", bounds = NodeBounds(20, 650, 400, 710))
+                            nodes += node("address-card-a/detail", "广东省广州市天园街道一号", parentPath = "address-card-a", bounds = NodeBounds(20, 720, 900, 780))
                            nodes += node("address-card-b", "", clickable = true, bounds = NodeBounds(0, 840, 1080, 1040))
                            nodes += node("address-card-b/phone", "138****5678", parentPath = "address-card-b", bounds = NodeBounds(20, 870, 400, 930))
+                            nodes += node("address-card-b/detail", "广东省广州市天园街道二号", parentPath = "address-card-b", bounds = NodeBounds(20, 940, 900, 1000))
                        }
                        duplicatePhoneNodesSameCard -> {
                            nodes += node("address-card", "", clickable = true, bounds = NodeBounds(0, 620, 1080, 900))
@@ -788,6 +826,14 @@ class PurchaseLiveAutomationTest {
            return FreshActionResult.SUCCESS
        }

+        override fun clickAddressEntryFresh(target: SnapshotNode): FreshClickOutcome {
+            addressPathClicks++
+            if (addressEntryPathUnavailable) {
+                return FreshClickOutcome(FreshActionResult.NOT_FOUND, FreshClickReason.TARGET_NOT_FOUND)
+            }
+            return FreshClickOutcome(clickFresh(target), FreshClickReason.SUCCESS, 1, 1)
+        }
+
        override fun tapPurchaseFresh(target: SnapshotNode): FreshActionResult {
            addressTaps++
            clicked += target.label