From 2ce59eb75720e11e07bb6f1c57e615e61871ea0a Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Tue, 22 Sep 2026 16:01:04 +0800 Subject: [PATCH] fix(android): aim submit target at labelled leaf so clickFresh can re-find it (#335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-device result on build 0.9.62 (commit 34d09cd, task 594): the bottom container was selected correctly but the click never happened. GoAutoAccessibilityService.clickFreshDetailed re-finds its target live by (preferredOrDescendantLabel() == target.label, className, center ±32) and only THEN climbs to the nearest clickable ancestor and clicks it. The previous finalSubmitTargets returned the clickable FrameLayout container itself, whose SnapshotNode.label is "" (its text lives only in children), while the live preferredOrDescendantLabel() digs into a child and returns real text ("大促价，") - so the re-find always missed -> TARGET_NOT_FOUND -> PURCHASE_ORDER_RESULT_UNKNOWN right after the irreversible boundary. No order was created (user confirmed). Per the user's restated strategy, finalSubmitTargets now aims at the bottom-right-most VISIBLE, non-zero-size node that has a NON-BLANK OWN LABEL within the recognized panel container, requiring it to have a clickable ancestor (or be clickable itself) of non-zero size; ties go to the rightmost. This is the same "aim at the labelled descendant, let clickFresh climb to the clickable ancestor" convention already used by ImageSearchCandidatePolicy.clickTargetInside for PDD image-search result cards. The nearest clickable ancestor's subtree (not just the chosen leaf's own subtree) must not contain SUBMIT_TARGET_BLOCKED_MARKERS, since a payment word can live in a sibling leaf under the same clickable row. hasFinalSavedAddressEvidence, finalConfirmation and submitOrderOnce still all consume this one finalSubmitTargets, so the address-save evidence check and the final click stay consistent. Tests: added ReFindingDriver, a fake driver that emulates clickFreshDetailed's real re-find + climb-to-clickable-ancestor semantics against a fixed live node list, with a dedicated test on the task 570/594 structure that asserts the click actually lands on the clickable FrameLayout container, not the labelled leaf finalSubmitTargets aimed at - this is the class of test that would have caught the task 594 regression; prior StaticDriver-only tests could not, since StaticDriver's clickFresh just records target.label directly. Updated the existing 570 test and the zero-size test to assert on the now-correct leaf-label target. Verified every other #335 safety test (legacy 提交订单, tie/rightmost, payment-word blocked, outside-panel-container, no-determinable-container) still passes unmodified. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NTDbDcwbDw1TSAcE6wfh2F --- .../automation/PurchaseLiveAutomation.kt | 63 ++++++++++---- .../goauto/agent/SpecPanelRecognitionTest.kt | 87 ++++++++++++++++--- 2 files changed, 122 insertions(+), 28 deletions(-) diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt index b628f41..2fdc8d5 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt @@ -777,15 +777,35 @@ class PurchaseLiveAutomation( } /** - * #335: the order button's own text keeps changing (促销文案、价格文案等），因此不再依赖 - * [FINAL_SUBMIT_MARKERS] 文字匹配判定必要条件；改为用户确认的策略——在已识别的规格面板 - * （沿用 #331 的 [PURCHASE_CONFIRMATION_PANEL_TYPES] 判定）内，取属于该面板、最下沿最低、 - * 可点击、可用、尺寸非零的节点。多个节点并列最下沿时取最右侧并在诊断中标记“并列”。节点自身 - * 或其子树文字命中 [SUBMIT_TARGET_BLOCKED_MARKERS] 时明确失败、不返回目标。旧的文字匹配 - * （[FINAL_SUBMIT_MARKERS]）仅作诊断证据，不再是必要条件。 + * #335 (2nd revision, task 594 real-device failure): the returned target must be a + * node with a NON-BLANK OWN LABEL, not the (often label-less) clickable container. + * [PurchaseUiDriver.clickFresh] re-finds its target live by + * `preferredOrDescendantLabel() == target.label && className && center±32` and only + * THEN climbs to the nearest clickable ancestor and clicks it (see + * `GoAutoAccessibilityService.clickFreshDetailed`). A label-less clickable + * FrameLayout's [SnapshotNode.label] is `""`, but its live `preferredOrDescendantLabel()` + * digs into a child and returns real text — so the re-find never matches and the click + * silently fails as `TARGET_NOT_FOUND` right after the irreversible boundary (task 594: + * "大促价，" bottom bar selected correctly, click never happened). Aiming at the + * labelled leaf instead — the same convention already used by + * `ImageSearchCandidatePolicy.clickTargetInside` — lets the existing climb-to-ancestor + * logic land the tap on the right container. + * + * Rule: within the recognized spec panel's container (#331 judgment, [PURCHASE_CONFIRMATION_PANEL_TYPES]), + * among visible/enabled/non-zero-size nodes with a non-blank own label that have a + * clickable ancestor (or are clickable themselves) of non-zero size, pick the one with + * the lowest bottom edge; ties go to the rightmost. The nearest clickable ancestor's + * subtree must not contain [SUBMIT_TARGET_BLOCKED_MARKERS]. [FINAL_SUBMIT_MARKERS] text + * match is diagnostic only, never required. */ private fun finalSubmitTargets(snapshot: UiSnapshot): List { fun hasArea(node: SnapshotNode) = node.bounds.width > 0 && node.bounds.height > 0 + val byPath = snapshot.nodes.associateBy { it.path } + fun clickableAncestor(node: SnapshotNode): SnapshotNode? { + var current: SnapshotNode? = node + while (current != null && !current.clickable) current = current.parentPath?.let(byPath::get) + return current + } val screen = PddScreenParser.parse(snapshot, PurchaseRehearsalExecutor.DEFAULT_COLLECTOR, "", null) if (screen.specPanelType !in PURCHASE_CONFIRMATION_PANEL_TYPES) { panelDiagnostic("bottomSubmit;outcome=not_panel;type=${screen.specPanelType}") @@ -796,15 +816,16 @@ class PurchaseLiveAutomation( panelDiagnostic("bottomSubmit;outcome=no_container;type=${screen.specPanelType}") return emptyList() } - // A full-sheet or full-body wrapper is sometimes clickable too (it can intercept - // touches); it is never the order button itself, so it must not win a bottom-edge - // tie against the real, compact bottom bar. Same guard family as the #331 - // ENTRY_ROW_MAX_HEIGHT_PERCENT check used for address/payment rows. + // A full-sheet or full-body wrapper container is sometimes clickable too (it can + // intercept touches); this bounds a *candidate leaf's own* size only as extra + // defense — a label-less wrapper is already excluded by requiring a non-blank own + // label below, but a stray, unrealistically large labelled node should not win either. val screenHeight = snapshot.nodes.filter { it.visible }.maxOfOrNull { it.bounds.bottom }?.coerceAtLeast(1) ?: 1 val candidates = snapshot.nodes.filter { node -> - node.visible && node.enabled && node.clickable && hasArea(node) && + node.visible && node.enabled && hasArea(node) && node.label.isNotEmpty() && node.bounds.height.toLong() * 100 <= screenHeight.toLong() * SUBMIT_ROW_MAX_HEIGHT_PERCENT && - inside(node.bounds, container) + inside(node.bounds, container) && + clickableAncestor(node)?.let(::hasArea) == true }.distinctBy { it.path } if (candidates.isEmpty()) { panelDiagnostic("bottomSubmit;outcome=no_candidates;type=${screen.specPanelType}") @@ -814,21 +835,27 @@ class PurchaseLiveAutomation( val bottomRow = candidates.filter { it.bounds.bottom == maxBottom } val tie = bottomRow.size > 1 val chosen = bottomRow.maxBy { it.bounds.right } - val subtreeLabels = (listOf(chosen.label) + subtreeDescendants(chosen, snapshot).map(SnapshotNode::label)) + val ancestor = clickableAncestor(chosen) + if (ancestor == null) { + // Unreachable given the candidate filter above, kept as an explicit safety net. + panelDiagnostic("bottomSubmit;outcome=no_clickable_ancestor;tie=${tie.diagFlag()}") + return emptyList() + } + val subtreeLabels = (listOf(ancestor.label) + subtreeDescendants(ancestor, snapshot).map(SnapshotNode::label)) .joinToString("") if (SUBMIT_TARGET_BLOCKED_MARKERS.any(subtreeLabels::contains)) { panelDiagnostic( - "bottomSubmit;outcome=payment_blocked;tie=${tie.diagFlag()};class=${chosen.className};" + - "w=${chosen.bounds.width};h=${chosen.bounds.height}", + "bottomSubmit;outcome=payment_blocked;tie=${tie.diagFlag()};class=${ancestor.className};" + + "w=${ancestor.bounds.width};h=${ancestor.bounds.height}", ) return emptyList() } val hasPrice = PRICE_PRESENCE.containsMatchIn(subtreeLabels) val legacyMarker = FINAL_SUBMIT_MARKERS.any { subtreeLabels == it || subtreeLabels.startsWith(it) } panelDiagnostic( - "bottomSubmit;outcome=ok;tie=${tie.diagFlag()};class=${chosen.className};" + - "w=${chosen.bounds.width};h=${chosen.bounds.height};price=${hasPrice.diagFlag()};" + - "legacyMarker=${legacyMarker.diagFlag()};type=${screen.specPanelType}", + "bottomSubmit;outcome=ok;tie=${tie.diagFlag()};leafClass=${chosen.className};" + + "ancestorClass=${ancestor.className};w=${ancestor.bounds.width};h=${ancestor.bounds.height};" + + "price=${hasPrice.diagFlag()};legacyMarker=${legacyMarker.diagFlag()};type=${screen.specPanelType}", ) return listOf(chosen) } diff --git a/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt index f2f0fe8..3b2aa3e 100644 --- a/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt +++ b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt @@ -495,16 +495,17 @@ class SpecPanelRecognitionTest { } @Test - fun `zero size bottom node is ignored in favor of the next non-zero clickable node`() { - // #335: the click target is the CLICKABLE node itself; its own accessibility text - // is what a real driver taps, so the button carries its own label directly here - // (no separate text child) to make the assertion check the real click target. + fun `zero size bottom node is ignored in favor of the next non-zero labelled node`() { + // The click target is the labelled LEAF node (#335 2nd revision); the button + // carries its own label directly here (no separate text child) to make the + // assertion check the real click target. val base = SpecPanelFixtures.sheet(Sheet(submit = "none")) val snapshot = base.copy( nodes = base.nodes + listOf( node("r/real-submit", "选择颜色分类及尺码后，提交订单", NodeBounds(0, 2135, 1080, 2200), clickable = true, parent = "r"), - // A taller, zero-*area* node below it (right==left) must never win: it has no clickable area. - node("r/decoy", "", NodeBounds(500, 2200, 500, 2216), clickable = true, parent = "r"), + // Zero-*area* (right==left) and lower on screen: must never win despite + // having its own non-blank label and a clickable ancestor. + node("r/decoy", "¥0.0", NodeBounds(500, 2200, 500, 2216), clickable = true, parent = "r"), ), ) val driver = StaticDriver(snapshot) @@ -549,10 +550,13 @@ class SpecPanelRecognitionTest { } @Test - fun `task 570 promo priced bottom row is the unique submit target`() { + fun `task 570 promo priced bottom row picks the bottom-right-most labelled leaf`() { // #335 origin: goods 8580 task 570, the bottom button's text was split across // three sibling text nodes ("大促价，" / "仅 ¥30.8" / "¥39.9") instead of one - // "提交订单"-prefixed label. This is the exact case the strategy targets. + // "提交订单"-prefixed label. This checks finalSubmitTargets' own choice at the + // SnapshotNode level (fast, no re-find simulation); the full live re-find + + // climb-to-clickable-ancestor path is covered separately below with + // ReFindingDriver, which is what actually caught the task 594 regression. val base = SpecPanelFixtures.sheet(Sheet(submit = "none")) val snapshot = base.copy( nodes = base.nodes + listOf( @@ -566,8 +570,37 @@ class SpecPanelRecognitionTest { PurchaseLiveAutomation(driver, pause = {}).submitOrderOnce() - assertEquals(listOf(""), driver.clicked) - assertEquals(1, driver.clicked.size) + // #335 2nd revision: the target is the labelled leaf (bottom edge tied across + // l1/l2/l3, rightmost wins), NOT the label-less "r/submit" container — a + // label-less target can never be re-found live (task 594). + assertEquals(listOf("¥39.9"), driver.clicked) + } + + @Test + fun `task 570 leaf target is re-found live and the click lands on the clickable container`() { + // #335 2nd revision (task 594 real-device failure): StaticDriver's clickFresh just + // records target.label directly, so it cannot catch a target that a REAL driver + // could never re-find. ReFindingDriver instead emulates + // GoAutoAccessibilityService.clickFreshDetailed's real semantics: find the live + // node by (label, className, center within ±32), then climb to the nearest live + // clickable ancestor and click THAT. + val base = SpecPanelFixtures.sheet(Sheet(screenBottom = 2400, submit = "none")) + val snapshot = base.copy( + nodes = base.nodes + listOf( + node("r/sheet/submit", "", NodeBounds(0, 2181, 1080, 2328), clickable = true, parent = "r/sheet"), + node("r/sheet/submit/l1", "大促价，", NodeBounds(364, 2225, 568, 2284), parent = "r/sheet/submit"), + node("r/sheet/submit/l2", "仅 ¥30.8", NodeBounds(568, 2225, 748, 2284), parent = "r/sheet/submit"), + node("r/sheet/submit/l3", "¥39.9", NodeBounds(748, 2222, 896, 2287), parent = "r/sheet/submit"), + ), + ) + val driver = ReFindingDriver(snapshot) + + PurchaseLiveAutomation(driver, pause = {}).submitOrderOnce() + + // finalSubmitTargets aims at "r/sheet/submit/l3" ("¥39.9", the bottom-right-most + // labelled leaf); the real click must land on its clickable ancestor, the + // FrameLayout "r/sheet/submit" — not on the leaf itself. + assertEquals("r/sheet/submit", driver.clickedPath) } @Test @@ -680,6 +713,40 @@ class SpecPanelRecognitionTest { override fun capture() = snapshot } + /** + * #335 (2nd revision): emulates `GoAutoAccessibilityService.clickFreshDetailed`'s real + * re-find semantics against a fixed, static "live" node list — match by (own label, + * className, center within ±32 — every fixture node here already carries its own + * label so this is equivalent to `preferredOrDescendantLabel()`), then climb to the + * nearest clickable ancestor and click THAT, exactly like the real accessibility + * service. Exposes the PATH of the node that actually received the click, so a test + * can assert the tap landed on the container, not the labelled leaf `finalSubmitTargets` + * aimed at. + */ + private class ReFindingDriver(private val liveSnapshot: UiSnapshot) : BaseDriver() { + var clickedPath: String? = null + override fun capture() = liveSnapshot + + override fun clickFresh(target: SnapshotNode): FreshActionResult { + val byPath = liveSnapshot.nodes.associateBy { it.path } + val matches = liveSnapshot.nodes.filter { candidate -> + candidate.label == target.label && candidate.className == target.className && + kotlin.math.abs(candidate.bounds.centerX - target.bounds.centerX) <= 32 && + kotlin.math.abs(candidate.bounds.centerY - target.bounds.centerY) <= 32 + } + if (matches.isEmpty()) return FreshActionResult.NOT_FOUND + if (matches.size != 1) return FreshActionResult.AMBIGUOUS + var node = matches.single() + if (target.clickable && !node.clickable) return FreshActionResult.NOT_FOUND + while (!node.clickable) { + node = node.parentPath?.let(byPath::get) ?: return FreshActionResult.FAILED + } + clickedPath = node.path + clicked += node.label + return FreshActionResult.SUCCESS + } + } + /** Product page first; after the spec-entry click it serves [panels] in order (the last repeats unless [cycle]). */ private class SpecEntryDriver(private val panels: List, private val cycle: Boolean = false) : BaseDriver() { private var opened = false