diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt index b628f41..2fdc8d5 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt @@ -777,15 +777,35 @@ class PurchaseLiveAutomation( } /** - * #335: the order button's own text keeps changing (促销文案、价格文案等），因此不再依赖 - * [FINAL_SUBMIT_MARKERS] 文字匹配判定必要条件；改为用户确认的策略——在已识别的规格面板 - * （沿用 #331 的 [PURCHASE_CONFIRMATION_PANEL_TYPES] 判定）内，取属于该面板、最下沿最低、 - * 可点击、可用、尺寸非零的节点。多个节点并列最下沿时取最右侧并在诊断中标记“并列”。节点自身 - * 或其子树文字命中 [SUBMIT_TARGET_BLOCKED_MARKERS] 时明确失败、不返回目标。旧的文字匹配 - * （[FINAL_SUBMIT_MARKERS]）仅作诊断证据，不再是必要条件。 + * #335 (2nd revision, task 594 real-device failure): the returned target must be a + * node with a NON-BLANK OWN LABEL, not the (often label-less) clickable container. + * [PurchaseUiDriver.clickFresh] re-finds its target live by + * `preferredOrDescendantLabel() == target.label && className && center±32` and only + * THEN climbs to the nearest clickable ancestor and clicks it (see + * `GoAutoAccessibilityService.clickFreshDetailed`). A label-less clickable + * FrameLayout's [SnapshotNode.label] is `""`, but its live `preferredOrDescendantLabel()` + * digs into a child and returns real text — so the re-find never matches and the click + * silently fails as `TARGET_NOT_FOUND` right after the irreversible boundary (task 594: + * "大促价，" bottom bar selected correctly, click never happened). Aiming at the + * labelled leaf instead — the same convention already used by + * `ImageSearchCandidatePolicy.clickTargetInside` — lets the existing climb-to-ancestor + * logic land the tap on the right container. + * + * Rule: within the recognized spec panel's container (#331 judgment, [PURCHASE_CONFIRMATION_PANEL_TYPES]), + * among visible/enabled/non-zero-size nodes with a non-blank own label that have a + * clickable ancestor (or are clickable themselves) of non-zero size, pick the one with + * the lowest bottom edge; ties go to the rightmost. The nearest clickable ancestor's + * subtree must not contain [SUBMIT_TARGET_BLOCKED_MARKERS]. [FINAL_SUBMIT_MARKERS] text + * match is diagnostic only, never required. */ private fun finalSubmitTargets(snapshot: UiSnapshot): List { fun hasArea(node: SnapshotNode) = node.bounds.width > 0 && node.bounds.height > 0 + val byPath = snapshot.nodes.associateBy { it.path } + fun clickableAncestor(node: SnapshotNode): SnapshotNode? { + var current: SnapshotNode? = node + while (current != null && !current.clickable) current = current.parentPath?.let(byPath::get) + return current + } val screen = PddScreenParser.parse(snapshot, PurchaseRehearsalExecutor.DEFAULT_COLLECTOR, "", null) if (screen.specPanelType !in PURCHASE_CONFIRMATION_PANEL_TYPES) { panelDiagnostic("bottomSubmit;outcome=not_panel;type=${screen.specPanelType}") @@ -796,15 +816,16 @@ class PurchaseLiveAutomation( panelDiagnostic("bottomSubmit;outcome=no_container;type=${screen.specPanelType}") return emptyList() } - // A full-sheet or full-body wrapper is sometimes clickable too (it can intercept - // touches); it is never the order button itself, so it must not win a bottom-edge - // tie against the real, compact bottom bar. Same guard family as the #331 - // ENTRY_ROW_MAX_HEIGHT_PERCENT check used for address/payment rows. + // A full-sheet or full-body wrapper container is sometimes clickable too (it can + // intercept touches); this bounds a *candidate leaf's own* size only as extra + // defense — a label-less wrapper is already excluded by requiring a non-blank own + // label below, but a stray, unrealistically large labelled node should not win either. val screenHeight = snapshot.nodes.filter { it.visible }.maxOfOrNull { it.bounds.bottom }?.coerceAtLeast(1) ?: 1 val candidates = snapshot.nodes.filter { node -> - node.visible && node.enabled && node.clickable && hasArea(node) && + node.visible && node.enabled && hasArea(node) && node.label.isNotEmpty() && node.bounds.height.toLong() * 100 <= screenHeight.toLong() * SUBMIT_ROW_MAX_HEIGHT_PERCENT && - inside(node.bounds, container) + inside(node.bounds, container) && + clickableAncestor(node)?.let(::hasArea) == true }.distinctBy { it.path } if (candidates.isEmpty()) { panelDiagnostic("bottomSubmit;outcome=no_candidates;type=${screen.specPanelType}") @@ -814,21 +835,27 @@ class PurchaseLiveAutomation( val bottomRow = candidates.filter { it.bounds.bottom == maxBottom } val tie = bottomRow.size > 1 val chosen = bottomRow.maxBy { it.bounds.right } - val subtreeLabels = (listOf(chosen.label) + subtreeDescendants(chosen, snapshot).map(SnapshotNode::label)) + val ancestor = clickableAncestor(chosen) + if (ancestor == null) { + // Unreachable given the candidate filter above, kept as an explicit safety net. + panelDiagnostic("bottomSubmit;outcome=no_clickable_ancestor;tie=${tie.diagFlag()}") + return emptyList() + } + val subtreeLabels = (listOf(ancestor.label) + subtreeDescendants(ancestor, snapshot).map(SnapshotNode::label)) .joinToString("") if (SUBMIT_TARGET_BLOCKED_MARKERS.any(subtreeLabels::contains)) { panelDiagnostic( - "bottomSubmit;outcome=payment_blocked;tie=${tie.diagFlag()};class=${chosen.className};" + - "w=${chosen.bounds.width};h=${chosen.bounds.height}", + "bottomSubmit;outcome=payment_blocked;tie=${tie.diagFlag()};class=${ancestor.className};" + + "w=${ancestor.bounds.width};h=${ancestor.bounds.height}", ) return emptyList() } val hasPrice = PRICE_PRESENCE.containsMatchIn(subtreeLabels) val legacyMarker = FINAL_SUBMIT_MARKERS.any { subtreeLabels == it || subtreeLabels.startsWith(it) } panelDiagnostic( - "bottomSubmit;outcome=ok;tie=${tie.diagFlag()};class=${chosen.className};" + - "w=${chosen.bounds.width};h=${chosen.bounds.height};price=${hasPrice.diagFlag()};" + - "legacyMarker=${legacyMarker.diagFlag()};type=${screen.specPanelType}", + "bottomSubmit;outcome=ok;tie=${tie.diagFlag()};leafClass=${chosen.className};" + + "ancestorClass=${ancestor.className};w=${ancestor.bounds.width};h=${ancestor.bounds.height};" + + "price=${hasPrice.diagFlag()};legacyMarker=${legacyMarker.diagFlag()};type=${screen.specPanelType}", ) return listOf(chosen) } diff --git a/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt index f2f0fe8..3b2aa3e 100644 --- a/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt +++ b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt @@ -495,16 +495,17 @@ class SpecPanelRecognitionTest { } @Test - fun `zero size bottom node is ignored in favor of the next non-zero clickable node`() { - // #335: the click target is the CLICKABLE node itself; its own accessibility text - // is what a real driver taps, so the button carries its own label directly here - // (no separate text child) to make the assertion check the real click target. + fun `zero size bottom node is ignored in favor of the next non-zero labelled node`() { + // The click target is the labelled LEAF node (#335 2nd revision); the button + // carries its own label directly here (no separate text child) to make the + // assertion check the real click target. val base = SpecPanelFixtures.sheet(Sheet(submit = "none")) val snapshot = base.copy( nodes = base.nodes + listOf( node("r/real-submit", "选择颜色分类及尺码后，提交订单", NodeBounds(0, 2135, 1080, 2200), clickable = true, parent = "r"), - // A taller, zero-*area* node below it (right==left) must never win: it has no clickable area. - node("r/decoy", "", NodeBounds(500, 2200, 500, 2216), clickable = true, parent = "r"), + // Zero-*area* (right==left) and lower on screen: must never win despite + // having its own non-blank label and a clickable ancestor. + node("r/decoy", "¥0.0", NodeBounds(500, 2200, 500, 2216), clickable = true, parent = "r"), ), ) val driver = StaticDriver(snapshot) @@ -549,10 +550,13 @@ class SpecPanelRecognitionTest { } @Test - fun `task 570 promo priced bottom row is the unique submit target`() { + fun `task 570 promo priced bottom row picks the bottom-right-most labelled leaf`() { // #335 origin: goods 8580 task 570, the bottom button's text was split across // three sibling text nodes ("大促价，" / "仅 ¥30.8" / "¥39.9") instead of one - // "提交订单"-prefixed label. This is the exact case the strategy targets. + // "提交订单"-prefixed label. This checks finalSubmitTargets' own choice at the + // SnapshotNode level (fast, no re-find simulation); the full live re-find + + // climb-to-clickable-ancestor path is covered separately below with + // ReFindingDriver, which is what actually caught the task 594 regression. val base = SpecPanelFixtures.sheet(Sheet(submit = "none")) val snapshot = base.copy( nodes = base.nodes + listOf( @@ -566,8 +570,37 @@ class SpecPanelRecognitionTest { PurchaseLiveAutomation(driver, pause = {}).submitOrderOnce() - assertEquals(listOf(""), driver.clicked) - assertEquals(1, driver.clicked.size) + // #335 2nd revision: the target is the labelled leaf (bottom edge tied across + // l1/l2/l3, rightmost wins), NOT the label-less "r/submit" container — a + // label-less target can never be re-found live (task 594). + assertEquals(listOf("¥39.9"), driver.clicked) + } + + @Test + fun `task 570 leaf target is re-found live and the click lands on the clickable container`() { + // #335 2nd revision (task 594 real-device failure): StaticDriver's clickFresh just + // records target.label directly, so it cannot catch a target that a REAL driver + // could never re-find. ReFindingDriver instead emulates + // GoAutoAccessibilityService.clickFreshDetailed's real semantics: find the live + // node by (label, className, center within ±32), then climb to the nearest live + // clickable ancestor and click THAT. + val base = SpecPanelFixtures.sheet(Sheet(screenBottom = 2400, submit = "none")) + val snapshot = base.copy( + nodes = base.nodes + listOf( + node("r/sheet/submit", "", NodeBounds(0, 2181, 1080, 2328), clickable = true, parent = "r/sheet"), + node("r/sheet/submit/l1", "大促价，", NodeBounds(364, 2225, 568, 2284), parent = "r/sheet/submit"), + node("r/sheet/submit/l2", "仅 ¥30.8", NodeBounds(568, 2225, 748, 2284), parent = "r/sheet/submit"), + node("r/sheet/submit/l3", "¥39.9", NodeBounds(748, 2222, 896, 2287), parent = "r/sheet/submit"), + ), + ) + val driver = ReFindingDriver(snapshot) + + PurchaseLiveAutomation(driver, pause = {}).submitOrderOnce() + + // finalSubmitTargets aims at "r/sheet/submit/l3" ("¥39.9", the bottom-right-most + // labelled leaf); the real click must land on its clickable ancestor, the + // FrameLayout "r/sheet/submit" — not on the leaf itself. + assertEquals("r/sheet/submit", driver.clickedPath) } @Test @@ -680,6 +713,40 @@ class SpecPanelRecognitionTest { override fun capture() = snapshot } + /** + * #335 (2nd revision): emulates `GoAutoAccessibilityService.clickFreshDetailed`'s real + * re-find semantics against a fixed, static "live" node list — match by (own label, + * className, center within ±32 — every fixture node here already carries its own + * label so this is equivalent to `preferredOrDescendantLabel()`), then climb to the + * nearest clickable ancestor and click THAT, exactly like the real accessibility + * service. Exposes the PATH of the node that actually received the click, so a test + * can assert the tap landed on the container, not the labelled leaf `finalSubmitTargets` + * aimed at. + */ + private class ReFindingDriver(private val liveSnapshot: UiSnapshot) : BaseDriver() { + var clickedPath: String? = null + override fun capture() = liveSnapshot + + override fun clickFresh(target: SnapshotNode): FreshActionResult { + val byPath = liveSnapshot.nodes.associateBy { it.path } + val matches = liveSnapshot.nodes.filter { candidate -> + candidate.label == target.label && candidate.className == target.className && + kotlin.math.abs(candidate.bounds.centerX - target.bounds.centerX) <= 32 && + kotlin.math.abs(candidate.bounds.centerY - target.bounds.centerY) <= 32 + } + if (matches.isEmpty()) return FreshActionResult.NOT_FOUND + if (matches.size != 1) return FreshActionResult.AMBIGUOUS + var node = matches.single() + if (target.clickable && !node.clickable) return FreshActionResult.NOT_FOUND + while (!node.clickable) { + node = node.parentPath?.let(byPath::get) ?: return FreshActionResult.FAILED + } + clickedPath = node.path + clicked += node.label + return FreshActionResult.SUCCESS + } + } + /** Product page first; after the spec-entry click it serves [panels] in order (the last repeats unless [cycle]). */ private class SpecEntryDriver(private val panels: List, private val cycle: Boolean = false) : BaseDriver() { private var opened = false