From 99f238494ecf907ec38a463d69b1dd6fe22b6fbc Mon Sep 17 00:00:00 2001 From: QiuSW <105186638@qq.com> Date: Mon, 21 Sep 2026 18:30:48 +0800 Subject: [PATCH] fix(android): recognize spec panel by address, payment and quantity input (#331) - PddScreenParser: add REQUIRED_EVIDENCE fallback type after all existing branches; requires address entry (masked phone in a nearby clickable row), payment entry (specPanel.paymentAreaAliases in a nearby clickable row) and one enabled quantity EditText, plus at least one auxiliary signal (options, summary, close, +/- or submit action). +/- no longer required on this path; existing branches unchanged. - Collapse nested clickable nodes of one labelled option block into one spec value (task 535), no badge-text stripping. - Purchase spec-entry wait and post address-save wait: bounded 5000 ms, fallback panels need two identical structure samples, no Back while waiting, explicit failure on timeout; boolean/count diagnostics only. - Final submit target additionally rejects zero-size labels/containers. - Diagnostics: evidence flags in panel evidence and spec-panel entry event; selection-unconfirmed diagnostic lists parsed option texts and flags. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NTDbDcwbDw1TSAcE6wfh2F --- .../automation/PddProductDetailCollector.kt | 174 ++++++- .../automation/PurchaseLiveAutomation.kt | 107 +++- .../automation/PurchaseRehearsalExecutor.kt | 61 ++- .../agent/PurchaseRehearsalExecutorTest.kt | 3 +- .../ilapage/goauto/agent/SpecPanelFixtures.kt | 267 ++++++++++ .../goauto/agent/SpecPanelRecognitionTest.kt | 486 ++++++++++++++++++ 6 files changed, 1069 insertions(+), 29 deletions(-) create mode 100644 android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelFixtures.kt create mode 100644 android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PddProductDetailCollector.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PddProductDetailCollector.kt index 867c878..6b62821 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PddProductDetailCollector.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PddProductDetailCollector.kt @@ -90,7 +90,28 @@ data class VisibleSpecValue( val rawText: String = text, ) data class VisibleDimension(val key: String, val name: String, val values: List) -enum class SpecPanelType { UNKNOWN, NORMAL_SCROLLABLE, NON_SCROLLABLE_CONFIRMATION, QUICK_CONFIRMATION, ORDER_CONFIRMATION } +enum class SpecPanelType { + UNKNOWN, + NORMAL_SCROLLABLE, + NON_SCROLLABLE_CONFIRMATION, + QUICK_CONFIRMATION, + ORDER_CONFIRMATION, + /** + * #331 fallback: only reached when no stronger panel type matched. It + * requires an address entry, a payment-method entry and one quantity input + * box, plus at least one auxiliary signal. It is recognition only and never + * relaxes any click target (the final order-submit click stays strict). + */ + REQUIRED_EVIDENCE, +} + +/** #331 bounded wait for a spec panel that is still loading after an entry click or an address save. */ +object SpecPanelStabilityPolicy { + /** Upper bound for waiting until the required evidence is present and the structure is stable. */ + const val WAIT_BOUND_MS = 5_000L + /** Sampling interval used after an address save returns to the purchase panel. */ + const val ADDRESS_SAVE_POLL_MS = 200L +} object PddSoldOutRecoveryDefaults { const val EXACT_TEXT = "商品已售罄" @@ -134,7 +155,24 @@ data class ParsedPddScreen( val hasCloseControl: Boolean = false, val hasPaymentArea: Boolean = false, val purchaseContextMatched: Boolean = false, + // #331 evidence flags. Presence only: address text is never kept here. + val hasAddressEntry: Boolean = false, + val hasPaymentEntry: Boolean = false, + val hasQuantityInput: Boolean = false, + val hasQuantityAdjustControls: Boolean = false, + val quantityInputCount: Int = 0, ) { + /** The three #331 required spec-panel items are all visible. */ + val hasRequiredPanelEvidence: Boolean get() = hasAddressEntry && hasPaymentEntry && hasQuantityInput + + /** Boolean/count-only evidence string for diagnostics; contains no page text. */ + fun requiredEvidenceSummary(): String = + "addr=${hasAddressEntry.flag()};pay=${hasPaymentEntry.flag()};qin=$quantityInputCount;" + + "adj=${hasQuantityAdjustControls.flag()};summary=${hasSelectionSummary.flag()};" + + "close=${hasCloseControl.flag()};submit=${hasOrderSubmitAction.flag()}" + + private fun Boolean.flag(): Int = if (this) 1 else 0 + fun isTransientSoldOut( exactText: String, fallbackTopText: String = "相似商品", @@ -186,6 +224,13 @@ object PddScreenParser { // purchase/order/payment controls must never become collection click targets. private val nonConfigurableClickDenylist = listOf("提交订单", "确认订单", "支付", "付款") + /** Masked CN mobile number shown in the purchase address row, e.g. 138****5678. Presence only. */ + val MASKED_PHONE_PATTERN = Regex("(? @@ -203,6 +248,8 @@ object PddScreenParser { } } val labels = visible.map(SnapshotNode::label) + val visibleByPath = visible.associateBy(SnapshotNode::path) + val sourceByPath = visibleNodes.associateBy(SnapshotNode::path) val textAliases = config.textAliases val problem = PddPageClassifier.classify(snapshot.packageName, snapshot.activityName, labels) val compactLabels = labels.map { it.replace(" ", "") } @@ -240,7 +287,12 @@ object PddScreenParser { val decreaseControls = visible.filter { it.clickable && it.label.replace(" ", "") in textAliases.specPanel.quantityDecreaseAliases } val increaseControls = visible.filter { it.clickable && it.label.replace(" ", "") in textAliases.specPanel.quantityIncreaseAliases } val hasQuantityControls = quantityInputs.size == 1 && decreaseControls.size == 1 && increaseControls.size == 1 + // #331: the unique input box is required; the +/- buttons are only auxiliary. + val hasQuantityInput = quantityInputs.size == 1 && quantityInputs.single().enabled + val hasQuantityAdjustControls = decreaseControls.size == 1 && increaseControls.size == 1 val hasPaymentArea = compactLabels.any { label -> textAliases.specPanel.paymentAreaAliases.any(label::contains) } + val hasAddressEntry = hasAddressEntry(visibleNodes, screenHeight) + val hasPaymentEntry = hasPaymentEntry(visibleNodes, screenHeight, textAliases.specPanel.paymentAreaAliases) val hasQuickBuy = visible.any { node -> node.label.replace(" ", "").let { compact -> textAliases.specPanel.quickBuyAliases.any(compact::contains) } && node.bounds.centerY.toDouble() >= screenHeight * 0.75 @@ -276,27 +328,50 @@ object PddScreenParser { headings.forEachIndexed { index, heading -> val lower = headings.getOrNull(index + 1)?.bounds?.top ?: Int.MAX_VALUE val dimensionKey = dimensionKey(heading.label, config, any { it.key == "color" }) - val values = panelVisible.asSequence() - .filter { it.clickable && it.bounds.top >= heading.bounds.bottom && it.bounds.bottom <= lower } - .filter { it.bounds.width > 0 && it.bounds.height > 0 && it.label.length <= 80 } + fun inValueRegion(node: SnapshotNode) = + node.bounds.top >= heading.bounds.bottom && node.bounds.bottom <= lower && + node.bounds.width > 0 && node.bounds.height > 0 && node.label.length <= 80 + val candidates = panelVisible.asSequence() + .filter { it.clickable && inValueRegion(it) } .filterNot { isExactHeadingLabel(it.label, config) } .filterNot { node -> isExcludedOptionLabel(node.label) || descendants(node, visibleNodes).any { descendant -> isExcludedOptionLabel(descendant.label) } } - .map { - val rawText = it.label + .toList() + // #331/task 535: one option block may expose several clickable + // nodes (outer block, image, inner text with a badge such as + // "零差评"). Nodes nested in the same labelled option block are + // collapsed into one value named by the outer block. The click + // node is still chosen by safeOptionRank among the members. + val values = candidates + .groupBy { candidate -> + optionBlock(candidate, candidates, visibleByPath, sourceByPath, ::inValueRegion)?.path ?: candidate.path + } + .map { (blockPath, members) -> + val block = visibleByPath[blockPath] + val chosen = members.minBy { option -> safeOptionRank(option, visibleNodes) } + val selected = members.any(SnapshotNode::selected) || block?.selected == true + val checked = members.any(SnapshotNode::checked) || block?.checked == true + Triple(chosen.copy(selected = selected, checked = checked), block?.label ?: chosen.label, members) + } + .asSequence() + .map { (option, blockLabel, members) -> + val rawText = blockLabel val normalizedText = when (dimensionKey) { "color" -> SpecValueNormalizer.normalizeColor(rawText) "size" -> SpecValueNormalizer.normalizeSize(rawText) else -> rawText } - val stateText = (listOf(it.label) + descendants(it, visibleNodes).map(SnapshotNode::label)).joinToString(" ") + val stateText = (listOf(option.label) + descendants(option, visibleNodes).map(SnapshotNode::label)).joinToString(" ") VisibleSpecValue( normalizedText, - it.enabled && !stateText.containsUnavailableWord(), - it, - if (dimensionKey == "color") colorImageBoundsForOption(it, visibleNodes) else null, + option.enabled && !stateText.containsUnavailableWord(), + option, + if (dimensionKey == "color") { + (listOf(option) + members.filter { it.path != option.path }) + .firstNotNullOfOrNull { colorImageBoundsForOption(it, visibleNodes) } + } else null, rawText, ) } @@ -339,6 +414,14 @@ object PddScreenParser { panelScrollable == null && headings.size >= 2 && dimensions.size >= 2 && dimensions.sumOf { it.values.size } >= 2 && hasQuantityControls && hasOrderSubmitAction -> SpecPanelType.NON_SCROLLABLE_CONFIRMATION + // #331 fallback, evaluated only after every stronger branch failed. + // Required: address entry + payment entry + one enabled quantity + // input. Auxiliary (at least one): grouped options, selection + // summary, close control, +/- controls or the lower submit action. + hasAddressEntry && hasPaymentEntry && hasQuantityInput && ( + dimensions.any { it.values.isNotEmpty() } || hasSelectionSummary || hasClose || + hasQuantityAdjustControls || hasOrderSubmitAction + ) -> SpecPanelType.REQUIRED_EVIDENCE else -> SpecPanelType.UNKNOWN } val panelOpen = specPanelType != SpecPanelType.UNKNOWN @@ -439,6 +522,11 @@ object PddScreenParser { purchaseContextMatched = continuedPurchasePanel, hasQuantityControls = hasQuantityControls, hasOrderSubmitAction = hasOrderSubmitAction, + hasAddressEntry = hasAddressEntry, + hasPaymentEntry = hasPaymentEntry, + hasQuantityInput = hasQuantityInput, + hasQuantityAdjustControls = hasQuantityAdjustControls, + quantityInputCount = quantityInputs.size, explicitSpecEntryCount = explicitSpecEntries.size, nestedSpecEntryCount = nestedSpecEntries.size, bottomPurchaseEntryCount = bottomSpecEntries.size, @@ -453,6 +541,70 @@ object PddScreenParser { return compact in excludedExactOptionLabels || excludedOptionWords.any { compact.contains(it) } } + /** + * Outermost labelled clickable ancestor inside the value region whose label + * prefixes every candidate nested in it. Rows that contain several distinct + * options never qualify because their members do not share its label. + */ + private fun optionBlock( + candidate: SnapshotNode, + candidates: List, + visibleByPath: Map, + sourceByPath: Map, + inValueRegion: (SnapshotNode) -> Boolean, + ): SnapshotNode? { + fun compact(value: String) = value.replace(Regex("\\s+"), "") + var block: SnapshotNode? = null + var parentPath = candidate.parentPath + while (parentPath != null) { + val raw = sourceByPath[parentPath] ?: break + val resolved = visibleByPath[parentPath] + // Only a block with its own label names the option; a blank row + // container resolved from its first child never merges options. + if (raw.clickable && raw.label.isNotBlank() && resolved != null && inValueRegion(resolved)) { + val outer = compact(resolved.label) + val members = candidates.filter { it.path == resolved.path || it.path.startsWith("${resolved.path}/") } + if (outer.isEmpty() || members.any { !compact(it.label).startsWith(outer) }) break + block = resolved + } + parentPath = raw.parentPath + } + return block + } + + private fun hasClickableEntryRow(node: SnapshotNode, sourceByPath: Map, screenHeight: Int): Boolean { + var current: SnapshotNode? = node + var depth = 0 + while (current != null && depth <= ENTRY_CLICKABLE_ANCESTOR_MAX_DEPTH) { + if (current.clickable) { + return current.visible && current.enabled && current.bounds.width > 0 && current.bounds.height > 0 && + (screenHeight <= 0 || current.bounds.height * 100 <= screenHeight * ENTRY_ROW_MAX_HEIGHT_PERCENT) + } + current = current.parentPath?.let(sourceByPath::get) + depth++ + } + return false + } + + /** Address row presence: a masked phone inside a nearby clickable row. The address text is never read out. */ + private fun hasAddressEntry(source: List, screenHeight: Int): Boolean { + val byPath = source.associateBy(SnapshotNode::path) + return source.any { node -> + node.visible && node.bounds.width > 0 && node.bounds.height > 0 && + MASKED_PHONE_PATTERN.containsMatchIn(node.label) && hasClickableEntryRow(node, byPath, screenHeight) + } + } + + /** Payment-method entry presence via the configurable payment aliases inside a nearby clickable row. */ + private fun hasPaymentEntry(source: List, screenHeight: Int, aliases: List): Boolean { + val byPath = source.associateBy(SnapshotNode::path) + return source.any { node -> + node.visible && node.bounds.width > 0 && node.bounds.height > 0 && + node.label.replace(Regex("\\s+"), "").let { compact -> aliases.any(compact::contains) } && + hasClickableEntryRow(node, byPath, screenHeight) + } + } + private fun orderSubmitCandidatesAncestorFree(candidate: SnapshotNode, source: List, aliases: List): Boolean { val candidatePrefix = "${candidate.path}/" return source.none { node -> @@ -1572,6 +1724,8 @@ class PddProductDetailCollector( "type=${it.specPanelType.name};scroll=${it.panelScrollableCount};head=${it.panelHeadingCount};" + "option=${it.panelOptionCount};summary=${it.hasSelectionSummary.toInt()};" + "quantity=${it.hasQuantityControls.toInt()};submit=${it.hasOrderSubmitAction.toInt()};" + + "addr=${it.hasAddressEntry.toInt()};pay=${it.hasPaymentEntry.toInt()};qin=${it.quantityInputCount};" + + "adj=${it.hasQuantityAdjustControls.toInt()};close=${it.hasCloseControl.toInt()};" + "changed=${screenChanged.toInt()};" + "recovery=${when { recoverySucceeded -> "success" diff --git a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt index 14aed4d..2f0376a 100644 --- a/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt +++ b/android/app/src/main/java/cn/ilapage/goauto/agent/automation/PurchaseLiveAutomation.kt @@ -40,6 +40,8 @@ class PurchaseLiveException(val code: String, message: String, val actualUnitPri class PurchaseLiveAutomation( private val driver: PurchaseUiDriver, private val pause: (Long) -> Unit = Thread::sleep, + /** Boolean/count-only diagnostics; never receives address text. */ + private val panelDiagnostic: (String) -> Unit = {}, ) { private var submitAttempted = false var lastOrderReadFailure: PurchaseOrderReadFailure? = null @@ -523,7 +525,7 @@ class PurchaseLiveAutomation( val save = uniqueClickable(stable, stable.nodes.filter { it.visible && it.enabled && it.label == "保存" }) if (save.size != 1) fail("PURCHASE_ADDRESS_UPDATE_FAILED", "地址保存按钮不唯一，未创建订单") click(save.single(), "保存地址") - val savedEvidence = waitForStableAddressEditorExit() + val savedEvidence = waitForStableAddressEditorExit(expected, suffix) if (!hasFinalSavedAddressEvidence(savedEvidence, expected, suffix)) { if (isPurchaseConfirmationPanel(savedEvidence)) { restoreFinalEvidenceInCurrentPanel(savedEvidence, expected, suffix) @@ -540,11 +542,7 @@ class PurchaseLiveAutomation( private fun isPurchaseConfirmationPanel(snapshot: UiSnapshot): Boolean { if (snapshot.packageName != PDD_PACKAGE || shippingAddressEditors(snapshot).isNotEmpty()) return false val screen = PddScreenParser.parse(snapshot, PurchaseRehearsalExecutor.DEFAULT_COLLECTOR, "", null) - return screen.specPanelType in setOf( - SpecPanelType.NORMAL_SCROLLABLE, - SpecPanelType.NON_SCROLLABLE_CONFIRMATION, - SpecPanelType.ORDER_CONFIRMATION, - ) + return screen.specPanelType in PURCHASE_CONFIRMATION_PANEL_TYPES } private fun restoreFinalEvidenceInCurrentPanel( @@ -598,14 +596,14 @@ class PurchaseLiveAutomation( private fun hasFinalSavedAddressEvidence(snapshot: UiSnapshot, expected: String, suffix: String): Boolean = hasSavedAddressEvidence(snapshot, expected, suffix) && finalSubmitTargets(snapshot).size == 1 - private fun waitForStableAddressEditorExit(): UiSnapshot { + private fun waitForStableAddressEditorExit(expected: String, suffix: String): UiSnapshot { var consecutiveExits = 0 repeat(50) { val snapshot = driver.capture() pageProblem(snapshot) if (snapshot.packageName == PDD_PACKAGE && shippingAddressEditors(snapshot).isEmpty()) { consecutiveExits++ - if (consecutiveExits >= 2) return snapshot + if (consecutiveExits >= 2) return waitForSettledPanelAfterAddressSave(snapshot, expected, suffix) } else { consecutiveExits = 0 } @@ -614,6 +612,68 @@ class PurchaseLiveAutomation( fail("PURCHASE_ADDRESS_SAVE_TIMEOUT", "地址保存超时，未创建订单") } + /** + * #331: after the editor closes PDD may still be rebuilding the purchase + * sheet. Accept immediately when the final evidence or a stronger panel type + * is already present. A sheet recognized only through the required-evidence + * fallback must also be structurally stable for two consecutive samples. A + * stable page without any purchase-sheet marker (quantity input or payment + * entry), e.g. the address list, keeps the existing caller path. A partially + * loaded sheet is waited for up to [SpecPanelStabilityPolicy.WAIT_BOUND_MS] + * without pressing Back and then fails explicitly. + */ + private fun waitForSettledPanelAfterAddressSave(initial: UiSnapshot, expected: String, suffix: String): UiSnapshot { + var snapshot = initial + var previousSignature: String? = null + var waitedMs = 0L + var samples = 0 + while (true) { + samples++ + val screen = PddScreenParser.parse(snapshot, PurchaseRehearsalExecutor.DEFAULT_COLLECTOR, "", null) + val editorGone = snapshot.packageName == PDD_PACKAGE && shippingAddressEditors(snapshot).isEmpty() + val signature = structureSignature(snapshot) + val stable = editorGone && signature == previousSignature + val outcome = when { + editorGone && hasFinalSavedAddressEvidence(snapshot, expected, suffix) -> "final_evidence" + editorGone && screen.specPanelType in STRONG_PURCHASE_PANEL_TYPES -> "recognized" + stable && screen.specPanelType == SpecPanelType.REQUIRED_EVIDENCE -> "stable_required" + stable && !screen.hasQuantityInput && !screen.hasPaymentEntry -> "non_panel" + else -> null + } + if (outcome != null) { + addressSaveDiagnostic(outcome, waitedMs, samples, screen) + return snapshot + } + if (waitedMs >= SpecPanelStabilityPolicy.WAIT_BOUND_MS) { + addressSaveDiagnostic("timeout", waitedMs, samples, screen) + fail( + "PURCHASE_ADDRESS_SAVE_TIMEOUT", + "地址保存后规格面板未完整加载，未按返回键，未创建订单 [waitedMs=$waitedMs;samples=$samples;${screen.requiredEvidenceSummary()}]", + ) + } + previousSignature = signature + pause(SpecPanelStabilityPolicy.ADDRESS_SAVE_POLL_MS) + waitedMs += SpecPanelStabilityPolicy.ADDRESS_SAVE_POLL_MS + snapshot = driver.capture() + pageProblem(snapshot) + } + } + + private fun addressSaveDiagnostic(outcome: String, waitedMs: Long, samples: Int, screen: ParsedPddScreen) { + panelDiagnostic( + "addressSaveWait;outcome=$outcome;waitedMs=$waitedMs;samples=$samples;backPressed=false;" + + "type=${screen.specPanelType};${screen.requiredEvidenceSummary()}", + ) + } + + /** In-memory structure only (path, class, bounds, flags); no text is included. */ + private fun structureSignature(snapshot: UiSnapshot): String = snapshot.nodes + .filter { it.visible } + .joinToString("|") { node -> + listOf(node.path, node.className.orEmpty(), node.bounds.left, node.bounds.top, node.bounds.right, node.bounds.bottom, node.clickable, node.scrollable, node.enabled) + .joinToString(":") + } + private fun normalizeAddressText(value: String): String = value.filterNot(Char::isWhitespace) private fun hasExactTaskSuffix(value: String, suffix: String): Boolean = @@ -716,10 +776,25 @@ class PurchaseLiveAutomation( pause(500) } - private fun finalSubmitTargets(snapshot: UiSnapshot): List = uniqueClickable( - snapshot, - snapshot.nodes.filter { node -> node.visible && node.enabled && FINAL_SUBMIT_MARKERS.any { node.label == it || node.label.startsWith(it) } }, - ) + private fun finalSubmitTargets(snapshot: UiSnapshot): List { + val byPath = snapshot.nodes.associateBy { it.path } + fun hasArea(node: SnapshotNode) = node.bounds.width > 0 && node.bounds.height > 0 + // #331: recognition may treat the submit action as auxiliary, but the + // final click target stays strict. A zero-size label or a zero-size + // clickable container is never a click target. + fun clickableAncestorHasArea(node: SnapshotNode): Boolean { + var current: SnapshotNode? = node + while (current != null && !current.clickable) current = current.parentPath?.let(byPath::get) + return current != null && hasArea(current) + } + return uniqueClickable( + snapshot, + snapshot.nodes.filter { node -> + node.visible && node.enabled && hasArea(node) && clickableAncestorHasArea(node) && + FINAL_SUBMIT_MARKERS.any { node.label == it || node.label.startsWith(it) } + }, + ) + } private fun orderConfirmationReady(snapshot: UiSnapshot): Boolean { if (snapshot.packageName != PDD_PACKAGE) return false @@ -793,7 +868,13 @@ class PurchaseLiveAutomation( private companion object { const val PDD_PACKAGE = "com.xunmeng.pinduoduo" const val WECHAT_PACKAGE = "com.tencent.mm" - val MASKED_PHONE = Regex("(?() - val live = PurchaseLiveAutomation(driver, pause) + val live = PurchaseLiveAutomation(driver, pause, panelDiagnostic) for (action in rule.actions) { // The immediate phase-two handoff can reuse the PDD page retained by // spec_probe. A later manual retry may start from Agent (or another @@ -446,17 +446,51 @@ class PurchaseRehearsalExecutor( private fun waitForSpecPanel(input: PurchaseExecutionInput, beforeSignature: List): SpecPanelWait { var last = currentScreen(input) var changed = false - repeat(SPEC_POST_CLICK_VERIFY_POLLS) { + var polls = 0 + var waitedMs = 0L + var requiredEvidenceSeen = false + var previousStructure: List? = null + fun waitDiagnostic(outcome: String) { + if (requiredEvidenceSeen) { + panelDiagnostic( + "specPanelWait;outcome=$outcome;waitedMs=$waitedMs;samples=$polls;backPressed=false;" + + last.requiredEvidenceSummary(), + ) + } + } + while (true) { last = currentScreen(input) panelDiagnostic(panelEvidence(last)) if (last.reviewPageOpen) { return SpecPanelWait(last, false, true, leaveUnexpectedReviewPage(input)) } last.problem?.let { return SpecPanelWait(last, false, true, failure(it.code, it.message)) } - if (last.specPanelOpen) return SpecPanelWait(last, true, true) - changed = changed || specActionSignature(last) != beforeSignature + val structure = specActionSignature(last) + // Stronger panel types keep their immediate acceptance. The #331 + // fallback type additionally needs two identical consecutive + // structures so a still-loading sheet is not acted on too early. + if (last.specPanelOpen && (last.specPanelType != SpecPanelType.REQUIRED_EVIDENCE || structure == previousStructure)) { + waitDiagnostic("opened") + return SpecPanelWait(last, true, true) + } + changed = changed || structure != beforeSignature + requiredEvidenceSeen = requiredEvidenceSeen || + last.hasAddressEntry || last.hasPaymentEntry || last.hasQuantityInput + previousStructure = structure pause(SPEC_SELECTION_POLL_MILLIS) + waitedMs += SPEC_SELECTION_POLL_MILLIS + polls++ + // A partially loaded purchase sheet may take longer than the plain + // entry verification window; it gets the bounded #331 wait. No Back + // is pressed while waiting; the caller fails explicitly on timeout. + val limitReached = if (requiredEvidenceSeen) { + waitedMs >= SpecPanelStabilityPolicy.WAIT_BOUND_MS + } else { + polls >= SPEC_POST_CLICK_VERIFY_POLLS + } + if (limitReached) break } + waitDiagnostic("timeout") return SpecPanelWait(last, false, changed) } @@ -478,7 +512,8 @@ class PurchaseRehearsalExecutor( private fun panelEvidence(screen: ParsedPddScreen): String = "type=${screen.specPanelType};scrollables=${screen.panelScrollableCount};headings=${screen.panelHeadingCount};" + "options=${screen.panelOptionCount};summary=${screen.hasSelectionSummary};quantity=${screen.hasQuantityControls};" + - "orderAction=${screen.hasOrderSubmitAction};pageEvidence=${screen.pageEvidenceMatched}" + "orderAction=${screen.hasOrderSubmitAction};pageEvidence=${screen.pageEvidenceMatched};" + + screen.requiredEvidenceSummary() private fun specEntryEvidence(screen: ParsedPddScreen, candidateCount: Int, entryReadyWaitPolls: Int = 0): String = "specEntryCandidates=$candidateCount;explicit=${screen.explicitSpecEntryCount};" + @@ -574,9 +609,11 @@ class PurchaseRehearsalExecutor( verification = waitForExactSelection(input, dimension, target, selectionProofs, SPEC_SELECTION_SUCCESS_VERIFY_POLLS) verification.failure?.let { return it } if (verification.confirmed) return null + panelDiagnostic(selectionFailureEvidence(currentScreen(input), dimension, target)) return failure(SPEC_SELECTION_UNCONFIRMED, "无障碍点击和受控手势后未能确认精确选中状态") } if (outcome.result == FreshActionResult.SUCCESS) { + panelDiagnostic(selectionFailureEvidence(currentScreen(input), dimension, target)) return failure(SPEC_SELECTION_UNCONFIRMED, "点击规格后未能确认精确选中状态") } val subreason = when (outcome.reason) { @@ -589,6 +626,19 @@ class PurchaseRehearsalExecutor( return failure(SPEC_CLICK_FAILED, subreason) } + /** + * #331/task 535: parsed option texts and selected flags of one dimension. + * Spec values are product attributes, not personal data; each is truncated. + */ + private fun selectionFailureEvidence(screen: ParsedPddScreen, dimension: String, target: String): String { + fun clip(value: String) = value.replace(Regex("[\r\n\t;|]+"), " ").take(SELECTION_DIAGNOSTIC_TEXT_CHARS) + val values = screen.dimensions.filter { it.key == dimension }.flatMap { it.values } + return "specSelectionUnconfirmed;dimension=$dimension;target=${clip(target)};valueCount=${values.size};values=" + + values.joinToString("|") { value -> + "${clip(value.text)}:${(value.node.selected || value.node.checked).toString().first()}" + } + } + private data class ExactSelectionWait( val confirmed: Boolean, val failure: PurchaseExecutionOutcome? = null, @@ -1175,6 +1225,7 @@ class PurchaseRehearsalExecutor( private const val SPEC_SELECTION_SUCCESS_VERIFY_POLLS = 20 private const val SPEC_SELECTION_FAILED_VERIFY_POLLS = 5 private const val SPEC_SELECTION_POLL_MILLIS = 100L + private const val SELECTION_DIAGNOSTIC_TEXT_CHARS = 40 private const val SOLD_OUT_PANEL_CLOSE_MILLIS = 300L private const val SPEC_ENTRY_NOT_FOUND = "PURCHASE_SPEC_ENTRY_NOT_FOUND" private const val SPEC_ENTRY_TARGET_AMBIGUOUS = "PURCHASE_SPEC_ENTRY_TARGET_AMBIGUOUS" diff --git a/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseRehearsalExecutorTest.kt b/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseRehearsalExecutorTest.kt index e6faabe..1930676 100644 --- a/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseRehearsalExecutorTest.kt +++ b/android/app/src/test/java/cn/ilapage/goauto/agent/PurchaseRehearsalExecutorTest.kt @@ -781,7 +781,8 @@ class PurchaseRehearsalExecutorTest { assertEquals("PURCHASE_SPEC_PANEL_EVIDENCE_NOT_MATCHED", outcome.errorCode) assertEquals( - "规格入口点击后页面已变化，但规格面板强证据不足 [type=UNKNOWN;scrollables=0;headings=0;options=0;summary=false;quantity=false;orderAction=false;pageEvidence=true]", + "规格入口点击后页面已变化，但规格面板强证据不足 [type=UNKNOWN;scrollables=0;headings=0;options=0;summary=false;quantity=false;orderAction=false;pageEvidence=true;" + + "addr=0;pay=0;qin=0;adj=0;summary=0;close=0;submit=0]", outcome.message, ) assertEquals(0, driver.specTapCount) diff --git a/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelFixtures.kt b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelFixtures.kt new file mode 100644 index 0000000..0835200 --- /dev/null +++ b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelFixtures.kt @@ -0,0 +1,267 @@ +package cn.ilapage.goauto.agent + +import cn.ilapage.goauto.agent.automation.NodeBounds +import cn.ilapage.goauto.agent.automation.SnapshotNode +import cn.ilapage.goauto.agent.automation.UiSnapshot + +/** + * #331 sanitized spec-panel fixtures. The layout mirrors the structure of real + * PDD purchase sheets, but every name, phone number and address is fake. + */ +internal object SpecPanelFixtures { + const val PDD = "com.xunmeng.pinduoduo" + const val ACTIVITY = "com.xunmeng.pinduoduo.activity.NewPageActivity" + const val FAKE_PHONE = "139****0000" + + class Tree(private val screenBottom: Int) { + val nodes = mutableListOf() + + init { + add("r", "", NodeBounds(0, 0, 1080, screenBottom), className = "android.widget.FrameLayout") + } + + fun add( + path: String, + text: String, + bounds: NodeBounds, + className: String = "android.widget.TextView", + clickable: Boolean = false, + scrollable: Boolean = false, + selected: Boolean = false, + visible: Boolean = true, + enabled: Boolean = true, + description: String? = null, + ) { + val parent = path.substringBeforeLast('/', "").ifEmpty { null } + nodes += SnapshotNode( + path, parent, text, description, null, className, bounds, + clickable, scrollable, selected, false, enabled, visible, + ) + } + + fun snapshot(packageName: String = PDD, activity: String = ACTIVITY) = UiSnapshot(packageName, activity, nodes.toList()) + } + + /** Options for the purchase sheet shaped like the two unselected real samples. */ + data class Sheet( + val screenBottom: Int = 2216, + val address: Boolean = true, + val payment: Boolean = true, + val quantityInput: Boolean = true, + val adjustButtons: Boolean = true, + val close: Boolean = true, + val summary: Boolean = true, + val sizeDimension: Boolean = true, + val listScrollable: Boolean = true, + /** "visible" = normal submit bar; "hidden" = Samsung-like 31px bar whose [0,0][0,0] label is not visible; "none". */ + val submit: String = "visible", + val offset: Int = 0, + ) + + fun sheet(options: Sheet = Sheet()): UiSnapshot { + val o = options.offset + val t = Tree(options.screenBottom) + t.add("r/sheet", "", NodeBounds(0, 84, 1080, options.screenBottom), "android.widget.LinearLayout", clickable = true) + t.add("r/sheet/top", "", NodeBounds(0, 219, 1080, 330), "android.widget.ViewSwitcher", clickable = true) + t.add("r/sheet/top/tags", "#示例标签#七天退换", NodeBounds(193, 240, 886, 308)) + if (options.close) { + t.add("r/sheet/close", "", NodeBounds(975, 237, 1050, 312), "android.widget.ImageView", clickable = true, description = "关闭") + } + t.add("r/sheet/body", "", NodeBounds(0, 330, 1080, options.screenBottom), "android.view.ViewGroup", clickable = true) + if (options.address) { + // Address row: clickable row five levels above the masked phone text. + t.add("r/sheet/body/addr", "", NodeBounds(0, 330 + o, 1080, 484 + o), "android.view.ViewGroup", clickable = true) + t.add("r/sheet/body/addr/a", "", NodeBounds(0, 340 + o, 1080, 470 + o), "android.widget.LinearLayout") + t.add("r/sheet/body/addr/a/b", "", NodeBounds(36, 340 + o, 1044, 400 + o), "android.widget.LinearLayout") + t.add("r/sheet/body/addr/a/b/c", "", NodeBounds(36, 340 + o, 1044, 400 + o), "android.widget.LinearLayout") + t.add("r/sheet/body/addr/a/b/c/tag", "7天无理由退货", NodeBounds(138, 348 + o, 394, 394 + o)) + t.add("r/sheet/body/addr/a/b/c/phone", "测试，$FAKE_PHONE，示例省示例市", NodeBounds(412, 346 + o, 993, 395 + o)) + t.add("r/sheet/body/addr/a/detail", "示例区示例街道示例路1号", NodeBounds(132, 408 + o, 937, 457 + o)) + } + t.add("r/sheet/body/image", "", NodeBounds(36, 638, 360, 962), "android.widget.FrameLayout", clickable = true, description = "商品主图") + t.add("r/sheet/body/price", "¥19.6", NodeBounds(384, 640, 523, 707)) + if (options.summary) t.add("r/sheet/body/summary", "请选择: 颜色分类 尺码", NodeBounds(384, 791, 1068, 852)) + t.add("r/sheet/body/qty", "", NodeBounds(384, 888, 633, 963), "android.widget.LinearLayout") + if (options.adjustButtons) { + t.add("r/sheet/body/qty/dec", "", NodeBounds(384, 888, 462, 963), "android.widget.ImageView", clickable = true, description = "减少数量") + t.add("r/sheet/body/qty/inc", "", NodeBounds(555, 888, 633, 963), "android.widget.ImageView", clickable = true, description = "增加数量") + } + if (options.quantityInput) { + t.add("r/sheet/body/qty/input", "1", NodeBounds(468, 888, 549, 963), "android.widget.EditText", clickable = true) + } + val list = "r/sheet/body/list" + t.add(list, "", NodeBounds(0, 1136, 1080, 2036), "androidx.recyclerview.widget.RecyclerView", scrollable = options.listScrollable) + t.add("$list/color", "", NodeBounds(0, 1136, 1080, 1686), "android.widget.LinearLayout") + t.add("$list/color/h", "颜色分类", NodeBounds(36, 1161, 216, 1222)) + listOf("黑色 示例款" to 36, "卡其色 示例款" to 382).forEachIndexed { index, (label, left) -> + val block = "$list/color/o$index" + t.add(block, "", NodeBounds(left, 1247, left + 316, 1650), "android.view.ViewGroup", clickable = true, description = label) + t.add("$block/img", "", NodeBounds(left, 1247, left + 316, 1563), "android.widget.ImageView", clickable = true, description = label) + t.add("$block/big", "", NodeBounds(left, 1247, left + 111, 1358), "android.widget.ImageView", clickable = true, description = "打开大图") + t.add("$block/l", "", NodeBounds(left, 1563, left + 316, 1650), "android.widget.LinearLayout") + t.add("$block/l/t", label, NodeBounds(left, 1563, left + 316, 1650), clickable = true) + } + if (options.sizeDimension) { + t.add("$list/size", "", NodeBounds(0, 1686, 1080, 2036), "android.widget.LinearLayout") + t.add("$list/size/row", "", NodeBounds(36, 1686, 1044, 1767), "android.widget.LinearLayout", clickable = true) + t.add("$list/size/row/h", "尺码", NodeBounds(36, 1700, 126, 1753)) + t.add("$list/size/row/hint", "查看尺码建议", NodeBounds(150, 1702, 402, 1751)) + listOf("S 建议75-90斤", "M 建议90-105斤", "L 建议105-115斤").forEachIndexed { index, label -> + val top = 1779 + index * 85 + t.add("$list/size/o$index", "", NodeBounds(36, top, 400, top + 80), "android.view.ViewGroup", clickable = true) + t.add("$list/size/o$index/t", label, NodeBounds(36, top, 400, top + 80), clickable = true) + } + } + if (options.payment) { + t.add("r/sheet/pay", "", NodeBounds(0, 2036, 1080, 2135), "android.view.ViewGroup", clickable = true) + t.add("r/sheet/pay/l", "", NodeBounds(94, 2057, 930, 2114), "android.widget.LinearLayout") + t.add("r/sheet/pay/l/t", "使用#微信支付，更换先用后付可0元下单", NodeBounds(112, 2057, 930, 2114)) + } + when (options.submit) { + "visible" -> { + t.add("r/sheet/submit", "", NodeBounds(0, 2135, 1080, options.screenBottom), "android.widget.FrameLayout", clickable = true) + t.add("r/sheet/submit/l", "", NodeBounds(145, 2135, 934, options.screenBottom), "android.widget.LinearLayout") + t.add("r/sheet/submit/l/t", "选择颜色分类及尺码后，提交订单", NodeBounds(157, 2179, 922, options.screenBottom)) + } + "hidden" -> { + t.add("r/sheet/submit", "", NodeBounds(0, options.screenBottom - 31, 1080, options.screenBottom), "android.widget.FrameLayout", clickable = true) + t.add("r/sheet/submit/l", "", NodeBounds(0, 0, 0, 0), "android.widget.LinearLayout", visible = false) + t.add("r/sheet/submit/l/t", "选择颜色分类及尺码后，提交订单", NodeBounds(0, 0, 0, 0), visible = false) + } + } + return t.snapshot() + } + + /** + * Shape reported by live accessibility for the failing product: no bounded + * scrollable, a single heading, no summary, no +/- and no readable submit. + */ + fun liveShapedSheet(options: Sheet = Sheet()): UiSnapshot = sheet( + options.copy(summary = false, listScrollable = false, sizeDimension = false, adjustButtons = false, submit = "hidden"), + ) + + /** Task 535 layout: the selected option block exposes an outer block, an image and an inner text with a badge. */ + fun taskOptionDedupSheet(): UiSnapshot { + val t = Tree(2328) + t.add("r/sheet", "", NodeBounds(0, 120, 1080, 2328), "android.widget.LinearLayout", clickable = true) + t.add("r/sheet/close", "", NodeBounds(975, 273, 1050, 348), "android.widget.ImageView", clickable = true, description = "关闭") + t.add("r/sheet/body", "", NodeBounds(0, 366, 1080, 2328), "android.view.ViewGroup", clickable = true) + t.add("r/sheet/body/addr", "", NodeBounds(0, 366, 1080, 520), "android.view.ViewGroup", clickable = true) + t.add("r/sheet/body/addr/a", "", NodeBounds(0, 376, 1080, 510), "android.widget.LinearLayout") + t.add("r/sheet/body/addr/a/phone", "测试，$FAKE_PHONE，示例省示例市", NodeBounds(412, 382, 993, 431)) + t.add("r/sheet/body/addr/a/detail", "示例区示例路2号", NodeBounds(132, 444, 993, 493)) + t.add("r/sheet/body/price", "快抢光 ¥23.99", NodeBounds(396, 568, 722, 635)) + t.add("r/sheet/body/selected", "#快抢光#兰条纹 2XL建议130-150斤", NodeBounds(396, 719, 1053, 781)) + t.add("r/sheet/body/qty", "", NodeBounds(396, 817, 614, 895), "android.widget.LinearLayout") + t.add("r/sheet/body/qty/dec", "", NodeBounds(396, 817, 462, 895), "android.widget.ImageView", clickable = true, description = "减少数量") + t.add("r/sheet/body/qty/input", "1", NodeBounds(463, 817, 547, 895), "android.widget.EditText", clickable = true) + t.add("r/sheet/body/qty/inc", "", NodeBounds(548, 817, 614, 895), "android.widget.ImageView", clickable = true, description = "增加数量") + val list = "r/sheet/body/list" + t.add(list, "", NodeBounds(0, 933, 1080, 2079), "androidx.recyclerview.widget.RecyclerView", scrollable = true) + t.add("$list/color", "", NodeBounds(0, 933, 1080, 1483), "android.widget.LinearLayout") + t.add("$list/color/h", "颜色分类", NodeBounds(36, 958, 216, 1019)) + val blue = "$list/color/o0" + t.add(blue, "", NodeBounds(36, 1044, 352, 1447), "android.view.ViewGroup", clickable = true, selected = true, description = " 兰条纹") + t.add("$blue/img", "", NodeBounds(36, 1044, 352, 1360), "android.widget.ImageView", clickable = true, selected = true, description = " 兰条纹") + t.add("$blue/big", "", NodeBounds(36, 1044, 147, 1155), "android.widget.ImageView", clickable = true, selected = true, description = "打开大图") + t.add("$blue/l", "", NodeBounds(36, 1339, 352, 1447), "android.widget.LinearLayout", selected = true) + t.add("$blue/l/t", " 兰条纹\n 零差评", NodeBounds(36, 1339, 352, 1447), clickable = true, selected = true) + val white = "$list/color/o1" + t.add(white, "", NodeBounds(382, 1044, 698, 1447), "android.view.ViewGroup", clickable = true, description = "白条纹") + t.add("$white/img", "", NodeBounds(382, 1044, 698, 1360), "android.widget.ImageView", clickable = true, description = "白条纹") + t.add("$white/big", "", NodeBounds(382, 1044, 493, 1155), "android.widget.ImageView", clickable = true, description = "打开大图") + t.add("$white/badge", "快要抢光", NodeBounds(521, 1044, 698, 1093)) + t.add("$white/l", "", NodeBounds(382, 1339, 698, 1447), "android.widget.LinearLayout") + t.add("$white/l/t", "白条纹", NodeBounds(382, 1339, 698, 1447), clickable = true) + t.add("$list/size", "", NodeBounds(0, 1483, 1080, 1938), "android.widget.LinearLayout") + t.add("$list/size/row", "", NodeBounds(36, 1483, 1044, 1564), "android.widget.LinearLayout", clickable = true) + t.add("$list/size/row/h", "尺码", NodeBounds(36, 1497, 126, 1550)) + listOf( + Triple("L建议100-115斤", NodeBounds(36, 1576, 371, 1661), false), + Triple("XL建议115-130斤", NodeBounds(401, 1576, 762, 1661), false), + Triple("2XL建议130-150斤", NodeBounds(36, 1691, 430, 1776), true), + Triple("3XL建议150-170斤", NodeBounds(460, 1691, 853, 1776), false), + Triple("4XL建议170-190斤", NodeBounds(36, 1806, 431, 1891), false), + ).forEachIndexed { index, (label, bounds, selected) -> + t.add("$list/size/o$index", "", bounds, "android.view.ViewGroup", clickable = true) + t.add("$list/size/o$index/t", label, bounds, clickable = true, selected = selected) + } + t.add("r/sheet/pay", "", NodeBounds(0, 2079, 1080, 2178), "android.view.ViewGroup", clickable = true) + t.add("r/sheet/pay/t", "使用#微信支付，更换先用后付可0元下单", NodeBounds(112, 2100, 930, 2157)) + t.add("r/sheet/submit", "", NodeBounds(0, 2181, 1080, 2328), "android.widget.FrameLayout", clickable = true) + t.add("r/sheet/submit/t", "提交订单 ¥23.99", NodeBounds(365, 2225, 715, 2284)) + return t.snapshot() + } + + fun productDetailPage(): UiSnapshot { + val t = Tree(2216) + t.add("r/pager", "示例商品标题连衣裙夏季新款示例", NodeBounds(0, 84, 1080, 1100), "androidx.viewpager.widget.ViewPager") + t.add("r/price", "¥19.6", NodeBounds(36, 1120, 300, 1200)) + t.add("r/sales", "已拼1万+件", NodeBounds(700, 1120, 1044, 1200)) + t.add("r/deliver", "配送至 示例省示例市", NodeBounds(36, 1300, 1044, 1360)) + t.add("r/spec", "请选择 颜色分类 尺码", NodeBounds(36, 1400, 1044, 1480), clickable = true) + t.add("r/review", "商品评价(2000+)", NodeBounds(36, 1500, 1044, 1580), clickable = true) + t.add("r/single", "单独购买", NodeBounds(420, 2080, 720, 2200), clickable = true) + t.add("r/group", "发起拼单", NodeBounds(740, 2080, 1060, 2200), clickable = true) + return t.snapshot() + } + + fun reviewPage(): UiSnapshot { + val t = Tree(2216) + t.add("r/title", "商品评价", NodeBounds(400, 120, 680, 200)) + t.add("r/all", "全部", NodeBounds(36, 260, 150, 320), clickable = true) + t.add("r/media", "有图/视频", NodeBounds(170, 260, 360, 320), clickable = true) + t.add("r/body", "质量不错，微信支付很方便", NodeBounds(36, 400, 1044, 500)) + return t.snapshot() + } + + fun addressListPage(): UiSnapshot { + val t = Tree(2216) + t.add("r/title", "收货地址", NodeBounds(400, 120, 680, 200)) + listOf(0, 1).forEach { index -> + val top = 300 + index * 260 + t.add("r/card$index", "", NodeBounds(0, top, 1080, top + 240), "android.view.ViewGroup", clickable = true) + t.add("r/card$index/phone", "测试 $FAKE_PHONE", NodeBounds(36, top + 20, 700, top + 80)) + t.add("r/card$index/detail", "示例省示例市示例路${index + 1}号", NodeBounds(36, top + 100, 900, top + 160)) + t.add("r/card$index/edit", "修改", NodeBounds(950, top + 20, 1044, top + 80), clickable = true) + } + return t.snapshot() + } + + fun addressEditPage(): UiSnapshot { + val t = Tree(2216) + t.add("r/title", "修改收货地址", NodeBounds(360, 120, 720, 200)) + t.add("r/name", "测试收货人", NodeBounds(180, 190, 900, 270), "android.widget.EditText", clickable = true) + t.add("r/phone", "13900000000", NodeBounds(180, 290, 900, 370), "android.widget.EditText", clickable = true) + t.add("r/detailLabel", "详细地址", NodeBounds(20, 400, 160, 460)) + t.add("r/detail", "示例路1号", NodeBounds(180, 390, 900, 480), "android.widget.EditText", clickable = true) + t.add("r/save", "保存", NodeBounds(36, 2080, 1044, 2180), clickable = true) + return t.snapshot() + } + + fun paymentMethodDialog(): UiSnapshot { + val t = Tree(2216) + t.add("r/title", "选择支付方式", NodeBounds(360, 1200, 720, 1280)) + t.add("r/wechat", "", NodeBounds(0, 1300, 1080, 1420), "android.view.ViewGroup", clickable = true) + t.add("r/wechat/t", "微信支付", NodeBounds(120, 1330, 400, 1390)) + t.add("r/later", "", NodeBounds(0, 1440, 1080, 1560), "android.view.ViewGroup", clickable = true) + t.add("r/later/t", "先用后付", NodeBounds(120, 1470, 400, 1530)) + t.add("r/ok", "确定", NodeBounds(36, 2080, 1044, 2180), clickable = true) + return t.snapshot() + } + + fun postSubmitPaymentPage(): UiSnapshot { + val t = Tree(2216) + t.add("r/amount", "¥19.60", NodeBounds(360, 300, 720, 400)) + t.add("r/wechat", "", NodeBounds(0, 600, 1080, 720), "android.view.ViewGroup", clickable = true) + t.add("r/wechat/t", "微信支付", NodeBounds(120, 630, 400, 690)) + t.add("r/pay", "立即支付", NodeBounds(36, 2080, 1044, 2180), clickable = true) + return t.snapshot(activity = "com.xunmeng.pinduoduo.app_pay.core.PayActivity") + } + + fun quantityOnlyPage(): UiSnapshot { + val t = Tree(2216) + t.add("r/input", "1", NodeBounds(468, 888, 549, 963), "android.widget.EditText", clickable = true) + return t.snapshot() + } +} diff --git a/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt new file mode 100644 index 0000000..41b86e6 --- /dev/null +++ b/android/app/src/test/java/cn/ilapage/goauto/agent/SpecPanelRecognitionTest.kt @@ -0,0 +1,486 @@ +package cn.ilapage.goauto.agent + +import cn.ilapage.goauto.agent.SpecPanelFixtures.Sheet +import cn.ilapage.goauto.agent.automation.FreshActionResult +import cn.ilapage.goauto.agent.automation.NodeBounds +import cn.ilapage.goauto.agent.automation.PddScreenParser +import cn.ilapage.goauto.agent.automation.PurchaseAgentCapabilities +import cn.ilapage.goauto.agent.automation.PurchaseExecutionInput +import cn.ilapage.goauto.agent.automation.PurchaseLiveAutomation +import cn.ilapage.goauto.agent.automation.PurchaseLiveException +import cn.ilapage.goauto.agent.automation.PurchaseRehearsalExecutor +import cn.ilapage.goauto.agent.automation.PurchaseRuleParser +import cn.ilapage.goauto.agent.automation.PurchaseUiDriver +import cn.ilapage.goauto.agent.automation.SnapshotNode +import cn.ilapage.goauto.agent.automation.SpecPanelStabilityPolicy +import cn.ilapage.goauto.agent.automation.SpecPanelType +import cn.ilapage.goauto.agent.automation.SwipeDirection +import cn.ilapage.goauto.agent.automation.UiSnapshot +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** #331 spec-panel recognition, option dedup, bounded waits and strict submit targets. */ +class SpecPanelRecognitionTest { + private fun parse(snapshot: UiSnapshot) = + PddScreenParser.parse(snapshot, PurchaseRehearsalExecutor.DEFAULT_COLLECTOR, "719834019024", null) + + // --- Recognition ------------------------------------------------------- + + @Test + fun `both sanitized unselected sheet samples are recognized as spec panels`() { + val first = parse(SpecPanelFixtures.sheet()) + val samsung = parse(SpecPanelFixtures.sheet(Sheet(screenBottom = 2020, submit = "hidden"))) + + listOf(first, samsung).forEach { screen -> + assertTrue(screen.specPanelOpen) + assertTrue(screen.hasAddressEntry) + assertTrue(screen.hasPaymentEntry) + assertTrue(screen.hasQuantityInput) + } + // The zero-size submit label is not visible, so the submit action is absent. + assertFalse(samsung.hasOrderSubmitAction) + } + + @Test + fun `live shaped samples are recognized only through the required evidence fallback`() { + listOf(2216, 2020).forEach { bottom -> + val screen = parse(SpecPanelFixtures.liveShapedSheet(Sheet(screenBottom = bottom))) + + assertEquals(SpecPanelType.REQUIRED_EVIDENCE, screen.specPanelType) + assertTrue(screen.specPanelOpen) + assertEquals(0, screen.panelScrollableCount) + assertFalse(screen.hasSelectionSummary) + assertFalse(screen.hasQuantityControls) + assertFalse(screen.hasQuantityAdjustControls) + assertFalse(screen.hasOrderSubmitAction) + assertEquals(listOf("黑色 示例款", "卡其色 示例款"), screen.dimensions.single().values.map { it.text }) + } + } + + @Test + fun `zero size submit label sample is still recognized because submit is auxiliary`() { + val screen = parse(SpecPanelFixtures.liveShapedSheet(Sheet(screenBottom = 2020))) + + assertFalse(screen.hasOrderSubmitAction) + assertEquals(SpecPanelType.REQUIRED_EVIDENCE, screen.specPanelType) + } + + @Test + fun `missing address entry is not recognized`() { + val screen = parse(SpecPanelFixtures.liveShapedSheet(Sheet(address = false))) + assertFalse(screen.hasAddressEntry) + assertEquals(SpecPanelType.UNKNOWN, screen.specPanelType) + } + + @Test + fun `missing payment entry is not recognized`() { + val screen = parse(SpecPanelFixtures.liveShapedSheet(Sheet(payment = false))) + assertFalse(screen.hasPaymentEntry) + assertEquals(SpecPanelType.UNKNOWN, screen.specPanelType) + } + + @Test + fun `missing quantity input is not recognized`() { + val screen = parse(SpecPanelFixtures.liveShapedSheet(Sheet(quantityInput = false))) + assertFalse(screen.hasQuantityInput) + assertEquals(SpecPanelType.UNKNOWN, screen.specPanelType) + } + + @Test + fun `required items without any auxiliary evidence are not recognized`() { + val t = SpecPanelFixtures.Tree(2216) + t.add("r/addr", "", NodeBounds(0, 330, 1080, 484), "android.view.ViewGroup", clickable = true) + t.add("r/addr/phone", "测试，${SpecPanelFixtures.FAKE_PHONE}", NodeBounds(412, 346, 993, 395)) + t.add("r/qty", "1", NodeBounds(468, 888, 549, 963), "android.widget.EditText", clickable = true) + t.add("r/pay", "", NodeBounds(0, 2036, 1080, 2135), "android.view.ViewGroup", clickable = true) + t.add("r/pay/t", "使用微信支付", NodeBounds(112, 2057, 930, 2114)) + + val screen = parse(t.snapshot()) + + assertTrue(screen.hasRequiredPanelEvidence) + assertEquals(SpecPanelType.UNKNOWN, screen.specPanelType) + } + + @Test + fun `quantity input alone is not recognized`() { + val screen = parse(SpecPanelFixtures.quantityOnlyPage()) + assertTrue(screen.hasQuantityInput) + assertEquals(SpecPanelType.UNKNOWN, screen.specPanelType) + assertFalse(screen.specPanelOpen) + } + + @Test + fun `non panel pages are never recognized as spec panels`() { + mapOf( + "product" to SpecPanelFixtures.productDetailPage(), + "review" to SpecPanelFixtures.reviewPage(), + "addressList" to SpecPanelFixtures.addressListPage(), + "addressEdit" to SpecPanelFixtures.addressEditPage(), + "paymentDialog" to SpecPanelFixtures.paymentMethodDialog(), + "paymentPage" to SpecPanelFixtures.postSubmitPaymentPage(), + ).forEach { (name, snapshot) -> + val screen = parse(snapshot) + assertEquals(name, SpecPanelType.UNKNOWN, screen.specPanelType) + assertFalse(name, screen.specPanelOpen) + } + } + + @Test + fun `whole sheet clickable containers do not count as an address row`() { + val t = SpecPanelFixtures.Tree(2216) + t.add("r/sheet", "", NodeBounds(0, 84, 1080, 2216), "android.widget.LinearLayout", clickable = true) + t.add("r/sheet/phone", "测试，${SpecPanelFixtures.FAKE_PHONE}", NodeBounds(412, 346, 993, 395)) + assertFalse(parse(t.snapshot()).hasAddressEntry) + } + + @Test + fun `single dimension sheet without adjust buttons is recognized`() { + val screen = parse(SpecPanelFixtures.sheet(Sheet(sizeDimension = false, adjustButtons = false, summary = false, listScrollable = false, submit = "none"))) + + assertEquals(SpecPanelType.REQUIRED_EVIDENCE, screen.specPanelType) + assertEquals(listOf("color"), screen.dimensions.map { it.key }) + } + + @Test + fun `required evidence diagnostics hold booleans and counts only`() { + val screen = parse(SpecPanelFixtures.liveShapedSheet()) + val summary = screen.requiredEvidenceSummary() + + assertEquals("addr=1;pay=1;qin=1;adj=0;summary=0;close=1;submit=0", summary) + assertFalse(summary.contains("****")) + assertFalse(summary.contains("示例")) + } + + // --- Task 535 option dedup --------------------------------------------- + + @Test + fun `nested clickable nodes of one option block form one value`() { + val screen = parse(SpecPanelFixtures.taskOptionDedupSheet()) + val colors = screen.dimensions.single { it.key == "color" }.values + val sizes = screen.dimensions.single { it.key == "size" }.values + + assertEquals(listOf("兰条纹", "白条纹"), colors.map { it.text }) + assertTrue(colors.single { it.text == "兰条纹" }.node.selected) + assertFalse(colors.single { it.text == "白条纹" }.node.selected) + assertEquals(5, sizes.size) + assertEquals(listOf("2XL建议130-150斤"), sizes.filter { it.node.selected }.map { it.rawText }) + } + + // --- Bounded wait after the first spec-entry click ----------------------- + + @Test + fun `loading sheet is accepted after required evidence becomes complete and stable without back`() { + val loading = SpecPanelFixtures.liveShapedSheet(Sheet(address = false)) + val ready = SpecPanelFixtures.liveShapedSheet() + val driver = SpecEntryDriver(listOf(loading, loading, loading, ready, ready)) + val diagnostics = mutableListOf() + val steps = mutableListOf() + + PurchaseRehearsalExecutor(driver, { true }, { null }, pause = {}, stepChanged = steps::add, panelDiagnostic = diagnostics::add) + .execute(input(), PurchaseRuleParser.parse(rule()), PurchaseAgentCapabilities.supported) + + assertTrue(steps.contains("selectSpec")) + assertEquals(0, driver.backs) + assertTrue(diagnostics.any { it.startsWith("specPanelWait;outcome=opened;") && it.contains("backPressed=false") }) + } + + @Test + fun `always incomplete sheet fails explicitly at the wait bound without back`() { + val loading = SpecPanelFixtures.liveShapedSheet(Sheet(address = false)) + val driver = SpecEntryDriver(listOf(loading)) + val diagnostics = mutableListOf() + + val outcome = PurchaseRehearsalExecutor(driver, { true }, { null }, pause = {}, panelDiagnostic = diagnostics::add) + .execute(input(), PurchaseRuleParser.parse(rule()), PurchaseAgentCapabilities.supported) + + assertEquals("PURCHASE_SPEC_PANEL_EVIDENCE_NOT_MATCHED", outcome.errorCode) + assertEquals(0, driver.backs) + val timeout = diagnostics.single { it.startsWith("specPanelWait;outcome=timeout;") } + assertTrue(timeout.contains("waitedMs=${SpecPanelStabilityPolicy.WAIT_BOUND_MS};")) + } + + @Test + fun `fallback sheet whose structure keeps changing is never accepted early`() { + val first = SpecPanelFixtures.liveShapedSheet(Sheet(screenBottom = 2216)) + val second = SpecPanelFixtures.liveShapedSheet(Sheet(screenBottom = 2200)) + assertEquals(SpecPanelType.REQUIRED_EVIDENCE, parse(second).specPanelType) + val driver = SpecEntryDriver(listOf(first, second), cycle = true) + val diagnostics = mutableListOf() + + val outcome = PurchaseRehearsalExecutor(driver, { true }, { null }, pause = {}, panelDiagnostic = diagnostics::add) + .execute(input(), PurchaseRuleParser.parse(rule()), PurchaseAgentCapabilities.supported) + + assertEquals("PURCHASE_SPEC_PANEL_EVIDENCE_NOT_MATCHED", outcome.errorCode) + assertEquals(0, driver.backs) + assertFalse(diagnostics.any { it.startsWith("specPanelWait;outcome=opened") }) + assertTrue(diagnostics.any { it.contains("waitedMs=${SpecPanelStabilityPolicy.WAIT_BOUND_MS};") }) + } + + @Test + fun `stronger panel type is still accepted immediately`() { + val driver = SpecEntryDriver(listOf(SpecPanelFixtures.sheet())) + val diagnostics = mutableListOf() + val steps = mutableListOf() + + PurchaseRehearsalExecutor(driver, { true }, { null }, pause = {}, stepChanged = steps::add, panelDiagnostic = diagnostics::add) + .execute(input(), PurchaseRuleParser.parse(rule()), PurchaseAgentCapabilities.supported) + + assertTrue(steps.contains("selectSpec")) + assertFalse(diagnostics.any { it.startsWith("specPanelWait;") }) + } + + // --- Bounded wait after the address save --------------------------------- + + @Test + fun `address save waits for the loading sheet without pressing back`() { + val loading = SpecPanelFixtures.liveShapedSheet(Sheet(address = false)) + val driver = AddressSaveDriver(listOf(loading, loading, loading)) { expected -> readySheet(expected) } + val diagnostics = mutableListOf() + + val proof = PurchaseLiveAutomation(driver, pause = {}, panelDiagnostic = diagnostics::add).updateShippingAddress("_cg31") + + assertEquals("_cg31", proof.suffix) + assertEquals(0, driver.backs) + val done = diagnostics.single { it.startsWith("addressSaveWait;") } + assertTrue(done.contains("outcome=final_evidence")) + assertTrue(done.contains("waitedMs=${2 * SpecPanelStabilityPolicy.ADDRESS_SAVE_POLL_MS};")) + assertTrue(done.contains("backPressed=false")) + assertFalse(diagnostics.any { it.contains("_cg31") || it.contains("示例") || it.contains("****") }) + } + + @Test + fun `address save with a sheet that never completes fails at the bound without back`() { + val loading = SpecPanelFixtures.liveShapedSheet(Sheet(address = false)) + val driver = AddressSaveDriver(listOf(loading)) { loading } + val diagnostics = mutableListOf() + + val error = runCatching { PurchaseLiveAutomation(driver, pause = {}, panelDiagnostic = diagnostics::add).updateShippingAddress("_cg32") } + .exceptionOrNull() as PurchaseLiveException + + assertEquals("PURCHASE_ADDRESS_SAVE_TIMEOUT", error.code) + assertEquals(0, driver.backs) + assertTrue(diagnostics.single().contains("outcome=timeout;waitedMs=${SpecPanelStabilityPolicy.WAIT_BOUND_MS};")) + assertFalse(error.message.orEmpty().contains("示例")) + } + + @Test + fun `address save with an unstable fallback sheet does not decide early`() { + val first = SpecPanelFixtures.liveShapedSheet(Sheet(screenBottom = 2216)) + val second = SpecPanelFixtures.liveShapedSheet(Sheet(screenBottom = 2200)) + val driver = AddressSaveDriver(listOf(first, second), cycle = true) { first } + val diagnostics = mutableListOf() + + val error = runCatching { PurchaseLiveAutomation(driver, pause = {}, panelDiagnostic = diagnostics::add).updateShippingAddress("_cg33") } + .exceptionOrNull() as PurchaseLiveException + + assertEquals("PURCHASE_ADDRESS_SAVE_TIMEOUT", error.code) + assertEquals(0, driver.backs) + assertTrue(diagnostics.single().contains("outcome=timeout")) + } + + @Test + fun `address save with a stable fallback sheet is accepted as purchase panel without back`() { + val sheet = SpecPanelFixtures.liveShapedSheet() + val driver = AddressSaveDriver(listOf(sheet)) { sheet } + val diagnostics = mutableListOf() + + // The stable sheet has no saved suffix and no scroll region, so the + // existing in-panel restore path fails closed; it must not press Back. + val error = runCatching { PurchaseLiveAutomation(driver, pause = {}, panelDiagnostic = diagnostics::add).updateShippingAddress("_cg34") } + .exceptionOrNull() as PurchaseLiveException + + assertEquals(0, driver.backs) + assertTrue(diagnostics.single().contains("outcome=stable_required;waitedMs=${SpecPanelStabilityPolicy.ADDRESS_SAVE_POLL_MS};")) + assertNotEquals("PURCHASE_ADDRESS_UPDATE_FAILED", error.code) + } + + // --- Strict final submit click ------------------------------------------- + + @Test + fun `submit click fails for zero size hidden unmatched or duplicated buttons`() { + fun page(build: SpecPanelFixtures.Tree.() -> Unit) = SpecPanelFixtures.Tree(2216).apply(build).snapshot() + val cases = mapOf( + "hiddenZeroLabel" to SpecPanelFixtures.sheet(Sheet(submit = "hidden")), + "zeroSizeVisibleLabel" to page { + add("r/submit", "", NodeBounds(0, 2185, 1080, 2216), "android.widget.FrameLayout", clickable = true) + add("r/submit/t", "提交订单", NodeBounds(0, 0, 0, 0)) + }, + "zeroSizeContainer" to page { + add("r/submit", "", NodeBounds(0, 0, 0, 0), "android.widget.FrameLayout", clickable = true) + add("r/submit/t", "提交订单", NodeBounds(157, 2179, 922, 2216)) + }, + "labelMismatch" to SpecPanelFixtures.sheet(), + "duplicated" to page { + add("r/a", "", NodeBounds(0, 2000, 540, 2216), "android.widget.FrameLayout", clickable = true) + add("r/a/t", "提交订单", NodeBounds(20, 2050, 520, 2150)) + add("r/b", "", NodeBounds(540, 2000, 1080, 2216), "android.widget.FrameLayout", clickable = true) + add("r/b/t", "提交订单", NodeBounds(560, 2050, 1060, 2150)) + }, + ) + cases.forEach { (name, snapshot) -> + val driver = StaticDriver(snapshot) + val error = runCatching { PurchaseLiveAutomation(driver, pause = {}).submitOrderOnce() }.exceptionOrNull() as? PurchaseLiveException + + assertEquals(name, "PURCHASE_SUBMIT_TARGET_AMBIGUOUS", error?.code) + assertTrue(name, driver.clicked.isEmpty()) + } + } + + @Test + fun `unique visible submit button is still clickable exactly once`() { + val driver = StaticDriver( + SpecPanelFixtures.Tree(2216).apply { + add("r/submit", "", NodeBounds(0, 2135, 1080, 2216), "android.widget.FrameLayout", clickable = true) + add("r/submit/t", "提交订单", NodeBounds(157, 2150, 922, 2210)) + }.snapshot(), + ) + + PurchaseLiveAutomation(driver, pause = {}).submitOrderOnce() + + assertEquals(listOf("提交订单"), driver.clicked) + } + + // --- Helpers ------------------------------------------------------------- + + private fun readySheet(expected: String): UiSnapshot { + val base = SpecPanelFixtures.liveShapedSheet() + val extra = listOf( + node("r/sheet/body/addr/a/saved", expected, NodeBounds(132, 460, 937, 480), parent = "r/sheet/body/addr/a"), + node("r/final", "", NodeBounds(600, 2150, 1060, 2216), clickable = true, parent = "r"), + node("r/final/t", "提交订单", NodeBounds(620, 2160, 1040, 2210), parent = "r/final"), + ) + return base.copy(nodes = base.nodes + extra) + } + + private fun input() = PurchaseExecutionInput( + taskId = 331, + executionMode = "rehearsal", + phase = "purchase", + url = "https://mobile.yangkeduo.com/goods.html?goods_id=719834019024", + goodsId = "719834019024", + mappedColor = "黑色 示例款", + mappedSize = "", + quantity = 1, + minUnitPriceCent = 1_000, + maxUnitPriceCent = 3_000, + ) + + private fun rule() = """{ + "schemaVersion":1, + "ruleType":"pddPurchase", + "requiredCapabilities":["purchase.rehearsal.v1"], + "actions":[ + {"type":"openProduct"},{"type":"verifyProduct"},{"type":"openSpecPanel"},{"type":"selectSpec"}, + {"type":"setQuantity"},{"type":"verifyUnitPrice"},{"type":"verifyOrderSummary"} + ] + }""" + + private open class BaseDriver : PurchaseUiDriver { + var backs = 0 + val clicked = mutableListOf() + override fun capture(): UiSnapshot = UiSnapshot(null, null, emptyList()) + override fun clickFresh(target: SnapshotNode): FreshActionResult { + clicked += target.label + return FreshActionResult.SUCCESS + } + override fun tapPurchaseFresh(target: SnapshotNode) = FreshActionResult.FAILED + override fun inputFresh(target: SnapshotNode, value: String) = FreshActionResult.FAILED + override fun swipePurchase(direction: SwipeDirection, durationMs: Long) = false + override fun swipePurchaseIn(target: SnapshotNode, direction: SwipeDirection, durationMs: Long) = false + override fun backPurchase(): Boolean { + backs++ + return false + } + } + + private class StaticDriver(private val snapshot: UiSnapshot) : BaseDriver() { + override fun capture() = snapshot + } + + /** Product page first; after the spec-entry click it serves [panels] in order (the last repeats unless [cycle]). */ + private class SpecEntryDriver(private val panels: List, private val cycle: Boolean = false) : BaseDriver() { + private var opened = false + var panelCaptures = 0 + + override fun capture(): UiSnapshot { + if (!opened) return SpecPanelFixtures.productDetailPage() + val index = if (cycle) panelCaptures % panels.size else minOf(panelCaptures, panels.lastIndex) + panelCaptures++ + return panels[index] + } + + override fun clickFresh(target: SnapshotNode): FreshActionResult { + clicked += target.label + if (target.label.startsWith("请选择")) opened = true + return FreshActionResult.SUCCESS + } + } + + /** Address flow driver: sheet -> address list -> editor -> [afterSave] samples -> [settled]. */ + private class AddressSaveDriver( + private val afterSave: List, + private val cycle: Boolean = false, + private val settled: (String) -> UiSnapshot, + ) : BaseDriver() { + private var page = "sheet" + private var address = "示例路1号-old" + private var afterSaveCaptures = 0 + + override fun capture(): UiSnapshot = when (page) { + "sheet" -> SpecPanelFixtures.sheet() + "list" -> SpecPanelFixtures.Tree(2216).apply { + add("r/title", "收货地址", NodeBounds(400, 120, 680, 200)) + add("r/modify", "", NodeBounds(900, 300, 1044, 380), "android.view.ViewGroup", clickable = true) + add("r/modify/t", "修改", NodeBounds(920, 310, 1030, 370)) + }.snapshot() + "edit" -> SpecPanelFixtures.Tree(2216).apply { + add("r/title", "修改收货地址", NodeBounds(360, 120, 720, 200)) + add("r/detailLabel", "详细地址", NodeBounds(20, 400, 160, 460)) + add("r/detail", address, NodeBounds(180, 390, 900, 480), "android.widget.EditText", clickable = true) + add("r/save", "", NodeBounds(36, 2080, 1044, 2180), "android.view.ViewGroup", clickable = true) + add("r/save/t", "保存", NodeBounds(400, 2100, 700, 2160)) + }.snapshot() + else -> { + val index = afterSaveCaptures++ + when { + cycle -> afterSave[index % afterSave.size] + index < afterSave.size -> afterSave[index] + else -> settled(address) + } + } + } + + override fun clickFresh(target: SnapshotNode): FreshActionResult { + clicked += target.label + when (target.label) { + "修改" -> page = "edit" + "保存" -> page = "saved" + } + return FreshActionResult.SUCCESS + } + + override fun clickAddressEntryFresh(target: SnapshotNode): cn.ilapage.goauto.agent.automation.FreshClickOutcome { + page = "list" + return cn.ilapage.goauto.agent.automation.FreshClickOutcome( + FreshActionResult.SUCCESS, + cn.ilapage.goauto.agent.automation.FreshClickReason.SUCCESS, + 1, + 1, + ) + } + + override fun inputFresh(target: SnapshotNode, value: String): FreshActionResult { + address = value + return FreshActionResult.SUCCESS + } + } + + private companion object { + fun node(path: String, text: String, bounds: NodeBounds, clickable: Boolean = false, parent: String?) = SnapshotNode( + path, parent, text, null, null, "android.widget.TextView", bounds, clickable, false, false, false, true, true, + ) + } +}