OPC/goauto
2
0
Fork 0
Code Issues 227 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(android): track horizontal color rows frame to frame (#370)

Browse Source 
Follow each horizontal color row by overlap with its previous frame (with a
position fallback for whole-page moves on multi-row grids), skip reverse
sweeps when rows share one container, and keep the earliest abnormal
COLOR_DISCOVERY reason.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NTDbDcwbDw1TSAcE6wfh2F
This commit is contained in:
QiuSW
2026-10-10 15:04:06 +08:00
co-authored by Claude Opus 5.5
parent 23dafcf73f
commit d8b4f3a9d5
2 changed files with 233 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android/app/src/main/java/cn/ilapage/goauto/agent/automation/PddProductDetailCollector.kt
+85 -10
View File
@@ -1412,6 +1412,14 @@ class PddProductDetailCollector(
// These signatures stay in memory; only the aggregate tri-state is saved.
val diagnosticHorizontalRows = mutableMapOf<Set<String>, Boolean?>()
var diagnosticTermination = AgentDiagnosticReason.COLOR_FOUND
// Keep the earliest abnormal reason; later normal ends must not hide it.
fun terminate(reason: AgentDiagnosticReason) {
if (diagnosticTermination == AgentDiagnosticReason.COLOR_FOUND ||
diagnosticTermination == AgentDiagnosticReason.COLOR_EDGE_REACHED
) {
diagnosticTermination = reason
}
}
val imageAttempts = mutableSetOf<String>()
fun collectVisibleImages(values: List<VisibleSpecValue>) {
@@ -1561,7 +1569,18 @@ class PddProductDetailCollector(
// established left/right paging behavior.
if (rows.any { it.size > 1 }) {
val rowSeeds = rows.map { row -> row.map(VisibleSpecValue::text).toSet() }
// Frame-to-frame row tracking: rows are followed by overlap with
// their previous frame, not by the pass's starting frame.
val tracks = rows.map(::colorRowTrack).toMutableList()
val coveredRows = mutableSetOf<Int>()
var coveredIncomplete: Boolean? = null
rowSeeds.forEachIndexed { rowIndex, seed ->
if (rowIndex in coveredRows) {
// A shared container already swept this row's values.
diagnosticHorizontalRows[seed] = coveredIncomplete
?: diagnosticHorizontalRows[seed]?.takeIf { it }
return@forEachIndexed
}
val moveRight = rowIndex % 2 == 0
val horizontalSignatureReads = mutableMapOf<List<String>, Int>()
var diagnosticPreviousSignature: List<String>? = null
@@ -1569,11 +1588,13 @@ class PddProductDetailCollector(
var diagnosticMoved = false
var diagnosticRowUncertain = false
var diagnosticRowIncomplete: Boolean? = null
var sharedMoves = 0
var sharedBroken = tracks.size < 2
for (horizontalPass in 0..config.limits.getValue("specHorizontalSwipes")) {
var diagnosticBeforeClicks: List<String>? = null
collectVisibleColors { observedRows ->
diagnosticBeforeClicks = observedRows.filter { row -> row.any { it.text in seed } }
.singleOrNull()?.sortedBy { it.node.bounds.left }
diagnosticBeforeClicks = matchColorRow(tracks[rowIndex], observedRows, tracks.size, requireUniqueOverlap = true)
?.sortedBy { it.node.bounds.left }
?.let { if (moveRight) it else it.reversed() }?.let(::optionSignature)
}?.let { return it }
screen = parse(goodsId, config, evidence)
@@ -1581,11 +1602,25 @@ class PddProductDetailCollector(
if (!screen.pageEvidenceMatched) return failure("RULE_NOT_MATCHED", "采集期间离开 PDD 商品详情页")
rows = colorRows(screen)
observeColorDiscovery(screen, rows)
val matchedRow = rows.maxByOrNull { row -> row.count { it.text in seed } }
?.takeIf { row -> row.any { it.text in seed } }
val matches = tracks.map { matchColorRow(it, rows, tracks.size) }
val matchedRow = matches[rowIndex]
if (matchedRow != null &&
matchedRow.map(VisibleSpecValue::text).toSet() != tracks[rowIndex].texts
) {
sharedMoves++
val everyOtherRowMoved = tracks.indices.all { other ->
other == rowIndex || matches[other]?.let { match ->
match.map(VisibleSpecValue::text).toSet() != tracks[other].texts
} == true
}
if (!everyOtherRowMoved) sharedBroken = true
}
matches.forEachIndexed { index, match ->
match?.let { tracks[index] = colorRowTrack(it) }
}
if (matchedRow == null) {
diagnosticHorizontalUnknown = true
diagnosticTermination = AgentDiagnosticReason.COLOR_ROW_REFLOWED
terminate(AgentDiagnosticReason.COLOR_ROW_REFLOWED)
break
}
val currentRow = matchedRow.sortedBy { it.node.bounds.left }
@@ -1608,6 +1643,13 @@ class PddProductDetailCollector(
if (signatureReads > config.limits.getValue("stableEdgeReads")) {
if (!diagnosticRowUncertain && diagnosticStableReads >= config.limits.getValue("stableEdgeReads")) {
diagnosticRowIncomplete = false
// Every other row changed with every move of this
// row and this row is stable at its edge: they share
// one container and were swept along with it.
if (!sharedBroken && sharedMoves > 0) {
coveredRows += (rowIndex + 1 until tracks.size)
coveredIncomplete = false
}
}
break
}
@@ -1640,7 +1682,7 @@ class PddProductDetailCollector(
diagnosticTrailingEmptyReads = if (rows.isEmpty()) diagnosticTrailingEmptyReads + 1 else 0
if (rows.isEmpty()) diagnosticHorizontalUnknown = true
if (!discoverVertically) {
diagnosticTermination = AgentDiagnosticReason.COLOR_EDGE_REACHED
terminate(AgentDiagnosticReason.COLOR_EDGE_REACHED)
break
}
@@ -1652,21 +1694,21 @@ class PddProductDetailCollector(
}
previousVerticalSignature = verticalSignature
if (verticalStable >= config.limits.getValue("stableEdgeReads")) {
diagnosticTermination = AgentDiagnosticReason.COLOR_EDGE_REACHED
terminate(AgentDiagnosticReason.COLOR_EDGE_REACHED)
break
}
if (verticalPass == config.limits.getValue("specVerticalSwipes")) {
diagnosticTermination = AgentDiagnosticReason.COLOR_SCAN_LIMIT
terminate(AgentDiagnosticReason.COLOR_SCAN_LIMIT)
break
}
val anchor = rows.flatten().firstOrNull()?.node ?: specPanelContainer
if (anchor == null) {
diagnosticHorizontalUnknown = true
diagnosticTermination = AgentDiagnosticReason.COLOR_CONTAINER_UNAVAILABLE
terminate(AgentDiagnosticReason.COLOR_CONTAINER_UNAVAILABLE)
break
}
if (!driver.swipeSpec(SwipeDirection.UP, anchor)) {
diagnosticTermination = AgentDiagnosticReason.COLOR_SWIPE_FAILED
terminate(AgentDiagnosticReason.COLOR_SWIPE_FAILED)
break
}
diagnosticVerticalSwipes++
@@ -2219,6 +2261,39 @@ class PddProductDetailCollector(
const val COLOR_IMAGE_MAX_TOTAL_BYTES = 8 * 1024 * 1024
}
/** Last observed frame of one color row, used to follow it across horizontal swipes. */
private class ColorRowTrack(val texts: Set<String>, val centerY: Int, val tolerance: Int)
private fun colorRowTrack(row: List<VisibleSpecValue>) = ColorRowTrack(
row.map(VisibleSpecValue::text).toSet(),
row.map { it.node.bounds.centerY }.average().toInt(),
(row.first().node.bounds.height / 2).coerceIn(24, 80),
)
/**
* Finds the current-frame row for a tracked row. Overlap with the previous
* frame wins. With no overlap (a whole-page move) a multi-row grid whose
* row count is unchanged may match by vertical position, but only when
* exactly one current row sits within the colorRows tolerance. Anything
* else is not the same row and stays a reflow.
*/
private fun matchColorRow(
track: ColorRowTrack,
rows: List<List<VisibleSpecValue>>,
trackedRowCount: Int,
requireUniqueOverlap: Boolean = false,
): List<VisibleSpecValue>? {
val overlapping = rows.filter { row -> row.any { it.text in track.texts } }
if (overlapping.isNotEmpty()) {
if (requireUniqueOverlap && overlapping.size > 1) return null
return overlapping.maxByOrNull { row -> row.count { it.text in track.texts } }
}
if (trackedRowCount < 2 || rows.size != trackedRowCount) return null
return rows.filter { row ->
kotlin.math.abs(row.map { it.node.bounds.centerY }.average().toInt() - track.centerY) <= track.tolerance
}.singleOrNull()
}
private fun colorRows(screen: ParsedPddScreen): List<List<VisibleSpecValue>> {
val values = screen.dimensions.filter { it.key == "color" }.flatMap { it.values }
.sortedWith(compareBy({ it.node.bounds.centerY }, { it.node.bounds.left }))
android/app/src/test/java/cn/ilapage/goauto/agent/PddProductDetailCollectorTest.kt
+148 -7
View File
@@ -2092,6 +2092,122 @@ class PddProductDetailCollectorTest {
}
}
private fun gridColors(count: Int) = (1..count).map { "款%02d色".format(it) }
private fun gridConfig() = config().copy(
timeoutsMs = config().timeoutsMs + ("overall" to 600_000),
limits = config().limits + mapOf("specHorizontalSwipes" to 12, "specVerticalSwipes" to 3, "stableEdgeReads" to 2),
)
private fun sharedGridCollects47Colors(step: Int) {
// 47 colors in 2 rows x 24 columns sharing one horizontal container.
val colors = gridColors(47)
val grid = listOf(colors.filterIndexed { i, _ -> i % 2 == 0 }, colors.filterIndexed { i, _ -> i % 2 == 1 })
assertEquals(listOf(24, 23), grid.map { it.size })
val prices = colors.mapIndexed { i, c -> c to 1000L + i * 10L }.toMap()
val events = mutableListOf<AgentDiagnosticEvent>()
val driver = FakeCollectorDriver(
colors = colors,
prices = prices,
horizontalGrid = grid,
gridVisibleColumns = 4,
gridStep = step,
sizePages = listOf(listOf("S"), listOf("M")),
hideColorHeadingAfterFirstVerticalPage = true,
)
var clock = 0L
val result = PddProductDetailCollector(driver, { clock }, { clock += it }, taskId = 370, diagnostic = events::add)
.collect(GOODS_ID, rule(gridConfig()))
assertTrue(result.successful)
val payload = requireNotNull(result.payload)
assertEquals("completed", payload.status)
assertEquals(colors.toSet(), payload.dimensions.first { it.key == "color" }.values.toSet())
assertEquals(47, payload.colorPrices.size)
assertEquals(prices, payload.colorPrices.associate { it.color to it.priceCent })
// 47 colors x 2 sizes, each SKU carrying its own color's price.
assertEquals(94, payload.skus.size)
assertEquals(setOf("S", "M"), payload.skus.map { it.specs.getValue("size") }.toSet())
payload.skus.forEach { assertEquals(prices.getValue(it.specs.getValue("color")), it.priceCent) }
assertTrue(payload.missing.isEmpty())
// The shared container is swept once: no reverse sweep away from an edge.
assertTrue(driver.gridSwipeLog.none { it.first == SwipeDirection.RIGHT && it.second > 0 })
assertTrue(driver.gridSwipeLog.count { it.first == SwipeDirection.LEFT } <= 12)
val event = events.single { it.stage == AgentDiagnosticStage.COLOR_DISCOVERY }
assertEquals(47, event.clickableColorCount)
}
@Test
fun `shared container two row grid collects all colors when swipes overlap`() = sharedGridCollects47Colors(step = 3)
@Test
fun `shared container two row grid collects all colors when swipes move whole pages`() = sharedGridCollects47Colors(step = 4)
@Test
fun `independent row grid keeps the snake sweep and does not claim a shared container`() {
val colors = gridColors(20)
val grid = listOf(colors.take(10), colors.drop(10))
val driver = FakeCollectorDriver(
colors = colors,
horizontalGrid = grid,
gridVisibleColumns = 4,
gridStep = 3,
gridSharedContainer = false,
sizePages = listOf(listOf("S"), listOf("M")),
hideColorHeadingAfterFirstVerticalPage = true,
)
var clock = 0L
val result = PddProductDetailCollector(driver, { clock }, { clock += it }).collect(GOODS_ID, rule(gridConfig()))
assertTrue(result.successful)
val collected = requireNotNull(result.payload).dimensions.first { it.key == "color" }.values
// Row 0 is swept to its end; row 1 only exposes its first window because
// it starts at its own left edge, exactly as before.
assertEquals((colors.take(10) + colors.drop(10).take(4)).toSet(), collected.toSet())
assertEquals(14, collected.size)
assertTrue(driver.gridSwipeLog.any { it.first == SwipeDirection.RIGHT && it.second > 0 })
}
@Test
fun `row disappearing keeps reflow reason after later empty vertical reads`() {
val colors = listOf("A色", "B色", "C色", "D色", "E色", "F色")
val events = mutableListOf<AgentDiagnosticEvent>()
val driver = FakeCollectorDriver(
colors = colors,
colorPages = listOf(colors.take(4), colors.takeLast(2)),
rowSize = 2,
sizePages = listOf(listOf("S"), listOf("M")),
hideColorHeadingAfterFirstVerticalPage = true,
)
var clock = 0L
val result = PddProductDetailCollector(driver, { clock }, { clock += it }, taskId = 370, diagnostic = events::add)
.collect(GOODS_ID, rule(gridConfig()))
assertTrue(result.successful)
val event = events.single { it.stage == AgentDiagnosticStage.COLOR_DISCOVERY }
assertEquals(AgentDiagnosticReason.COLOR_ROW_REFLOWED, event.reason)
assertTrue(requireNotNull(event.colorTrailingEmptyReadCount) > 0)
assertEquals(null, event.colorHorizontalIncomplete)
}
@Test
fun `single row disjoint reflow keeps reflow reason`() {
val colors = listOf("A色", "B色", "C色", "D色")
val events = mutableListOf<AgentDiagnosticEvent>()
var clock = 0L
val result = PddProductDetailCollector(
FakeCollectorDriver(colors = colors, colorPages = listOf(colors.take(2), colors.takeLast(2))),
{ clock }, { clock += it }, taskId = 370, diagnostic = events::add,
).collect(GOODS_ID, rule())
assertTrue(result.successful)
assertEquals(AgentDiagnosticReason.COLOR_ROW_REFLOWED, events.single { it.stage == AgentDiagnosticStage.COLOR_DISCOVERY }.reason)
}
private class FakeCollectorDriver(
colors: List<String> = listOf("红色"),
sizes: List<String> = listOf("S"),
@@ -2146,6 +2262,14 @@ class PddProductDetailCollectorTest {
// restored to top. Heading count drops but color/size values are
// untouched, so this must NOT be treated as a collapsed panel.
private val specPanelDropsExtraDimensionAfterTopSwipe: Boolean = false,
// #370: horizontal multi-row color grid. Each swipe moves gridStep
// columns; gridSharedContainer moves all rows together, otherwise only
// the anchored row. Left/right swipes are logged with the largest
// column offset before the gesture.
private val horizontalGrid: List<List<String>>? = null,
private val gridVisibleColumns: Int = 4,
private val gridStep: Int = 3,
private val gridSharedContainer: Boolean = true,
) : PddCollectorDriver {
var captureCount = 0
var clickCount = 0
@@ -2155,6 +2279,8 @@ class PddProductDetailCollectorTest {
var backCount = 0
var entryClickCount = 0
var restoreGestures = 0
val gridSwipeLog = mutableListOf<Pair<SwipeDirection, Int>>()
private val gridOffsets = IntArray(horizontalGrid?.size ?: 0)
private var selected: String? = initialSelectedColor
private var previousSelected: String? = null
private var horizontalPage = 0
@@ -2224,19 +2350,22 @@ class PddProductDetailCollectorTest {
val continuationPage = hideDimensionHeadingsAfterFirstVerticalPage && verticalPage > 0
val panelCollapsedNow = specPanelCollapsesAfterTopSwipe && downSwipeCount >= 1
val hideColorNow = specPanelHidesColorInitially && downSwipeCount == 0
val visibleColors = colorVerticalPages?.get(verticalPage.coerceAtMost(colorVerticalPages.lastIndex))
val gridPlaced = horizontalGrid?.flatMapIndexed { r, row ->
row.drop(gridOffsets[r]).take(gridVisibleColumns).mapIndexed { c, color -> Triple(color, r, c) }
}
val visibleColors = gridPlaced?.map { it.first }
?: colorVerticalPages?.get(verticalPage.coerceAtMost(colorVerticalPages.lastIndex))
?: colorPages[horizontalPage.coerceAtMost(colorPages.lastIndex)]
val colorRowCount = (visibleColors.size + rowSize - 1) / rowSize
val colorRowCount = horizontalGrid?.size ?: ((visibleColors.size + rowSize - 1) / rowSize)
val placedColors = gridPlaced ?: visibleColors
.filterNot { hideSelectedColorOption && it == selected }
.mapIndexed { index, color -> Triple(color, index / rowSize, index % rowSize) }
val sizeHeadingTop = maxOf(700, 470 + colorRowCount * 90 + 20)
if (!continuationPage && !panelCollapsedNow && !(hideColorHeadingAfterFirstVerticalPage && verticalPage > 0)) {
nodes += node("scroll/color-heading", "颜色分类", 20, 400, 300, 450, parentPath = "scroll")
}
if (!continuationPage && !hideColorNow && !panelCollapsedNow) {
visibleColors
.filterNot { hideSelectedColorOption && it == selected }
.forEachIndexed { index, color ->
val row = index / rowSize
val column = index % rowSize
placedColors.forEach { (color, row, column) ->
val path = "scroll/color-$color-$captureCount"
val left = 30 + column * 230
val top = 470 + row * 90
@@ -2357,6 +2486,18 @@ class PddProductDetailCollectorTest {
override fun swipeSpec(direction: SwipeDirection, anchor: SnapshotNode?): Boolean {
swipes += direction to anchor
if (horizontalGrid != null && (direction == SwipeDirection.LEFT || direction == SwipeDirection.RIGHT)) {
gridSwipeLog += direction to (gridOffsets.maxOrNull() ?: 0)
val delta = if (direction == SwipeDirection.LEFT) gridStep else -gridStep
val moved = if (gridSharedContainer) horizontalGrid.indices.toList()
else listOf((((anchor?.bounds?.top ?: 470) - 470) / 90).coerceIn(0, horizontalGrid.lastIndex))
val sharedMax = horizontalGrid.maxOf { it.size } - gridVisibleColumns
moved.forEach { r ->
val max = if (gridSharedContainer) sharedMax else horizontalGrid[r].size - gridVisibleColumns
gridOffsets[r] = (gridOffsets[r] + delta).coerceIn(0, maxOf(0, max))
}
return true
}
if ((direction == SwipeDirection.UP || direction == SwipeDirection.DOWN) && !verticalSwipeSucceeds) return false
when (direction) {
SwipeDirection.LEFT -> horizontalPage = (horizontalPage + 1).coerceAtMost(colorPages.lastIndex)