fix(search): load all prefix matches in one ranked pass

Rows were read via peek(), which gives Paging no access hints, so prefix matches beyond the first page never loaded. Drop the head/tail paging: a query reads the whole prefix range of every active dictionary in one scan, merges and ranks it, and hands Paging a single page. The append error row and its strings go away; the prefix scan reuses search() with substring matches disabled. FileSource reads through a buffer, cutting per-entry syscall overhead.
2026-10-12 02:34:02 +08:00
1 parent 4fc765382e
commit a4197d2970
@@ -15,49 +15,17 @@ import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.CancellationException
 import java.io.File

-/**
- * Two-phase search paging.
- *
- * Page one (key = null) is the ranked head: the capped, relevance-ordered result the
- * search screen has always shown, merged across active dictionaries. Every range scan
- * that hit its cap hands back a [IndexSearcher.RangeCursor]; those become the tail key.
- *
- * Tail pages continue the unfinished range scans directly from the index files and
- * merge them alphabetically (k-way, by index word), so scrolling eventually surfaces
- * every prefix match in every active dictionary. Articles already shown -- by the head
- * or by an earlier tail page, including diacritic alias duplicates -- are filtered
- * through [emittedArticles], which grows only as far as the user actually scrolls.
- *
- * A new query builds a new PagingSource (see SearchViewModel's flatMapLatest), so an
- * in-flight head or tail load is cancelled by Paging when the user keeps typing.
- */
 class IndexEntryPagingSource(
    private val indexSearcher: IndexSearcher,
-    private val dictionaries: List<Dictionary>,
+    dictionaries: List<Dictionary>,
    private val query: String
-) : PagingSource<IndexEntryPagingSource.TailKey, IndexEntry>() {
+) : PagingSource<Int, IndexEntry>() {

-    data class StreamCursor(
-        val dictPosition: Int,
-        val cursor: IndexSearcher.RangeCursor
-    )
+    private val activeDictionaries = dictionaries.filter { it.isActive }

-    data class TailKey(val streams: List<StreamCursor>)
-
-    private data class EmittedKey(
-        val dictPosition: Int,
-        val word: String,
-        val offset: Long
-    )
-
-    private val emittedArticles = HashSet<EmittedKey>()
-
-    private val activeDictionaries = dictionaries.withIndex().filter { it.value.isActive }
-
-    override suspend fun load(params: LoadParams<TailKey>): LoadResult<TailKey, IndexEntry> {
+    override suspend fun load(params: LoadParams<Int>): LoadResult<Int, IndexEntry> {
        return try {
-            val key = params.key
-            if (key == null) loadHead() else loadTail(key, params.loadSize)
+            loadAll()
        } catch (e: CancellationException) {
            throw e
        } catch (e: Exception) {
@@ -65,21 +33,22 @@ class IndexEntryPagingSource(
        }
    }

-    private suspend fun loadHead(): LoadResult<TailKey, IndexEntry> =
+    private suspend fun loadAll(): LoadResult<Int, IndexEntry> =
        ReSearchTrace.asyncSection(ReSearchTrace.SEARCH_PAGING) {
            coroutineScope {
                val perDictionary = activeDictionaries
-                    .map { (position, dict) ->
+                    .map { dict ->
                        async {
                            val indexFile = File(dict.indexPath)
                            if (!indexFile.exists()) return@async null

                            try {
-                                val result = indexSearcher.searchWithCursors(
+                                val entries = indexSearcher.search(
                                    pathOrUri = indexFile.absolutePath,
-                                    query = query
+                                    query = query,
+                                    includeSubstringMatches = false
                                )
-                                Triple(position, dict, result)
+                                dict to entries
                            } catch (e: CancellationException) {
                                throw e
                            } catch (_: Exception) {
@@ -90,9 +59,9 @@ class IndexEntryPagingSource(
                    .awaitAll()
                    .filterNotNull()

-                val rawResults = perDictionary.flatMap { (position, dict, result) ->
-                    result.entries.map { entry ->
-                        position to entry.withDictionary(
+                val rawResults = perDictionary.flatMap { (dict, entries) ->
+                    entries.map { entry ->
+                        entry.withDictionary(
                            dict.metadata.name.ifBlank { dict.name },
                            dict.path
                        )
@@ -100,95 +69,19 @@ class IndexEntryPagingSource(
                }

                val normalizedQuery = query.sanitizeQuery().lowercase().trim()
-                val ranked = rawResults.map { it.second }
+                val ranked = rawResults
                    .rankBySearchRelevance(SearchRankingContext(normalizedQuery))
                    .distinctBy { Triple(it.dictionaryPath, it.word, it.offset.value) }

-                rawResults.forEach { (position, entry) -> emittedArticles.add(entry.emittedKey(position)) }
-
-                val streams = perDictionary.flatMap { (position, _, result) ->
-                    result.cursors.map { StreamCursor(position, it) }
-                }
                LoadResult.Page(
                    data = ranked,
                    prevKey = null,
-                    nextKey = if (streams.isEmpty()) null else TailKey(streams)
+                    nextKey = null
                )
            }
        }

-    private suspend fun loadTail(key: TailKey, loadSize: Int): LoadResult<TailKey, IndexEntry> {
-        val streams = key.streams.map { TailStream(it.dictPosition, it.cursor) }
-        val page = ArrayList<IndexEntry>(loadSize)
-        val pageKeys = HashSet<EmittedKey>()
-        val chunkSize = maxOf(loadSize, MIN_CHUNK)
-
-        while (page.size < loadSize) {
-            streams.forEach { it.ensureBuffered(chunkSize) }
-            val next = streams
-                .filter { it.hasBuffered }
-                .minByOrNull { it.peekWord }
-                ?: break
-
-            val scanned = next.consume()
-            val dict = dictionaries[next.dictPosition]
-            val entry = scanned.entry.withDictionary(
-                dict.metadata.name.ifBlank { dict.name },
-                dict.path
-            )
-            val emittedKey = entry.emittedKey(next.dictPosition)
-            if (emittedKey !in emittedArticles && pageKeys.add(emittedKey)) {
-                page.add(entry)
-            }
-        }
-
-        emittedArticles.addAll(pageKeys)
-
-        val remaining = streams.mapNotNull { stream ->
-            stream.nextCursor?.let { StreamCursor(stream.dictPosition, it) }
-        }
-        return LoadResult.Page(
-            data = page,
-            prevKey = null,
-            nextKey = if (remaining.isEmpty()) null else TailKey(remaining)
-        )
-    }
-
-    private inner class TailStream(
-        val dictPosition: Int,
-        initialCursor: IndexSearcher.RangeCursor
-    ) {
-        var nextCursor: IndexSearcher.RangeCursor? = initialCursor
-            private set
-        private var buffer: List<IndexSearcher.ScannedEntry> = emptyList()
-        private var position = 0
-
-        val hasBuffered: Boolean get() = position < buffer.size
-        val peekWord: String get() = buffer[position].entry.word
-
-        suspend fun ensureBuffered(chunkSize: Int) {
-            if (hasBuffered) return
-            val cursor = nextCursor ?: return
-            val indexPath = File(dictionaries[dictPosition].indexPath).absolutePath
-            buffer = indexSearcher.scanTailChunk(indexPath, cursor, chunkSize)
-            position = 0
-            if (buffer.isEmpty()) nextCursor = null
-        }
-
-        fun consume(): IndexSearcher.ScannedEntry {
-            val scanned = buffer[position]
-            position++
-            nextCursor = scanned.cursorAfter
-            return scanned
-        }
-    }
-
-    private fun IndexEntry.emittedKey(dictPosition: Int) =
-        EmittedKey(dictPosition, word, offset.value)
-
-    override fun getRefreshKey(state: PagingState<TailKey, IndexEntry>): TailKey? {
+    override fun getRefreshKey(state: PagingState<Int, IndexEntry>): Int? {
        return null
    }
 }
-
-private const val MIN_CHUNK = 48
@@ -18,6 +18,7 @@ import kotlinx.coroutines.yield
 import java.io.BufferedInputStream
 import java.io.Closeable
 import java.io.DataInputStream
+import java.io.EOFException
 import java.io.File
 import java.io.FileInputStream
 import java.io.RandomAccessFile
@@ -25,7 +26,6 @@ import java.io.RandomAccessFile
 class IndexSearcher(private val context: Context) {
    private val metadataCache = MetadataLruCache(maxBytes = 64L * 1024L * 1024L)
    private val resultsCache = ResultsLruCache<List<IndexEntry>>(maxEntries = 20)
-    private val cursorResultsCache = ResultsLruCache<CursorSearchResult>(maxEntries = 20)

    /**
     * Pre-allocated reusable buffer for readEntry() to reduce allocations.
@@ -45,27 +45,6 @@ class IndexSearcher(private val context: Context) {
        val dataStartOffset: Long
    )

-    data class RangeCursor(
-        val rangeQuery: String,
-        val absoluteIndex: Int,
-        val fileOffset: Long
-    )
-
-    internal class RangeScanResult(
-        val entries: List<IndexEntry>,
-        val nextCursor: RangeCursor?
-    )
-
-    class CursorSearchResult(
-        val entries: List<IndexEntry>,
-        val cursors: List<RangeCursor>
-    )
-
-    class ScannedEntry(
-        val entry: IndexEntry,
-        val cursorAfter: RangeCursor?
-    )
-
    suspend fun findFirstEntry(pathOrUri: String, query: String): IndexEntry? = withContext(Dispatchers.Default) {
        val normalizedQuery = query.sanitizeQuery().lowercase().trim()
        try {
@@ -109,7 +88,7 @@ class IndexSearcher(private val context: Context) {
        }
        val normalizedQuery = query.sanitizeQuery().lowercase().trim()
        val foldedQuery = normalizedQuery.foldLatinDiacritics()
-        val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}"
+        val cacheKey = "$pathOrUri:$includeSubstringMatches:${normalizedQuery.sanitizeCacheKey()}"
        resultsCache.get(cacheKey)?.let {
            return@withContext it
        }
@@ -118,9 +97,9 @@ class IndexSearcher(private val context: Context) {
            val count = metadata.totalCount
            if (count == 0) return@withContext emptyList()
            val results = openSource(pathOrUri).use { source ->
-                val prefixMatches = readPrefixMatches(source, metadata, normalizedQuery).entries.toMutableList()
+                val prefixMatches = readPrefixMatches(source, metadata, normalizedQuery).toMutableList()
                if (foldedQuery != normalizedQuery) {
-                    prefixMatches.addAll(readPrefixMatches(source, metadata, foldedQuery).entries)
+                    prefixMatches.addAll(readPrefixMatches(source, metadata, foldedQuery))
                }

                val substringMatches = mutableListOf<IndexEntry>()
@@ -165,81 +144,9 @@ class IndexSearcher(private val context: Context) {
        metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) }
    }

-    suspend fun searchWithCursors(
-        pathOrUri: String,
-        query: String
-    ): CursorSearchResult = withContext(Dispatchers.Default) {
-        if (query.isBlank()) {
-            return@withContext CursorSearchResult(emptyList(), emptyList())
-        }
-        val normalizedQuery = query.sanitizeQuery().lowercase().trim()
-        val foldedQuery = normalizedQuery.foldLatinDiacritics()
-        val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}"
-        cursorResultsCache.get(cacheKey)?.let {
-            return@withContext it
-        }
-        try {
-            val metadata = metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) }
-            if (metadata.totalCount == 0) {
-                return@withContext CursorSearchResult(emptyList(), emptyList())
-            }
-            val result = openSource(pathOrUri).use { source ->
-                val scans = buildList {
-                    add(readPrefixMatches(source, metadata, normalizedQuery))
-                    if (foldedQuery != normalizedQuery) {
-                        add(readPrefixMatches(source, metadata, foldedQuery))
-                    }
-                }
-                val entries = scans.flatMap { it.entries }
-                    .rankBySearchRelevance(SearchRankingContext(normalizedQuery, foldedQuery))
-                    .distinctBy { it.articleKey() }
-                CursorSearchResult(entries, scans.mapNotNull { it.nextCursor })
-            }
-            cursorResultsCache.put(cacheKey, result)
-            result
-        } catch (e: CancellationException) {
-            throw e
-        } catch (e: Exception) {
-            Log.e(TAG, "searchWithCursors() - error: ${e.message}", e)
-            if (e is DictionaryError) throw e
-            throw DictionaryError.SearchError.QueryFailed(context.getString(R.string.error_search_failed, query.take(20), pathOrUri.take(20)), e)
-        }
-    }
-
-    suspend fun scanTailChunk(
-        pathOrUri: String,
-        cursor: RangeCursor,
-        maxEntries: Int
-    ): List<ScannedEntry> = withContext(Dispatchers.Default) {
-        val metadata = metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) }
-        if (cursor.absoluteIndex >= metadata.totalCount) return@withContext emptyList()
-        openSource(pathOrUri).use { source ->
-            source.seek(cursor.fileOffset)
-            val out = ArrayList<ScannedEntry>(maxEntries)
-            var currentIdx = cursor.absoluteIndex
-            var iterationCount = 0
-            while (currentIdx < metadata.totalCount && out.size < maxEntries) {
-                if (iterationCount % 1000 == 0) yield()
-                val entry = readEntry(source, metadata)
-                val entryWordLower = entry.word.lowercase()
-                if (!entryWordLower.startsWith(cursor.rangeQuery)) break
-                currentIdx++
-                iterationCount++
-                val cursorAfter = if (currentIdx < metadata.totalCount) {
-                    RangeCursor(cursor.rangeQuery, currentIdx, source.position())
-                } else {
-                    null
-                }
-                out.add(ScannedEntry(entry.copy(word = entryWordLower), cursorAfter))
-            }
-            out
-        }
-    }
-
    fun trimMemory(level: Int) {
        metadataCache.trim(level)
        resultsCache.trim(level)
-        cursorResultsCache.trim(level)
    }

    suspend fun isMetadataLoaded(pathOrUri: String): Boolean {
@@ -349,9 +256,9 @@ class IndexSearcher(private val context: Context) {
        source: RandomAccessSource,
        metadata: IndexMetadata,
        query: String
-    ): RangeScanResult {
+    ): List<IndexEntry> {
        val firstMatch = findFirstMatch(source, metadata, query)
-            ?: return RangeScanResult(emptyList(), null)
+            ?: return emptyList()
        return readMatchingEntries(source, metadata, firstMatch, query)
    }

@@ -415,26 +322,21 @@ class IndexSearcher(private val context: Context) {
        metadata: IndexMetadata,
        firstMatch: Pair<Int, IndexEntry>,
        query: String
-    ): RangeScanResult {
+    ): List<IndexEntry> {
        val results = mutableListOf<IndexEntry>()
        results.add(firstMatch.second)
-        val uniqueArticles = hashSetOf(firstMatch.second.articleKey())
        var currentIdx = firstMatch.first + 1
        var iterationCount = 0
        while (currentIdx < metadata.totalCount) {
-            if (uniqueArticles.size >= MAX_PREFIX_RESULTS) {
-                return RangeScanResult(results, RangeCursor(query, currentIdx, source.position()))
-            }
            if (iterationCount % 1000 == 0) yield()
            val entry = readEntry(source, metadata)
            val entryWordLower = entry.word.lowercase()
            if (!entryWordLower.startsWith(query)) break
-            uniqueArticles.add(entry.articleKey())
            results.add(entry.copy(word = entryWordLower))
            currentIdx++
            iterationCount++
        }
-        return RangeScanResult(results, null)
+        return results
    }

    private fun readPooledString(source: RandomAccessSource, length: Int): String {
@@ -483,7 +385,6 @@ class IndexSearcher(private val context: Context) {
        private const val TAG = "IndexSearcher"
        private const val SUBSTRING_SCAN_MAX_ENTRIES = 200_000
        private const val SPARSE_INTERVAL_V1 = 128
-        private const val MAX_PREFIX_RESULTS = 200
        private const val MAX_SUBSTRING_RESULTS = 100
    }
 }
@@ -588,30 +489,89 @@ interface RandomAccessSource : Closeable {
 }
 class FileSource(file: File) : RandomAccessSource {
    private val raf = RandomAccessFile(file, "r")
+    private val buffer = ByteArray(BUFFER_SIZE)
+    private var bufferStart = 0L
+    private var bufferEnd = 0L
+    private var pos = 0L

    override fun seek(pos: Long) {
-        raf.seek(pos)
+        this.pos = pos
    }

-    override fun position(): Long = raf.filePointer
+    override fun position(): Long = pos

-    override fun readUnsignedByte(): Int = raf.readUnsignedByte()
+    private fun refill(minNeeded: Int): Boolean {
+        if (bufferEnd - pos >= minNeeded) return true
+        bufferStart = pos
+        raf.seek(pos)
+        val read = raf.read(buffer)
+        bufferEnd = bufferStart + maxOf(read, 0)
+        return bufferEnd - pos >= minNeeded
+    }

-    override fun readUnsignedShort(): Int = raf.readUnsignedShort()
+    override fun readUnsignedByte(): Int {
+        if (!refill(1)) throw EOFException()
+        return buffer[(pos++ - bufferStart).toInt()].toInt() and 0xFF
+    }

-    override fun readInt(): Int = raf.readInt()
+    override fun readUnsignedShort(): Int {
+        if (!refill(2)) throw EOFException()
+        val i = (pos - bufferStart).toInt()
+        pos += 2
+        return (buffer[i].toInt() and 0xFF shl 8) or (buffer[i + 1].toInt() and 0xFF)
+    }

-    override fun readLong(): Long = raf.readLong()
+    override fun readInt(): Int {
+        if (!refill(4)) throw EOFException()
+        val i = (pos - bufferStart).toInt()
+        pos += 4
+        return (buffer[i].toInt() shl 24) or
+            (buffer[i + 1].toInt() and 0xFF shl 16) or
+            (buffer[i + 2].toInt() and 0xFF shl 8) or
+            (buffer[i + 3].toInt() and 0xFF)
+    }
+
+    override fun readLong(): Long {
+        if (!refill(8)) throw EOFException()
+        val i = (pos - bufferStart).toInt()
+        pos += 8
+        var value = 0L
+        for (k in 0 until 8) {
+            value = (value shl 8) or (buffer[i + k].toLong() and 0xFF)
+        }
+        return value
+    }

    override fun readFully(b: ByteArray) {
-        raf.readFully(b)
+        readFully(b, 0, b.size)
    }

    override fun readFully(b: ByteArray, off: Int, len: Int) {
-        raf.readFully(b, off, len)
+        if (len > BUFFER_SIZE / 2) {
+            raf.seek(pos)
+            raf.readFully(b, off, len)
+            pos += len
+            bufferEnd = bufferStart
+            return
+        }
+        var remaining = len
+        var dst = off
+        while (remaining > 0) {
+            if (!refill(1)) throw EOFException()
+            val i = (pos - bufferStart).toInt()
+            val chunk = minOf(remaining, (bufferEnd - pos).toInt())
+            System.arraycopy(buffer, i, b, dst, chunk)
+            pos += chunk
+            dst += chunk
+            remaining -= chunk
+        }
    }

    override fun close() {
        raf.close()
    }
+
+    private companion object {
+        private const val BUFFER_SIZE = 64 * 1024
+    }
 }
@@ -6,17 +6,13 @@ import androidx.compose.foundation.lazy.*
 import androidx.compose.material3.*
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.remember
-import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.testTag
-import androidx.compose.ui.res.stringResource
 import androidx.compose.ui.semantics.semantics
 import androidx.compose.ui.semantics.testTagsAsResourceId
 import androidx.compose.ui.text.style.TextOverflow
 import androidx.compose.ui.unit.dp
-import androidx.paging.LoadState
 import androidx.paging.compose.*
-import com.example.research.R
 import com.example.research.common.ui.theme.dictionaryTitleLarge
 import com.example.research.core.domain.model.IndexEntry
 import com.example.research.core.util.removeDictionarySuffixes
@@ -53,7 +49,7 @@ fun SearchResultsList(
            contentType = { "search_result" }
        ) { index ->
            val entry = if (showStale) staleEntries.getOrNull(index)
-            else if (index in 0 until results.itemCount) results.peek(index) else null
+            else if (index in 0 until results.itemCount) results[index] else null
            if (entry != null) {
                SearchResultItem(
                    entry = entry,
@@ -63,26 +59,6 @@ fun SearchResultsList(
                )
            }
        }
-
-        when (results.loadState.append) {
-            is LoadState.Error -> {
-                item {
-                    Box(
-                        modifier = Modifier
-                            .fillMaxWidth()
-                            .padding(16.dp),
-                        contentAlignment = Alignment.Center
-                    ) {
-                        Text(
-                            text = stringResource(R.string.error_loading_more),
-                            style = MaterialTheme.typography.bodyMedium,
-                            color = MaterialTheme.colorScheme.error
-                        )
-                    }
-                }
-            }
-            else -> {}
-        }
    }
 }

@@ -62,7 +62,6 @@
    <string name="error_scan_directory_failed">Не удалось просканировать папку со словарями</string>
    <string name="error_search_failed">Ошибка поиска запроса \'%1$s\' в %2$s</string>
    <string name="error_failed_load_index_metadata">Не удалось загрузить метаданные индекса: %1$s</string>
-    <string name="error_loading_more">Ошибка загрузки</string>
    <plurals name="active_dictionaries_count">
        <item quantity="one">%1$d активный словарь</item>
        <item quantity="few">%1$d активных словаря</item>
@@ -62,7 +62,6 @@
    <string name="error_scan_directory_failed">Failed to scan dictionary folder</string>
    <string name="error_search_failed">Search failed for query \'%1$s\' in %2$s</string>
    <string name="error_failed_load_index_metadata">Failed to load index metadata: %1$s</string>
-    <string name="error_loading_more">Loading error</string>
    <plurals name="active_dictionaries_count">
        <item quantity="one">%1$d active dictionary</item>
        <item quantity="other">%1$d active dictionaries</item>