diff --git a/app/src/main/java/com/example/research/data/paging/IndexEntryPagingSource.kt b/app/src/main/java/com/example/research/data/paging/IndexEntryPagingSource.kt index 4e386aa..ec29031 100644 --- a/app/src/main/java/com/example/research/data/paging/IndexEntryPagingSource.kt +++ b/app/src/main/java/com/example/research/data/paging/IndexEntryPagingSource.kt @@ -15,49 +15,17 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.CancellationException import java.io.File -/** - * Two-phase search paging. - * - * Page one (key = null) is the ranked head: the capped, relevance-ordered result the - * search screen has always shown, merged across active dictionaries. Every range scan - * that hit its cap hands back a [IndexSearcher.RangeCursor]; those become the tail key. - * - * Tail pages continue the unfinished range scans directly from the index files and - * merge them alphabetically (k-way, by index word), so scrolling eventually surfaces - * every prefix match in every active dictionary. Articles already shown -- by the head - * or by an earlier tail page, including diacritic alias duplicates -- are filtered - * through [emittedArticles], which grows only as far as the user actually scrolls. - * - * A new query builds a new PagingSource (see SearchViewModel's flatMapLatest), so an - * in-flight head or tail load is cancelled by Paging when the user keeps typing. - */ class IndexEntryPagingSource( private val indexSearcher: IndexSearcher, - private val dictionaries: List, + dictionaries: List, private val query: String -) : PagingSource() { +) : PagingSource() { - data class StreamCursor( - val dictPosition: Int, - val cursor: IndexSearcher.RangeCursor - ) + private val activeDictionaries = dictionaries.filter { it.isActive } - data class TailKey(val streams: List) - - private data class EmittedKey( - val dictPosition: Int, - val word: String, - val offset: Long - ) - - private val emittedArticles = HashSet() - - private val activeDictionaries = dictionaries.withIndex().filter { it.value.isActive } - - override suspend fun load(params: LoadParams): LoadResult { + override suspend fun load(params: LoadParams): LoadResult { return try { - val key = params.key - if (key == null) loadHead() else loadTail(key, params.loadSize) + loadAll() } catch (e: CancellationException) { throw e } catch (e: Exception) { @@ -65,21 +33,22 @@ class IndexEntryPagingSource( } } - private suspend fun loadHead(): LoadResult = + private suspend fun loadAll(): LoadResult = ReSearchTrace.asyncSection(ReSearchTrace.SEARCH_PAGING) { coroutineScope { val perDictionary = activeDictionaries - .map { (position, dict) -> + .map { dict -> async { val indexFile = File(dict.indexPath) if (!indexFile.exists()) return@async null try { - val result = indexSearcher.searchWithCursors( + val entries = indexSearcher.search( pathOrUri = indexFile.absolutePath, - query = query + query = query, + includeSubstringMatches = false ) - Triple(position, dict, result) + dict to entries } catch (e: CancellationException) { throw e } catch (_: Exception) { @@ -90,9 +59,9 @@ class IndexEntryPagingSource( .awaitAll() .filterNotNull() - val rawResults = perDictionary.flatMap { (position, dict, result) -> - result.entries.map { entry -> - position to entry.withDictionary( + val rawResults = perDictionary.flatMap { (dict, entries) -> + entries.map { entry -> + entry.withDictionary( dict.metadata.name.ifBlank { dict.name }, dict.path ) @@ -100,95 +69,19 @@ class IndexEntryPagingSource( } val normalizedQuery = query.sanitizeQuery().lowercase().trim() - val ranked = rawResults.map { it.second } + val ranked = rawResults .rankBySearchRelevance(SearchRankingContext(normalizedQuery)) .distinctBy { Triple(it.dictionaryPath, it.word, it.offset.value) } - rawResults.forEach { (position, entry) -> emittedArticles.add(entry.emittedKey(position)) } - - val streams = perDictionary.flatMap { (position, _, result) -> - result.cursors.map { StreamCursor(position, it) } - } LoadResult.Page( data = ranked, prevKey = null, - nextKey = if (streams.isEmpty()) null else TailKey(streams) + nextKey = null ) } } - private suspend fun loadTail(key: TailKey, loadSize: Int): LoadResult { - val streams = key.streams.map { TailStream(it.dictPosition, it.cursor) } - val page = ArrayList(loadSize) - val pageKeys = HashSet() - val chunkSize = maxOf(loadSize, MIN_CHUNK) - - while (page.size < loadSize) { - streams.forEach { it.ensureBuffered(chunkSize) } - val next = streams - .filter { it.hasBuffered } - .minByOrNull { it.peekWord } - ?: break - - val scanned = next.consume() - val dict = dictionaries[next.dictPosition] - val entry = scanned.entry.withDictionary( - dict.metadata.name.ifBlank { dict.name }, - dict.path - ) - val emittedKey = entry.emittedKey(next.dictPosition) - if (emittedKey !in emittedArticles && pageKeys.add(emittedKey)) { - page.add(entry) - } - } - - emittedArticles.addAll(pageKeys) - - val remaining = streams.mapNotNull { stream -> - stream.nextCursor?.let { StreamCursor(stream.dictPosition, it) } - } - return LoadResult.Page( - data = page, - prevKey = null, - nextKey = if (remaining.isEmpty()) null else TailKey(remaining) - ) - } - - private inner class TailStream( - val dictPosition: Int, - initialCursor: IndexSearcher.RangeCursor - ) { - var nextCursor: IndexSearcher.RangeCursor? = initialCursor - private set - private var buffer: List = emptyList() - private var position = 0 - - val hasBuffered: Boolean get() = position < buffer.size - val peekWord: String get() = buffer[position].entry.word - - suspend fun ensureBuffered(chunkSize: Int) { - if (hasBuffered) return - val cursor = nextCursor ?: return - val indexPath = File(dictionaries[dictPosition].indexPath).absolutePath - buffer = indexSearcher.scanTailChunk(indexPath, cursor, chunkSize) - position = 0 - if (buffer.isEmpty()) nextCursor = null - } - - fun consume(): IndexSearcher.ScannedEntry { - val scanned = buffer[position] - position++ - nextCursor = scanned.cursorAfter - return scanned - } - } - - private fun IndexEntry.emittedKey(dictPosition: Int) = - EmittedKey(dictPosition, word, offset.value) - - override fun getRefreshKey(state: PagingState): TailKey? { + override fun getRefreshKey(state: PagingState): Int? { return null } } - -private const val MIN_CHUNK = 48 diff --git a/app/src/main/java/com/example/research/data/parser/IndexSearcher.kt b/app/src/main/java/com/example/research/data/parser/IndexSearcher.kt index de451eb..6b5efd9 100644 --- a/app/src/main/java/com/example/research/data/parser/IndexSearcher.kt +++ b/app/src/main/java/com/example/research/data/parser/IndexSearcher.kt @@ -18,6 +18,7 @@ import kotlinx.coroutines.yield import java.io.BufferedInputStream import java.io.Closeable import java.io.DataInputStream +import java.io.EOFException import java.io.File import java.io.FileInputStream import java.io.RandomAccessFile @@ -25,7 +26,6 @@ import java.io.RandomAccessFile class IndexSearcher(private val context: Context) { private val metadataCache = MetadataLruCache(maxBytes = 64L * 1024L * 1024L) private val resultsCache = ResultsLruCache>(maxEntries = 20) - private val cursorResultsCache = ResultsLruCache(maxEntries = 20) /** * Pre-allocated reusable buffer for readEntry() to reduce allocations. @@ -45,27 +45,6 @@ class IndexSearcher(private val context: Context) { val dataStartOffset: Long ) - data class RangeCursor( - val rangeQuery: String, - val absoluteIndex: Int, - val fileOffset: Long - ) - - internal class RangeScanResult( - val entries: List, - val nextCursor: RangeCursor? - ) - - class CursorSearchResult( - val entries: List, - val cursors: List - ) - - class ScannedEntry( - val entry: IndexEntry, - val cursorAfter: RangeCursor? - ) - suspend fun findFirstEntry(pathOrUri: String, query: String): IndexEntry? = withContext(Dispatchers.Default) { val normalizedQuery = query.sanitizeQuery().lowercase().trim() try { @@ -109,7 +88,7 @@ class IndexSearcher(private val context: Context) { } val normalizedQuery = query.sanitizeQuery().lowercase().trim() val foldedQuery = normalizedQuery.foldLatinDiacritics() - val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}" + val cacheKey = "$pathOrUri:$includeSubstringMatches:${normalizedQuery.sanitizeCacheKey()}" resultsCache.get(cacheKey)?.let { return@withContext it } @@ -118,9 +97,9 @@ class IndexSearcher(private val context: Context) { val count = metadata.totalCount if (count == 0) return@withContext emptyList() val results = openSource(pathOrUri).use { source -> - val prefixMatches = readPrefixMatches(source, metadata, normalizedQuery).entries.toMutableList() + val prefixMatches = readPrefixMatches(source, metadata, normalizedQuery).toMutableList() if (foldedQuery != normalizedQuery) { - prefixMatches.addAll(readPrefixMatches(source, metadata, foldedQuery).entries) + prefixMatches.addAll(readPrefixMatches(source, metadata, foldedQuery)) } val substringMatches = mutableListOf() @@ -165,81 +144,9 @@ class IndexSearcher(private val context: Context) { metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) } } - suspend fun searchWithCursors( - pathOrUri: String, - query: String - ): CursorSearchResult = withContext(Dispatchers.Default) { - if (query.isBlank()) { - return@withContext CursorSearchResult(emptyList(), emptyList()) - } - val normalizedQuery = query.sanitizeQuery().lowercase().trim() - val foldedQuery = normalizedQuery.foldLatinDiacritics() - val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}" - cursorResultsCache.get(cacheKey)?.let { - return@withContext it - } - try { - val metadata = metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) } - if (metadata.totalCount == 0) { - return@withContext CursorSearchResult(emptyList(), emptyList()) - } - val result = openSource(pathOrUri).use { source -> - val scans = buildList { - add(readPrefixMatches(source, metadata, normalizedQuery)) - if (foldedQuery != normalizedQuery) { - add(readPrefixMatches(source, metadata, foldedQuery)) - } - } - val entries = scans.flatMap { it.entries } - .rankBySearchRelevance(SearchRankingContext(normalizedQuery, foldedQuery)) - .distinctBy { it.articleKey() } - CursorSearchResult(entries, scans.mapNotNull { it.nextCursor }) - } - cursorResultsCache.put(cacheKey, result) - result - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - Log.e(TAG, "searchWithCursors() - error: ${e.message}", e) - if (e is DictionaryError) throw e - throw DictionaryError.SearchError.QueryFailed(context.getString(R.string.error_search_failed, query.take(20), pathOrUri.take(20)), e) - } - } - - suspend fun scanTailChunk( - pathOrUri: String, - cursor: RangeCursor, - maxEntries: Int - ): List = withContext(Dispatchers.Default) { - val metadata = metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) } - if (cursor.absoluteIndex >= metadata.totalCount) return@withContext emptyList() - openSource(pathOrUri).use { source -> - source.seek(cursor.fileOffset) - val out = ArrayList(maxEntries) - var currentIdx = cursor.absoluteIndex - var iterationCount = 0 - while (currentIdx < metadata.totalCount && out.size < maxEntries) { - if (iterationCount % 1000 == 0) yield() - val entry = readEntry(source, metadata) - val entryWordLower = entry.word.lowercase() - if (!entryWordLower.startsWith(cursor.rangeQuery)) break - currentIdx++ - iterationCount++ - val cursorAfter = if (currentIdx < metadata.totalCount) { - RangeCursor(cursor.rangeQuery, currentIdx, source.position()) - } else { - null - } - out.add(ScannedEntry(entry.copy(word = entryWordLower), cursorAfter)) - } - out - } - } - fun trimMemory(level: Int) { metadataCache.trim(level) resultsCache.trim(level) - cursorResultsCache.trim(level) } suspend fun isMetadataLoaded(pathOrUri: String): Boolean { @@ -349,9 +256,9 @@ class IndexSearcher(private val context: Context) { source: RandomAccessSource, metadata: IndexMetadata, query: String - ): RangeScanResult { + ): List { val firstMatch = findFirstMatch(source, metadata, query) - ?: return RangeScanResult(emptyList(), null) + ?: return emptyList() return readMatchingEntries(source, metadata, firstMatch, query) } @@ -415,26 +322,21 @@ class IndexSearcher(private val context: Context) { metadata: IndexMetadata, firstMatch: Pair, query: String - ): RangeScanResult { + ): List { val results = mutableListOf() results.add(firstMatch.second) - val uniqueArticles = hashSetOf(firstMatch.second.articleKey()) var currentIdx = firstMatch.first + 1 var iterationCount = 0 while (currentIdx < metadata.totalCount) { - if (uniqueArticles.size >= MAX_PREFIX_RESULTS) { - return RangeScanResult(results, RangeCursor(query, currentIdx, source.position())) - } if (iterationCount % 1000 == 0) yield() val entry = readEntry(source, metadata) val entryWordLower = entry.word.lowercase() if (!entryWordLower.startsWith(query)) break - uniqueArticles.add(entry.articleKey()) results.add(entry.copy(word = entryWordLower)) currentIdx++ iterationCount++ } - return RangeScanResult(results, null) + return results } private fun readPooledString(source: RandomAccessSource, length: Int): String { @@ -483,7 +385,6 @@ class IndexSearcher(private val context: Context) { private const val TAG = "IndexSearcher" private const val SUBSTRING_SCAN_MAX_ENTRIES = 200_000 private const val SPARSE_INTERVAL_V1 = 128 - private const val MAX_PREFIX_RESULTS = 200 private const val MAX_SUBSTRING_RESULTS = 100 } } @@ -588,30 +489,89 @@ interface RandomAccessSource : Closeable { } class FileSource(file: File) : RandomAccessSource { private val raf = RandomAccessFile(file, "r") + private val buffer = ByteArray(BUFFER_SIZE) + private var bufferStart = 0L + private var bufferEnd = 0L + private var pos = 0L override fun seek(pos: Long) { - raf.seek(pos) + this.pos = pos } - override fun position(): Long = raf.filePointer + override fun position(): Long = pos - override fun readUnsignedByte(): Int = raf.readUnsignedByte() + private fun refill(minNeeded: Int): Boolean { + if (bufferEnd - pos >= minNeeded) return true + bufferStart = pos + raf.seek(pos) + val read = raf.read(buffer) + bufferEnd = bufferStart + maxOf(read, 0) + return bufferEnd - pos >= minNeeded + } - override fun readUnsignedShort(): Int = raf.readUnsignedShort() + override fun readUnsignedByte(): Int { + if (!refill(1)) throw EOFException() + return buffer[(pos++ - bufferStart).toInt()].toInt() and 0xFF + } - override fun readInt(): Int = raf.readInt() + override fun readUnsignedShort(): Int { + if (!refill(2)) throw EOFException() + val i = (pos - bufferStart).toInt() + pos += 2 + return (buffer[i].toInt() and 0xFF shl 8) or (buffer[i + 1].toInt() and 0xFF) + } - override fun readLong(): Long = raf.readLong() + override fun readInt(): Int { + if (!refill(4)) throw EOFException() + val i = (pos - bufferStart).toInt() + pos += 4 + return (buffer[i].toInt() shl 24) or + (buffer[i + 1].toInt() and 0xFF shl 16) or + (buffer[i + 2].toInt() and 0xFF shl 8) or + (buffer[i + 3].toInt() and 0xFF) + } + + override fun readLong(): Long { + if (!refill(8)) throw EOFException() + val i = (pos - bufferStart).toInt() + pos += 8 + var value = 0L + for (k in 0 until 8) { + value = (value shl 8) or (buffer[i + k].toLong() and 0xFF) + } + return value + } override fun readFully(b: ByteArray) { - raf.readFully(b) + readFully(b, 0, b.size) } override fun readFully(b: ByteArray, off: Int, len: Int) { - raf.readFully(b, off, len) + if (len > BUFFER_SIZE / 2) { + raf.seek(pos) + raf.readFully(b, off, len) + pos += len + bufferEnd = bufferStart + return + } + var remaining = len + var dst = off + while (remaining > 0) { + if (!refill(1)) throw EOFException() + val i = (pos - bufferStart).toInt() + val chunk = minOf(remaining, (bufferEnd - pos).toInt()) + System.arraycopy(buffer, i, b, dst, chunk) + pos += chunk + dst += chunk + remaining -= chunk + } } override fun close() { raf.close() } + + private companion object { + private const val BUFFER_SIZE = 64 * 1024 + } } diff --git a/app/src/main/java/com/example/research/ui/main/components/SearchResultsList.kt b/app/src/main/java/com/example/research/ui/main/components/SearchResultsList.kt index c5dabe6..9b47292 100644 --- a/app/src/main/java/com/example/research/ui/main/components/SearchResultsList.kt +++ b/app/src/main/java/com/example/research/ui/main/components/SearchResultsList.kt @@ -6,17 +6,13 @@ import androidx.compose.foundation.lazy.* import androidx.compose.material3.* import androidx.compose.runtime.Composable import androidx.compose.runtime.remember -import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.testTag -import androidx.compose.ui.res.stringResource import androidx.compose.ui.semantics.semantics import androidx.compose.ui.semantics.testTagsAsResourceId import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp -import androidx.paging.LoadState import androidx.paging.compose.* -import com.example.research.R import com.example.research.common.ui.theme.dictionaryTitleLarge import com.example.research.core.domain.model.IndexEntry import com.example.research.core.util.removeDictionarySuffixes @@ -53,7 +49,7 @@ fun SearchResultsList( contentType = { "search_result" } ) { index -> val entry = if (showStale) staleEntries.getOrNull(index) - else if (index in 0 until results.itemCount) results.peek(index) else null + else if (index in 0 until results.itemCount) results[index] else null if (entry != null) { SearchResultItem( entry = entry, @@ -63,26 +59,6 @@ fun SearchResultsList( ) } } - - when (results.loadState.append) { - is LoadState.Error -> { - item { - Box( - modifier = Modifier - .fillMaxWidth() - .padding(16.dp), - contentAlignment = Alignment.Center - ) { - Text( - text = stringResource(R.string.error_loading_more), - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.error - ) - } - } - } - else -> {} - } } } diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index ef0cc31..6454402 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -62,7 +62,6 @@ Не удалось просканировать папку со словарями Ошибка поиска запроса \'%1$s\' в %2$s Не удалось загрузить метаданные индекса: %1$s - Ошибка загрузки %1$d активный словарь %1$d активных словаря diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index b9e55d4..036097b 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -62,7 +62,6 @@ Failed to scan dictionary folder Search failed for query \'%1$s\' in %2$s Failed to load index metadata: %1$s - Loading error %1$d active dictionary %1$d active dictionaries