5 Commits
Author SHA1 Message Date
OneWay 910bc8313c Release 1.4.0
- Fixed matching source URLs to installed dictionary files, so deleted dictionaries can be downloaded again from the same URL
2026-08-06 14:26:03 +03:00
OneWay 9d18d0c927 Release 1.3.0
- Blocked swipe-to-delete on dictionary rows while a download, extract,
  import or indexing pass is running. Deleting mid-update removed the
  dictionary's source from preferences while the pipeline kept its own
  snapshot, so the post-update rescan re-created the dictionary with its
  update source permanently lost.
- Fixed the dictionary icon leaving the row while swiping to delete: the
  item now clips to its bounds and the delete button is a static overlay
  instead of a counter-translated child of the sliding row.
2026-08-02 11:09:47 +03:00
OneWay 69bbb97b2a Release 1.2.1
- Fixed the dictionary update card taking too long to reappear after
  cancelling a download (DownloadManager).
- Bump agp 9.3.0 -> 9.3.1; migrate the release build type from
  isMinifyEnabled/isShrinkResources/proguardFiles to
  optimization { enable = true } with keep rules moved to
  app/src/release/keepRules/proguard-rules.keep.
2026-07-31 10:47:29 +03:00
OneWay c8adf537d4 Add app icon to fastlane metadata 2026-07-30 19:46:12 +03:00
OneWay 84612b26f2 Release 1.2.0
- Fixed a crash on broad queries (e.g. a single letter across several
  dictionaries): the results list keyed items by dictionary
  path/offset/word, and duplicate keys are illegal in a Compose list;
  drop the explicit key, let Paging identify items by position instead
  (SearchResultsList).
- Fixed a disabled dictionary reactivating itself on the next app
  launch: the active flag is now persisted through
  PreferencesManager/DataStore and reapplied whenever the dictionary
  list is rebuilt (LocalDictionaryRepository).
- Search results no longer stop at the first ~100-200 matches: the
  ranked head page hands back a cursor per unfinished prefix range,
  and IndexEntryPagingSource pages the rest straight from the index,
  merged alphabetically across dictionaries, until every match is
  surfaced (IndexSearcher, IndexEntryPagingSource).
- Added diacritic-insensitive search: folded index entries are merged
  into the primary index as aliases, so accented and plain spellings
  find each other (DslIndexer, IndexSearcher).
- Fixed indexing failures on dictionaries with dense non-Latin text: a
  truncated multi-byte sequence at a sample boundary was misread as
  invalid UTF-8 and the whole file fell back to UTF-16LE
  (DslCharsetDetector).
2026-07-22 16:29:16 +03:00
40 changed files with 894 additions and 395 deletions
+3 -3
View File
@@ -94,7 +94,7 @@ The main search screen remains usable as long as at least one indexed dictionary
## Tech Stack ## Tech Stack
- Kotlin 2.4.0 - Kotlin 2.4.10
- Jetpack Compose (BOM 2026.06.01) - Jetpack Compose (BOM 2026.06.01)
- Material 3 - Material 3
- Coroutines and Flow 1.11.0 - Coroutines and Flow 1.11.0
@@ -102,8 +102,8 @@ The main search screen remains usable as long as at least one indexed dictionary
- Paging 3.5.0 - Paging 3.5.0
- OkHttp 5.4.0 - OkHttp 5.4.0
- kotlinx.serialization 1.11.0 - kotlinx.serialization 1.11.0
- AboutLibraries 15.0.3 metadata generation - AboutLibraries 15.0.4 metadata generation
- Android Gradle Plugin 9.2.1 - Android Gradle Plugin 9.3.0
## Build ## Build
+5 -14
View File
@@ -92,14 +92,8 @@ android {
applicationId = "com.example.research" applicationId = "com.example.research"
minSdk = project.property("minSdk").toString().toInt() minSdk = project.property("minSdk").toString().toInt()
targetSdk = project.property("targetSdk").toString().toInt() targetSdk = project.property("targetSdk").toString().toInt()
versionCode = 3 versionCode = 7
versionName = "1.1.1" versionName = "1.4.0"
val isFdroid = project.hasProperty("fdroid")
if (isFdroid) {
applicationIdSuffix = ".fdroid"
versionNameSuffix = "-fdroid"
}
vectorDrawables { vectorDrawables {
useSupportLibrary = true useSupportLibrary = true
@@ -120,12 +114,9 @@ android {
abiFilters += "arm64-v8a" abiFilters += "arm64-v8a"
} }
isMinifyEnabled = true optimization {
isShrinkResources = true enable = true
proguardFiles( }
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-rules.pro"
)
} }
} }
@@ -45,7 +45,7 @@ class ReSearchApplication : Application() {
applicationScope.launch(Dispatchers.IO) { applicationScope.launch(Dispatchers.IO) {
preferencesManager.sanitizeDictionarySources() preferencesManager.sanitizeDictionarySources()
} }
localDictionaryRepository = LocalDictionaryRepository(this) localDictionaryRepository = LocalDictionaryRepository(this, preferencesManager = preferencesManager)
okHttpClient = OkHttpClient.Builder() okHttpClient = OkHttpClient.Builder()
.protocols(listOf(Protocol.HTTP_1_1)) .protocols(listOf(Protocol.HTTP_1_1))
@@ -13,7 +13,7 @@ object DateUtils {
return dateFormat.format(calendar.time) return dateFormat.format(calendar.time)
} }
fun extractDateFromFileName(fileName: String): String? { fun extractDateFromFileName(fileName: String): String? {
val regex = Regex("""_(\d{6})\.(gz|dsl|idx|dsl\.dz|dsl\.idx|dsl\.gz)$""") val regex = Regex("""_(\d{6})\.(gz|dsl|idx|idx\.fold|dsl\.dz|dsl\.idx|dsl\.idx\.fold|dsl\.gz)$""")
return regex.find(fileName)?.groupValues?.get(1) return regex.find(fileName)?.groupValues?.get(1)
} }
fun isDateBefore(date1: String, date2: String): Boolean { fun isDateBefore(date1: String, date2: String): Boolean {
@@ -13,7 +13,8 @@ data class IndexEntry(
val offset: ArticleOffset = ArticleOffset.ZERO, val offset: ArticleOffset = ArticleOffset.ZERO,
val length: ArticleLength = ArticleLength.ZERO, val length: ArticleLength = ArticleLength.ZERO,
val dictionaryName: String = "", val dictionaryName: String = "",
val dictionaryPath: String = "" val dictionaryPath: String = "",
val isAlias: Boolean = false
) : Comparable<IndexEntry> { ) : Comparable<IndexEntry> {
init { init {
require(word.isNotBlank()) { "Word cannot be blank" } require(word.isNotBlank()) { "Word cannot be blank" }
@@ -0,0 +1,27 @@
package com.example.research.core.domain.usecase
import com.example.research.core.domain.model.Dictionary
import com.example.research.core.domain.model.DictionarySource
import java.io.File
object DictionarySourceFileMatcher {
fun matches(source: DictionarySource, fileName: String): Boolean =
DictionarySource.matchesDictionaryFile(source.urlTemplate, fileName)
fun matches(source: DictionarySource, dictionary: Dictionary): Boolean =
matches(source, File(dictionary.path).name)
fun installedSources(
sources: List<DictionarySource>,
dictionaries: List<Dictionary>,
): List<DictionarySource> = sources.filter { source ->
dictionaries.any { dictionary -> matches(source, dictionary) }
}
fun installedSourcesForFileNames(
sources: List<DictionarySource>,
fileNames: Collection<String>,
): List<DictionarySource> = sources.filter { source ->
fileNames.any { fileName -> matches(source, fileName) }
}
}
@@ -43,12 +43,4 @@ class DictionarySourceValidator {
return ValidationResult.Valid return ValidationResult.Valid
} }
fun findMatchingSourceWithPrecomputedPrefixes(
dictionaryPrefix: String,
sourcesWithPrefixes: List<Pair<DictionarySource, String?>>
): DictionarySource? {
return sourcesWithPrefixes.find { (_, sourcePrefix) ->
sourcePrefix != null && dictionaryPrefix.equals(sourcePrefix, ignoreCase = true)
}?.first
}
} }
@@ -30,18 +30,24 @@ class ManageDictionarySourcesUseCase(
preferencesManager.removeDictionarySource(sourceId) preferencesManager.removeDictionarySource(sourceId)
} }
suspend fun removeSourceForDictionary(dictionary: Dictionary) { suspend fun removeSourceForDictionary(
val dictionaryPrefix = dictionary.name dictionary: Dictionary,
val sources = preferencesManager.dictionarySources.first() remainingDictionaryFileNames: Collection<String>?,
val sourcesWithPrefixes = sources.map { source -> ) {
source to DictionarySource.extractPrefix(source.urlTemplate) if (remainingDictionaryFileNames == null) return
}
val matchingSource = validator.findMatchingSourceWithPrecomputedPrefixes(dictionaryPrefix, sourcesWithPrefixes) val sources = preferencesManager.dictionarySources.first()
if (matchingSource != null) { val sourceIdsToRemove = sources
preferencesManager.removeDictionarySource(matchingSource.id) .filter { source ->
DictionarySourceFileMatcher.matches(source, dictionary) &&
remainingDictionaryFileNames.none { fileName ->
DictionarySourceFileMatcher.matches(source, fileName)
} }
} }
.map(DictionarySource::id)
preferencesManager.removeDictionarySources(sourceIdsToRemove)
}
sealed class AddSourceResult { sealed class AddSourceResult {
object Success : AddSourceResult() object Success : AddSourceResult()
@@ -9,6 +9,32 @@ fun String.sanitizeQuery(): String {
.take(MAX_QUERY_LENGTH) .take(MAX_QUERY_LENGTH)
} }
fun String.foldLatinDiacritics(): String {
val decomposed = java.text.Normalizer.normalize(this, java.text.Normalizer.Form.NFD)
val folded = StringBuilder(decomposed.length)
var followsLatinBase = false
var index = 0
while (index < decomposed.length) {
val codePoint = decomposed.codePointAt(index)
val type = Character.getType(codePoint)
val isMark = type == Character.NON_SPACING_MARK.toInt() ||
type == Character.COMBINING_SPACING_MARK.toInt() ||
type == Character.ENCLOSING_MARK.toInt()
if (!(isMark && followsLatinBase)) {
folded.appendCodePoint(codePoint)
}
if (!isMark) {
followsLatinBase = Character.UnicodeScript.of(codePoint) == Character.UnicodeScript.LATIN
}
index += Character.charCount(codePoint)
}
return java.text.Normalizer.normalize(folded, java.text.Normalizer.Form.NFC)
}
fun String.sanitizeCacheKey(): String { fun String.sanitizeCacheKey(): String {
return this.replace(":", "_").replace("\n", "_").replace("\t", "_") return this.replace(":", "_").replace("\n", "_").replace("\t", "_")
} }
@@ -6,6 +6,7 @@ import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.core.edit import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.booleanPreferencesKey import androidx.datastore.preferences.core.booleanPreferencesKey
import androidx.datastore.preferences.core.stringPreferencesKey import androidx.datastore.preferences.core.stringPreferencesKey
import androidx.datastore.preferences.core.stringSetPreferencesKey
import androidx.datastore.preferences.preferencesDataStore import androidx.datastore.preferences.preferencesDataStore
import com.example.research.core.domain.model.DictionarySource import com.example.research.core.domain.model.DictionarySource
import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.Flow
@@ -23,6 +24,7 @@ class PreferencesManager(private val context: Context) {
val IS_THEME_EXPANDED = booleanPreferencesKey("is_theme_expanded") val IS_THEME_EXPANDED = booleanPreferencesKey("is_theme_expanded")
val IS_LANGUAGE_EXPANDED = booleanPreferencesKey("is_language_expanded") val IS_LANGUAGE_EXPANDED = booleanPreferencesKey("is_language_expanded")
val IS_DICTIONARIES_EXPANDED = booleanPreferencesKey("is_dictionaries_expanded") val IS_DICTIONARIES_EXPANDED = booleanPreferencesKey("is_dictionaries_expanded")
val DISABLED_DICTIONARY_PATHS = stringSetPreferencesKey("disabled_dictionary_paths")
} }
private val json = Json { ignoreUnknownKeys = true } private val json = Json { ignoreUnknownKeys = true }
@@ -60,6 +62,33 @@ class PreferencesManager(private val context: Context) {
} }
} }
val disabledDictionaryPaths: Flow<Set<String>> = context.dataStore.data
.map { preferences ->
preferences[PreferencesKeys.DISABLED_DICTIONARY_PATHS] ?: emptySet()
}
suspend fun setDictionaryActive(path: String, isActive: Boolean) {
context.dataStore.edit { preferences ->
val disabledPaths = preferences[PreferencesKeys.DISABLED_DICTIONARY_PATHS]
.orEmpty()
.toMutableSet()
if (isActive) {
disabledPaths.remove(path)
} else {
disabledPaths.add(path)
}
if (disabledPaths.isEmpty()) {
preferences.remove(PreferencesKeys.DISABLED_DICTIONARY_PATHS)
} else {
preferences[PreferencesKeys.DISABLED_DICTIONARY_PATHS] = disabledPaths
}
}
}
suspend fun removeDictionaryActiveState(path: String) {
setDictionaryActive(path, true)
}
suspend fun saveThemeMode(theme: String) { suspend fun saveThemeMode(theme: String) {
context.dataStore.edit { preferences -> context.dataStore.edit { preferences ->
preferences[PreferencesKeys.THEME_MODE] = theme preferences[PreferencesKeys.THEME_MODE] = theme
@@ -15,74 +15,50 @@ import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
import java.io.File import java.io.File
/**
* Two-phase search paging.
*
* Page one (key = null) is the ranked head: the capped, relevance-ordered result the
* search screen has always shown, merged across active dictionaries. Every range scan
* that hit its cap hands back a [IndexSearcher.RangeCursor]; those become the tail key.
*
* Tail pages continue the unfinished range scans directly from the index files and
* merge them alphabetically (k-way, by index word), so scrolling eventually surfaces
* every prefix match in every active dictionary. Articles already shown -- by the head
* or by an earlier tail page, including diacritic alias duplicates -- are filtered
* through [emittedArticles], which grows only as far as the user actually scrolls.
*
* A new query builds a new PagingSource (see SearchViewModel's flatMapLatest), so an
* in-flight head or tail load is cancelled by Paging when the user keeps typing.
*/
class IndexEntryPagingSource( class IndexEntryPagingSource(
private val indexSearcher: IndexSearcher, private val indexSearcher: IndexSearcher,
private val dictionaries: List<Dictionary>, private val dictionaries: List<Dictionary>,
private val query: String private val query: String
) : PagingSource<Int, IndexEntry>() { ) : PagingSource<IndexEntryPagingSource.TailKey, IndexEntry>() {
private var searchResults: List<IndexEntry>? = null data class StreamCursor(
val dictPosition: Int,
val cursor: IndexSearcher.RangeCursor
)
override val jumpingSupported: Boolean = true data class TailKey(val streams: List<StreamCursor>)
override suspend fun load(params: LoadParams<Int>): LoadResult<Int, IndexEntry> { private data class EmittedKey(
val offset = params.key ?: 0 val dictPosition: Int,
val originalWord: String,
val offset: Long,
val length: Int
)
private val emittedArticles = HashSet<EmittedKey>()
private val activeDictionaries = dictionaries.withIndex().filter { it.value.isActive }
override suspend fun load(params: LoadParams<TailKey>): LoadResult<TailKey, IndexEntry> {
return try { return try {
val allResults = searchResults ?: ReSearchTrace.asyncSection(ReSearchTrace.SEARCH_PAGING) { val key = params.key
coroutineScope { if (key == null) loadHead() else loadTail(key, params.loadSize)
val rawResults = dictionaries
.filter { it.isActive }
.map { dict ->
async {
val indexFile = File(dict.indexPath)
if (!indexFile.exists()) return@async emptyList()
try {
val results = indexSearcher.search(
pathOrUri = indexFile.absolutePath,
query = query,
includeSubstringMatches = false
)
results.map {
it.withDictionary(
dict.metadata.name.ifBlank { dict.name },
dict.path
)
}
} catch (e: CancellationException) {
throw e
} catch (_: Exception) {
emptyList()
}
}
}
.awaitAll()
.flatten()
// Precompute sorting keys to optimize comparator performance.
// entry.word is already lowercase from indexing.
val normalizedQuery = query.sanitizeQuery().lowercase().trim()
rawResults.rankBySearchRelevance(
SearchRankingContext(normalizedQuery)
)
}
}.also { searchResults = it }
val totalSize = allResults.size
val startIndex = offset.coerceAtLeast(0).coerceAtMost(totalSize)
val endIndex = (startIndex + params.loadSize).coerceAtMost(totalSize)
val pageItems = allResults.subList(startIndex, endIndex)
val hasMore = endIndex < totalSize
val nextKey = if (hasMore) endIndex else null
val prevKey = if (offset == 0) null else (offset - params.loadSize).coerceAtLeast(0)
LoadResult.Page(
data = pageItems,
prevKey = prevKey,
nextKey = nextKey
)
} catch (e: CancellationException) { } catch (e: CancellationException) {
throw e throw e
} catch (e: Exception) { } catch (e: Exception) {
@@ -90,10 +66,129 @@ class IndexEntryPagingSource(
} }
} }
override fun getRefreshKey(state: PagingState<Int, IndexEntry>): Int? { private suspend fun loadHead(): LoadResult<TailKey, IndexEntry> =
return state.anchorPosition?.let { anchorPosition -> ReSearchTrace.asyncSection(ReSearchTrace.SEARCH_PAGING) {
state.closestPageToPosition(anchorPosition)?.prevKey?.plus(state.config.pageSize) coroutineScope {
?: state.closestPageToPosition(anchorPosition)?.nextKey?.minus(state.config.pageSize) val perDictionary = activeDictionaries
.map { (position, dict) ->
async {
val indexFile = File(dict.indexPath)
if (!indexFile.exists()) return@async null
try {
val result = indexSearcher.searchWithCursors(
pathOrUri = indexFile.absolutePath,
query = query
)
Triple(position, dict, result)
} catch (e: CancellationException) {
throw e
} catch (_: Exception) {
null
} }
} }
}
.awaitAll()
.filterNotNull()
val rawResults = perDictionary.flatMap { (position, dict, result) ->
result.entries.map { entry ->
position to entry.withDictionary(
dict.metadata.name.ifBlank { dict.name },
dict.path
)
}
}
val normalizedQuery = query.sanitizeQuery().lowercase().trim()
val ranked = rawResults.map { it.second }
.rankBySearchRelevance(SearchRankingContext(normalizedQuery))
rawResults.forEach { (position, entry) -> emittedArticles.add(entry.emittedKey(position)) }
val streams = perDictionary.flatMap { (position, _, result) ->
result.cursors.map { StreamCursor(position, it) }
}
LoadResult.Page(
data = ranked,
prevKey = null,
nextKey = if (streams.isEmpty()) null else TailKey(streams)
)
}
}
private suspend fun loadTail(key: TailKey, loadSize: Int): LoadResult<TailKey, IndexEntry> {
val streams = key.streams.map { TailStream(it.dictPosition, it.cursor) }
val page = ArrayList<IndexEntry>(loadSize)
val pageKeys = HashSet<EmittedKey>()
val chunkSize = maxOf(loadSize, MIN_CHUNK)
while (page.size < loadSize) {
streams.forEach { it.ensureBuffered(chunkSize) }
val next = streams
.filter { it.hasBuffered }
.minByOrNull { it.peekWord }
?: break
val scanned = next.consume()
val dict = dictionaries[next.dictPosition]
val entry = scanned.entry.withDictionary(
dict.metadata.name.ifBlank { dict.name },
dict.path
)
val emittedKey = entry.emittedKey(next.dictPosition)
if (emittedKey !in emittedArticles && pageKeys.add(emittedKey)) {
page.add(entry)
}
}
emittedArticles.addAll(pageKeys)
val remaining = streams.mapNotNull { stream ->
stream.nextCursor?.let { StreamCursor(stream.dictPosition, it) }
}
return LoadResult.Page(
data = page,
prevKey = null,
nextKey = if (remaining.isEmpty()) null else TailKey(remaining)
)
}
private inner class TailStream(
val dictPosition: Int,
initialCursor: IndexSearcher.RangeCursor
) {
var nextCursor: IndexSearcher.RangeCursor? = initialCursor
private set
private var buffer: List<IndexSearcher.ScannedEntry> = emptyList()
private var position = 0
val hasBuffered: Boolean get() = position < buffer.size
val peekWord: String get() = buffer[position].entry.word
suspend fun ensureBuffered(chunkSize: Int) {
if (hasBuffered) return
val cursor = nextCursor ?: return
val indexPath = File(dictionaries[dictPosition].indexPath).absolutePath
buffer = indexSearcher.scanTailChunk(indexPath, cursor, chunkSize)
position = 0
if (buffer.isEmpty()) nextCursor = null
}
fun consume(): IndexSearcher.ScannedEntry {
val scanned = buffer[position]
position++
nextCursor = scanned.cursorAfter
return scanned
}
}
private fun IndexEntry.emittedKey(dictPosition: Int) =
EmittedKey(dictPosition, originalWord, offset.value, length.value)
override fun getRefreshKey(state: PagingState<TailKey, IndexEntry>): TailKey? {
return null
}
} }
private const val MIN_CHUNK = 48
@@ -58,17 +58,23 @@ object DslCharsetDetector {
private fun looksLikeUtf8(inputStream: java.io.InputStream): Boolean { private fun looksLikeUtf8(inputStream: java.io.InputStream): Boolean {
if (!inputStream.markSupported()) return false if (!inputStream.markSupported()) return false
inputStream.mark(8192) inputStream.mark(8192)
return try { try {
val sample = ByteArray(8192) val sample = ByteArray(8192)
val read = inputStream.read(sample) val read = inputStream.read(sample)
if (read <= 0) return false if (read <= 0) return false
val decoder = Charsets.UTF_8.newDecoder() val decoder = Charsets.UTF_8.newDecoder()
.onMalformedInput(java.nio.charset.CodingErrorAction.REPORT) .onMalformedInput(java.nio.charset.CodingErrorAction.REPORT)
.onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPORT) .onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPORT)
decoder.decode(java.nio.ByteBuffer.wrap(sample, 0, read)) val byteBuffer = java.nio.ByteBuffer.wrap(sample, 0, read)
true val charBuffer = java.nio.CharBuffer.allocate(8192)
while (true) {
val result = decoder.decode(byteBuffer, charBuffer, false)
if (result.isError) return false
if (result.isUnderflow) return true
charBuffer.clear()
}
} catch (_: Exception) { } catch (_: Exception) {
false return false
} finally { } finally {
inputStream.reset() inputStream.reset()
} }
@@ -4,6 +4,7 @@ import android.util.Log
import com.example.research.core.domain.model.ArticleLength import com.example.research.core.domain.model.ArticleLength
import com.example.research.core.domain.model.ArticleOffset import com.example.research.core.domain.model.ArticleOffset
import com.example.research.core.domain.model.IndexEntry import com.example.research.core.domain.model.IndexEntry
import com.example.research.core.util.foldLatinDiacritics
import com.example.research.core.util.removeAccentTagsForIndexing import com.example.research.core.util.removeAccentTagsForIndexing
import kotlinx.coroutines.Deferred import kotlinx.coroutines.Deferred
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
@@ -23,7 +24,7 @@ class DslIndexer(context: Context? = null) {
companion object { companion object {
private const val TAG = "DslIndexer" private const val TAG = "DslIndexer"
const val INDEX_VERSION = 4 const val INDEX_VERSION = 6
const val BUFFER_SIZE = 1048576 const val BUFFER_SIZE = 1048576
const val BATCH_BUFFER_SIZE = 524288 const val BATCH_BUFFER_SIZE = 524288
private const val INDEX_SORT_PARALLELISM = 1 private const val INDEX_SORT_PARALLELISM = 1
@@ -40,6 +41,9 @@ class DslIndexer(context: Context? = null) {
private const val IN_MEMORY_SORT_MAX_ENTRIES = 140_000 private const val IN_MEMORY_SORT_MAX_ENTRIES = 140_000
private const val ESTIMATED_IN_MEMORY_ENTRY_BYTES = 768L private const val ESTIMATED_IN_MEMORY_ENTRY_BYTES = 768L
private const val IN_MEMORY_SORT_HEAP_RESERVE_BYTES = 96L * 1024L * 1024L private const val IN_MEMORY_SORT_HEAP_RESERVE_BYTES = 96L * 1024L * 1024L
fun foldedIndexFile(indexFile: File): File =
File(indexFile.parent ?: ".", "${indexFile.name}.fold")
} }
suspend fun createIndex( suspend fun createIndex(
@@ -56,21 +60,25 @@ class DslIndexer(context: Context? = null) {
java.io.BufferedInputStream(dslInputStream) java.io.BufferedInputStream(dslInputStream)
} }
val tempFile = File(indexFile.parent, "${indexFile.name}.tmp") val tempFile = File(indexFile.parent, "${indexFile.name}.tmp")
val entryCount: Int val counts: ParsedCounts
try { try {
try { foldedIndexFile(indexFile).delete() } catch (_: Throwable) { /* Legacy v5 leftover */ }
onOperationChange("Parsing file") onOperationChange("Parsing file")
onFileProgress(0.0f) onFileProgress(0.0f)
entryCount = parseDslToTempFile(bufferedStream, tempFile, counts = parseDslToTempFile(bufferedStream, tempFile,
onProgress, onFileProgress, onCharsetDetected) onProgress, onFileProgress, onCharsetDetected)
onFileProgress(0.5f) onFileProgress(0.5f)
onOperationChange("Sorting index") onOperationChange("Sorting index")
sortAndWriteIndexFile(tempFile, indexFile, entryCount, onOperationChange, onFileProgress) sortAndWriteIndexFile(tempFile, indexFile, counts, onOperationChange, onFileProgress)
onFileProgress(1f)
} finally { } finally {
try { tempFile.delete() } catch (_: Throwable) { /* Ignore cleanup failure */ } try { tempFile.delete() } catch (_: Throwable) { /* Ignore cleanup failure */ }
} }
try { onProgress(entryCount) } catch (_: Throwable) { /* Ignore progress callback failure */ } try { onProgress(counts.articleCount) } catch (_: Throwable) { /* Ignore progress callback failure */ }
return entryCount return counts.articleCount
} }
private data class ParsedCounts(val articleCount: Int, val totalEntries: Int)
// Blocking file I/O is safe across this file: every suspend fun here is reached only // Blocking file I/O is safe across this file: every suspend fun here is reached only
// from createIndex(), which the sole caller (KotlinDictionaryEngine) runs inside // from createIndex(), which the sole caller (KotlinDictionaryEngine) runs inside
// withContext(Dispatchers.IO). The dispatcher is invisible across the suspend-call // withContext(Dispatchers.IO). The dispatcher is invisible across the suspend-call
@@ -83,7 +91,7 @@ class DslIndexer(context: Context? = null) {
onProgress: (Int) -> Unit, onProgress: (Int) -> Unit,
onFileProgress: (Float) -> Unit = {}, onFileProgress: (Float) -> Unit = {},
onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {} onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {}
): Int { ): ParsedCounts {
val detectedCharset = DslCharsetDetector.detect(inputStream) val detectedCharset = DslCharsetDetector.detect(inputStream)
onCharsetDetected(detectedCharset) onCharsetDetected(detectedCharset)
@@ -98,6 +106,7 @@ class DslIndexer(context: Context? = null) {
) )
var entryCount = 0 var entryCount = 0
var aliasCount = 0
var lineCount = 0 var lineCount = 0
var headwordLineCount = 0 var headwordLineCount = 0
var bodyLineCount = 0 var bodyLineCount = 0
@@ -127,9 +136,10 @@ class DslIndexer(context: Context? = null) {
fun normalizeSearchableWord(searchableText: String, fallback: String): String { fun normalizeSearchableWord(searchableText: String, fallback: String): String {
// Parser already lowercases and strips accent tags for non-empty results; // Parser already lowercases and strips accent tags for non-empty results;
// only the raw-headword fallback still needs normalization. // only the raw-headword fallback still needs normalization.
return searchableText.ifEmpty { val normalized = searchableText.ifEmpty {
fallback.removeAccentTagsForIndexing().lowercase() fallback.removeAccentTagsForIndexing().lowercase()
} }
return java.text.Normalizer.normalize(normalized, java.text.Normalizer.Form.NFC)
} }
DataOutputStream(BufferedOutputStream(FileOutputStream(tempFile), BUFFER_SIZE)).use { output -> DataOutputStream(BufferedOutputStream(FileOutputStream(tempFile), BUFFER_SIZE)).use { output ->
@@ -144,12 +154,23 @@ class DslIndexer(context: Context? = null) {
return return
} }
val length = (groupEndOffset - groupStartOffset).coerceAtLeast(0L).toInt() val length = (groupEndOffset - groupStartOffset).coerceAtLeast(0L).toInt()
for (idx in groupSearchable.indices) { fun writeTempEntry(word: String, display: String, isAlias: Boolean) {
writer.write(groupSearchable[idx]) writer.write(word)
writer.write(groupDisplay[idx]) writer.write(display)
output.writeLong(groupStartOffset) output.writeLong(groupStartOffset)
output.writeInt(length) output.writeInt(length)
output.writeByte(if (isAlias) ENTRY_FLAG_ALIAS else 0)
}
for (idx in groupSearchable.indices) {
val searchableWord = groupSearchable[idx]
val displayWord = groupDisplay[idx]
writeTempEntry(searchableWord, displayWord, isAlias = false)
entryCount++ entryCount++
val foldedWord = searchableWord.foldLatinDiacritics()
if (foldedWord != searchableWord) {
writeTempEntry(foldedWord, displayWord, isAlias = true)
aliasCount++
}
} }
groupSearchable.clear() groupSearchable.clear()
groupDisplay.clear() groupDisplay.clear()
@@ -230,26 +251,28 @@ class DslIndexer(context: Context? = null) {
try { onProgress(entryCount) } catch (_: Throwable) { /* Ignore progress callback failure */ } try { onProgress(entryCount) } catch (_: Throwable) { /* Ignore progress callback failure */ }
} }
onFileProgress(0.5f) onFileProgress(0.5f)
return entryCount return ParsedCounts(articleCount = entryCount, totalEntries = entryCount + aliasCount)
} }
// Blocking file I/O safe: reached only from createIndex() (runs on Dispatchers.IO). See parseDslToTempFile(). // Blocking file I/O safe: reached only from createIndex() (runs on Dispatchers.IO). See parseDslToTempFile().
@Suppress("BlockingMethodInNonBlockingContext") @Suppress("BlockingMethodInNonBlockingContext")
private suspend fun sortAndWriteIndexFile(tempFile: File, indexFile: File, entryCount: Int, onOperationChange: (String) -> Unit = {}, onFileProgress: (Float) -> Unit = {}) { private suspend fun sortAndWriteIndexFile(tempFile: File, indexFile: File, counts: ParsedCounts, onOperationChange: (String) -> Unit = {}, onFileProgress: (Float) -> Unit = {}) {
if (entryCount == 0) { if (counts.totalEntries == 0) {
DataOutputStream(BufferedOutputStream(FileOutputStream(indexFile))).use { output -> DataOutputStream(BufferedOutputStream(FileOutputStream(indexFile))).use { output ->
output.writeInt(INDEX_VERSION) output.writeInt(INDEX_VERSION)
output.writeInt(0) output.writeInt(0)
output.writeInt(0) output.writeInt(0)
output.writeInt(0)
} }
return return
} }
if (shouldUseInMemorySort(entryCount)) { if (shouldUseInMemorySort(counts.totalEntries)) {
inMemorySortAndWrite(tempFile, indexFile, entryCount) inMemorySortAndWrite(tempFile, indexFile, counts)
onFileProgress(FINAL_INDEX_DONE_PROGRESS) onFileProgress(FINAL_INDEX_DONE_PROGRESS)
} else { } else {
externalSortAndWrite(tempFile, indexFile, entryCount, onOperationChange, onFileProgress) externalSortAndWrite(tempFile, indexFile, counts, onOperationChange, onFileProgress)
} }
} }
private fun shouldUseInMemorySort(entryCount: Int): Boolean { private fun shouldUseInMemorySort(entryCount: Int): Boolean {
if (entryCount <= IN_MEMORY_SORT_ALWAYS_ENTRIES) return true if (entryCount <= IN_MEMORY_SORT_ALWAYS_ENTRIES) return true
if (entryCount > IN_MEMORY_SORT_MAX_ENTRIES) return false if (entryCount > IN_MEMORY_SORT_MAX_ENTRIES) return false
@@ -264,30 +287,22 @@ class DslIndexer(context: Context? = null) {
} }
// Blocking file I/O safe: reached only from createIndex() (runs on Dispatchers.IO). See parseDslToTempFile(). // Blocking file I/O safe: reached only from createIndex() (runs on Dispatchers.IO). See parseDslToTempFile().
@Suppress("BlockingMethodInNonBlockingContext") @Suppress("BlockingMethodInNonBlockingContext")
private suspend fun inMemorySortAndWrite(tempFile: File, indexFile: File, entryCount: Int) { private suspend fun inMemorySortAndWrite(tempFile: File, indexFile: File, counts: ParsedCounts) {
val entries = ArrayList<IndexEntry>(entryCount) val entries = ArrayList<IndexEntry>(counts.totalEntries)
java.io.DataInputStream(java.io.BufferedInputStream(java.io.FileInputStream(tempFile), BATCH_BUFFER_SIZE)).use { input -> java.io.DataInputStream(java.io.BufferedInputStream(java.io.FileInputStream(tempFile), BATCH_BUFFER_SIZE)).use { input ->
repeat(entryCount) { repeat(counts.totalEntries) {
val word = input.readUtf8String() entries.add(input.readTempEntry())
val originalWord = input.readUtf8String()
val offset = input.readLong()
val length = input.readInt()
entries.add(IndexEntry(
word = word,
originalWord = originalWord,
offset = ArticleOffset(offset),
length = ArticleLength(length)
))
if (entries.size % 5000 == 0) { if (entries.size % 5000 == 0) {
try { yield() } catch (_: Throwable) { /* Ignore yield failure */ } try { yield() } catch (_: Throwable) { /* Ignore yield failure */ }
} }
} }
} }
entries.sort() entries.sort()
writeIndexFile(indexFile, entries) writeIndexFile(indexFile, entries, counts.articleCount)
entries.clear() entries.clear()
} }
private suspend fun externalSortAndWrite(tempFile: File, indexFile: File, entryCount: Int, onOperationChange: (String) -> Unit, onFileProgress: (Float) -> Unit) { private suspend fun externalSortAndWrite(tempFile: File, indexFile: File, counts: ParsedCounts, onOperationChange: (String) -> Unit, onFileProgress: (Float) -> Unit) {
val entryCount = counts.totalEntries
val adaptiveConfig = resourceManager?.calculateAdaptiveConfig(estimatedEntryCount = entryCount.toLong()) val adaptiveConfig = resourceManager?.calculateAdaptiveConfig(estimatedEntryCount = entryCount.toLong())
?: ResourceManager.AdaptiveConfig( ?: ResourceManager.AdaptiveConfig(
batchSize = 100_000, batchSize = 100_000,
@@ -317,16 +332,7 @@ class DslIndexer(context: Context? = null) {
val batch = ArrayList<IndexEntry>(currentBatchSize) val batch = ArrayList<IndexEntry>(currentBatchSize)
val batchEnd = minOf(processed + currentBatchSize, entryCount) val batchEnd = minOf(processed + currentBatchSize, entryCount)
while (processed < batchEnd) { while (processed < batchEnd) {
val word = input.readUtf8String() batch.add(input.readTempEntry())
val originalWord = input.readUtf8String()
val offset = input.readLong()
val length = input.readInt()
batch.add(IndexEntry(
word = word,
originalWord = originalWord,
offset = ArticleOffset(offset),
length = ArticleLength(length)
))
processed++ processed++
} }
@@ -422,7 +428,7 @@ class DslIndexer(context: Context? = null) {
onFileProgress(MERGE_START_PROGRESS) onFileProgress(MERGE_START_PROGRESS)
onOperationChange("Merging ${sortedBatches.size} batches") onOperationChange("Merging ${sortedBatches.size} batches")
mergeSortedBatches(sortedBatches, indexFile, entryCount, onOperationChange, onFileProgress) mergeSortedBatches(sortedBatches, indexFile, counts, onOperationChange, onFileProgress)
onFileProgress(FINAL_INDEX_DONE_PROGRESS) onFileProgress(FINAL_INDEX_DONE_PROGRESS)
} finally { } finally {
sortedBatches.forEach { try { it.delete() } catch (_: Throwable) { /* Ignore cleanup failure */ } } sortedBatches.forEach { try { it.delete() } catch (_: Throwable) { /* Ignore cleanup failure */ } }
@@ -443,6 +449,7 @@ class DslIndexer(context: Context? = null) {
writer.write(entry.originalWord) writer.write(entry.originalWord)
output.writeLong(entry.offset.value) output.writeLong(entry.offset.value)
output.writeInt(entry.length.value) output.writeInt(entry.length.value)
output.writeByte(if (entry.isAlias) ENTRY_FLAG_ALIAS else 0)
} }
} }
} }
@@ -453,15 +460,16 @@ class DslIndexer(context: Context? = null) {
private suspend fun mergeSortedBatches( private suspend fun mergeSortedBatches(
batchFiles: List<File>, batchFiles: List<File>,
indexFile: File, indexFile: File,
totalEntries: Int, counts: ParsedCounts,
onOperationChange: (String) -> Unit = {}, onOperationChange: (String) -> Unit = {},
onFileProgress: (Float) -> Unit, onFileProgress: (Float) -> Unit,
) { ) {
if (batchFiles.isEmpty()) return if (batchFiles.isEmpty()) return
val totalEntries = counts.totalEntries
val sparsePoints = mutableListOf<SparsePoint>() val sparsePoints = mutableListOf<SparsePoint>()
val prefixCounts = HashMap<String, Int>(minOf(totalEntries / 10, 100_000)) val prefixCounts = HashMap<String, Int>(minOf(totalEntries / 10, 100_000))
val headerSize = 4 + 4 + 4 val headerSize = 4 + 4 + 4 + 4
val dataScratchFile = File(indexFile.parent, "${indexFile.name}.data") val dataScratchFile = File(indexFile.parent, "${indexFile.name}.data")
val pq = PriorityQueue<BatchReader>() val pq = PriorityQueue<BatchReader>()
@@ -508,7 +516,8 @@ class DslIndexer(context: Context? = null) {
val origSize = writer.write(entry.originalWord) val origSize = writer.write(entry.originalWord)
dataOutput.writeLong(entry.offset.value) dataOutput.writeLong(entry.offset.value)
dataOutput.writeInt(entry.length.value) dataOutput.writeInt(entry.length.value)
currentByteOffset += 2 + wordSize + 2 + origSize + 8 + 4 dataOutput.writeByte(if (entry.isAlias) ENTRY_FLAG_ALIAS else 0)
currentByteOffset += 2 + wordSize + 2 + origSize + 8 + 4 + 1
entryIndex++ entryIndex++
@@ -542,6 +551,7 @@ class DslIndexer(context: Context? = null) {
DataOutputStream(BufferedOutputStream(FileOutputStream(indexFile), BUFFER_SIZE)).use { output -> DataOutputStream(BufferedOutputStream(FileOutputStream(indexFile), BUFFER_SIZE)).use { output ->
output.writeInt(INDEX_VERSION) output.writeInt(INDEX_VERSION)
output.writeInt(totalEntries) output.writeInt(totalEntries)
output.writeInt(counts.articleCount)
output.writeInt(sparsePoints.size) output.writeInt(sparsePoints.size)
sparsePoints.forEachIndexed { idx, point -> sparsePoints.forEachIndexed { idx, point ->
@@ -632,18 +642,14 @@ class DslIndexer(context: Context? = null) {
} }
private fun readNextEntry(input: java.io.DataInputStream): IndexEntry? { private fun readNextEntry(input: java.io.DataInputStream): IndexEntry? {
return try { return try {
val word = input.readUtf8String() input.readTempEntry()
val originalWord = input.readUtf8String()
val offset = ArticleOffset(input.readLong())
val length = ArticleLength(input.readInt())
IndexEntry(word, originalWord, offset, length)
} catch (_: java.io.EOFException) { } catch (_: java.io.EOFException) {
null null
} }
} }
private fun writeIndexFile(indexFile: File, entries: List<IndexEntry>) { private fun writeIndexFile(indexFile: File, entries: List<IndexEntry>, articleCount: Int) {
val sparseData = buildAdaptiveSparseIndex(entries) val sparseData = buildAdaptiveSparseIndex(entries)
val headerSize = 4 + 4 + 4 val headerSize = 4 + 4 + 4 + 4
// Cache sparse table bytes to avoid redundant conversions // Cache sparse table bytes to avoid redundant conversions
val sparseTableBytes = sparseData.map { CachedBytes.from(it.word) } val sparseTableBytes = sparseData.map { CachedBytes.from(it.word) }
@@ -657,13 +663,14 @@ class DslIndexer(context: Context? = null) {
// Calculate sizes without allocating byte arrays // Calculate sizes without allocating byte arrays
val wordSize = utf8ByteSize(entry.word) val wordSize = utf8ByteSize(entry.word)
val origSize = utf8ByteSize(entry.originalWord) val origSize = utf8ByteSize(entry.originalWord)
currentByteOffset += 2 + wordSize + 2 + origSize + 8 + 4 currentByteOffset += 2 + wordSize + 2 + origSize + 8 + 4 + 1
} }
DataOutputStream(BufferedOutputStream(FileOutputStream(indexFile))).use { output -> DataOutputStream(BufferedOutputStream(FileOutputStream(indexFile))).use { output ->
val writer = Utf8Writer(output) val writer = Utf8Writer(output)
output.writeInt(INDEX_VERSION) output.writeInt(INDEX_VERSION)
output.writeInt(entries.size) output.writeInt(entries.size)
output.writeInt(articleCount)
output.writeInt(sparseData.size) output.writeInt(sparseData.size)
sparseData.forEach { point -> sparseData.forEach { point ->
@@ -677,6 +684,7 @@ class DslIndexer(context: Context? = null) {
writer.write(entry.originalWord) writer.write(entry.originalWord)
output.writeLong(entry.offset.value) output.writeLong(entry.offset.value)
output.writeInt(entry.length.value) output.writeInt(entry.length.value)
output.writeByte(if (entry.isAlias) ENTRY_FLAG_ALIAS else 0)
} }
} }
} }
@@ -707,7 +715,7 @@ class DslIndexer(context: Context? = null) {
// Calculate sizes without allocating byte arrays // Calculate sizes without allocating byte arrays
val wordSize = utf8ByteSize(entry.word) val wordSize = utf8ByteSize(entry.word)
val origSize = utf8ByteSize(entry.originalWord) val origSize = utf8ByteSize(entry.originalWord)
currentByteOffset += 2 + wordSize + 2 + origSize + 8 + 4 currentByteOffset += 2 + wordSize + 2 + origSize + 8 + 4 + 1
} }
return sparsePoints return sparsePoints
@@ -824,6 +832,24 @@ private fun java.io.DataInputStream.readUtf8String(): String {
readFully(bytes) readFully(bytes)
return String(bytes, Charsets.UTF_8) return String(bytes, Charsets.UTF_8)
} }
internal const val ENTRY_FLAG_ALIAS = 1
private fun java.io.DataInputStream.readTempEntry(): IndexEntry {
val word = readUtf8String()
val originalWord = readUtf8String()
val offset = readLong()
val length = readInt()
val flags = readUnsignedByte()
return IndexEntry(
word = word,
originalWord = originalWord,
offset = ArticleOffset(offset),
length = ArticleLength(length),
isAlias = flags and ENTRY_FLAG_ALIAS != 0
)
}
private enum class LineKind { private enum class LineKind {
HEADWORD, HEADWORD,
BODY, BODY,
@@ -5,6 +5,7 @@ import com.example.research.R
import com.example.research.core.domain.model.ArticleLength import com.example.research.core.domain.model.ArticleLength
import com.example.research.core.domain.model.ArticleOffset import com.example.research.core.domain.model.ArticleOffset
import com.example.research.core.domain.model.IndexEntry import com.example.research.core.domain.model.IndexEntry
import com.example.research.core.util.foldLatinDiacritics
import com.example.research.core.util.DictionaryError import com.example.research.core.util.DictionaryError
import com.example.research.core.util.sanitizeCacheKey import com.example.research.core.util.sanitizeCacheKey
import com.example.research.core.util.sanitizeQuery import com.example.research.core.util.sanitizeQuery
@@ -23,7 +24,8 @@ import java.io.RandomAccessFile
class IndexSearcher(private val context: Context) { class IndexSearcher(private val context: Context) {
private val metadataCache = MetadataLruCache(maxBytes = 64L * 1024L * 1024L) private val metadataCache = MetadataLruCache(maxBytes = 64L * 1024L * 1024L)
private val resultsCache = ResultsLruCache(maxEntries = 20) private val resultsCache = ResultsLruCache<List<IndexEntry>>(maxEntries = 20)
private val cursorResultsCache = ResultsLruCache<CursorSearchResult>(maxEntries = 20)
/** /**
* Pre-allocated reusable buffer for readEntry() to reduce allocations. * Pre-allocated reusable buffer for readEntry() to reduce allocations.
@@ -49,6 +51,27 @@ class IndexSearcher(private val context: Context) {
val comparedCount: Int val comparedCount: Int
) )
data class RangeCursor(
val rangeQuery: String,
val absoluteIndex: Int,
val fileOffset: Long
)
internal class RangeScanResult(
val entries: List<IndexEntry>,
val nextCursor: RangeCursor?
)
class CursorSearchResult(
val entries: List<IndexEntry>,
val cursors: List<RangeCursor>
)
class ScannedEntry(
val entry: IndexEntry,
val cursorAfter: RangeCursor?
)
class IndexComparisonException(message: String) : IllegalStateException(message) class IndexComparisonException(message: String) : IllegalStateException(message)
suspend fun findFirstEntry(pathOrUri: String, query: String): IndexEntry? = withContext(Dispatchers.Default) { suspend fun findFirstEntry(pathOrUri: String, query: String): IndexEntry? = withContext(Dispatchers.Default) {
@@ -66,7 +89,7 @@ class IndexSearcher(private val context: Context) {
var iterationCount = 0 var iterationCount = 0
while (currentIdx < count) { while (currentIdx < count) {
if (iterationCount % 1000 == 0) yield() if (iterationCount % 1000 == 0) yield()
val nextEntry = readEntry(source) val nextEntry = readEntry(source, metadata)
val nextWordLower = nextEntry.word.lowercase() val nextWordLower = nextEntry.word.lowercase()
if (!nextWordLower.startsWith(normalizedQuery)) break if (!nextWordLower.startsWith(normalizedQuery)) break
if (nextWordLower == normalizedQuery) return@withContext nextEntry.copy(word = nextWordLower) if (nextWordLower == normalizedQuery) return@withContext nextEntry.copy(word = nextWordLower)
@@ -93,6 +116,7 @@ class IndexSearcher(private val context: Context) {
return@withContext emptyList() return@withContext emptyList()
} }
val normalizedQuery = query.sanitizeQuery().lowercase().trim() val normalizedQuery = query.sanitizeQuery().lowercase().trim()
val foldedQuery = normalizedQuery.foldLatinDiacritics()
val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}" val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}"
resultsCache.get(cacheKey)?.let { resultsCache.get(cacheKey)?.let {
return@withContext it return@withContext it
@@ -102,33 +126,34 @@ class IndexSearcher(private val context: Context) {
val count = metadata.totalCount val count = metadata.totalCount
if (count == 0) return@withContext emptyList() if (count == 0) return@withContext emptyList()
val results = openSource(pathOrUri).use { source -> val results = openSource(pathOrUri).use { source ->
val prefixMatches = mutableListOf<IndexEntry>() val prefixMatches = readPrefixMatches(source, metadata, normalizedQuery).entries.toMutableList()
val substringMatches = mutableListOf<IndexEntry>() if (foldedQuery != normalizedQuery) {
val firstMatch = findFirstMatch(source, metadata, normalizedQuery) prefixMatches.addAll(readPrefixMatches(source, metadata, foldedQuery).entries)
if (firstMatch != null) {
prefixMatches.addAll(readMatchingEntries(source, count, firstMatch, normalizedQuery))
} }
val substringMatches = mutableListOf<IndexEntry>()
val seenArticles = prefixMatches.mapTo(HashSet()) { it.articleKey() }
if (includeSubstringMatches && if (includeSubstringMatches &&
count <= SUBSTRING_SCAN_MAX_ENTRIES && count <= SUBSTRING_SCAN_MAX_ENTRIES &&
prefixMatches.size < 100 && seenArticles.size < MAX_SUBSTRING_RESULTS &&
normalizedQuery.length >= 2 normalizedQuery.length >= 2
) { ) {
val prefixMatchWords = prefixMatches.map { it.word }.toSet()
source.seek(metadata.dataStartOffset) source.seek(metadata.dataStartOffset)
for (i in 0 until count) { for (i in 0 until count) {
if (i % 1000 == 0) yield() if (i % 1000 == 0) yield()
if (substringMatches.size >= 100) break if (substringMatches.size >= MAX_SUBSTRING_RESULTS) break
val entry = readEntry(source) val entry = readEntry(source, metadata)
val entryWordLower = entry.word.lowercase() val entryWordLower = entry.word.lowercase()
if (entryWordLower.contains(normalizedQuery) && entryWordLower !in prefixMatchWords) { if (entryWordLower.contains(normalizedQuery) &&
seenArticles.add(entry.articleKey())
) {
substringMatches.add(entry.copy(word = entryWordLower)) substringMatches.add(entry.copy(word = entryWordLower))
} }
} }
} }
val combinedResults = prefixMatches + substringMatches (prefixMatches + substringMatches)
combinedResults.rankBySearchRelevance( .rankBySearchRelevance(SearchRankingContext(normalizedQuery, foldedQuery))
SearchRankingContext(normalizedQuery) .distinctBy { it.articleKey() }
)
} }
resultsCache.put(cacheKey, results) resultsCache.put(cacheKey, results)
results results
@@ -148,6 +173,77 @@ class IndexSearcher(private val context: Context) {
metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) } metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) }
} }
suspend fun searchWithCursors(
pathOrUri: String,
query: String
): CursorSearchResult = withContext(Dispatchers.Default) {
if (query.isBlank()) {
return@withContext CursorSearchResult(emptyList(), emptyList())
}
val normalizedQuery = query.sanitizeQuery().lowercase().trim()
val foldedQuery = normalizedQuery.foldLatinDiacritics()
val cacheKey = "$pathOrUri:${normalizedQuery.sanitizeCacheKey()}"
cursorResultsCache.get(cacheKey)?.let {
return@withContext it
}
try {
val metadata = metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) }
if (metadata.totalCount == 0) {
return@withContext CursorSearchResult(emptyList(), emptyList())
}
val result = openSource(pathOrUri).use { source ->
val scans = buildList {
add(readPrefixMatches(source, metadata, normalizedQuery))
if (foldedQuery != normalizedQuery) {
add(readPrefixMatches(source, metadata, foldedQuery))
}
}
val entries = scans.flatMap { it.entries }
.rankBySearchRelevance(SearchRankingContext(normalizedQuery, foldedQuery))
.distinctBy { it.articleKey() }
CursorSearchResult(entries, scans.mapNotNull { it.nextCursor })
}
cursorResultsCache.put(cacheKey, result)
result
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
Log.e(TAG, "searchWithCursors() - error: ${e.message}", e)
if (e is DictionaryError) throw e
throw DictionaryError.SearchError.QueryFailed(context.getString(R.string.error_search_failed, query.take(20), pathOrUri.take(20)), e)
}
}
suspend fun scanTailChunk(
pathOrUri: String,
cursor: RangeCursor,
maxEntries: Int
): List<ScannedEntry> = withContext(Dispatchers.Default) {
val metadata = metadataCache.getOrPut(pathOrUri) { loadMetadata(pathOrUri) }
if (cursor.absoluteIndex >= metadata.totalCount) return@withContext emptyList()
openSource(pathOrUri).use { source ->
source.seek(cursor.fileOffset)
val out = ArrayList<ScannedEntry>(maxEntries)
var currentIdx = cursor.absoluteIndex
var iterationCount = 0
while (currentIdx < metadata.totalCount && out.size < maxEntries) {
if (iterationCount % 1000 == 0) yield()
val entry = readEntry(source, metadata)
val entryWordLower = entry.word.lowercase()
if (!entryWordLower.startsWith(cursor.rangeQuery)) break
currentIdx++
iterationCount++
val cursorAfter = if (currentIdx < metadata.totalCount) {
RangeCursor(cursor.rangeQuery, currentIdx, source.position())
} else {
null
}
out.add(ScannedEntry(entry.copy(word = entryWordLower), cursorAfter))
}
out
}
}
suspend fun compareEntryGroups( suspend fun compareEntryGroups(
expectedIndexPath: String, expectedIndexPath: String,
actualIndexPath: String, actualIndexPath: String,
@@ -168,8 +264,8 @@ class IndexSearcher(private val context: Context) {
expectedSource.seek(expectedMetadata.dataStartOffset) expectedSource.seek(expectedMetadata.dataStartOffset)
actualSource.seek(actualMetadata.dataStartOffset) actualSource.seek(actualMetadata.dataStartOffset)
var expectedNext = readEntryOrNull(expectedSource, expectedMetadata.totalCount > 0) var expectedNext = readEntryOrNull(expectedSource, expectedMetadata, expectedMetadata.totalCount > 0)
var actualNext = readEntryOrNull(actualSource, actualMetadata.totalCount > 0) var actualNext = readEntryOrNull(actualSource, actualMetadata, actualMetadata.totalCount > 0)
var comparedCount = 0 var comparedCount = 0
while (expectedNext != null && actualNext != null) { while (expectedNext != null && actualNext != null) {
@@ -191,6 +287,7 @@ class IndexSearcher(private val context: Context) {
comparedCount++ comparedCount++
expectedNext = readEntryOrNull( expectedNext = readEntryOrNull(
expectedSource, expectedSource,
expectedMetadata,
comparedCount < expectedMetadata.totalCount comparedCount < expectedMetadata.totalCount
) )
} }
@@ -203,6 +300,7 @@ class IndexSearcher(private val context: Context) {
actualReadCount++ actualReadCount++
actualNext = readEntryOrNull( actualNext = readEntryOrNull(
actualSource, actualSource,
actualMetadata,
actualReadCount < actualMetadata.totalCount actualReadCount < actualMetadata.totalCount
) )
} }
@@ -233,12 +331,16 @@ class IndexSearcher(private val context: Context) {
} }
} }
private fun readEntryOrNull(source: RandomAccessSource, shouldRead: Boolean): IndexEntry? = private fun readEntryOrNull(
if (shouldRead) readEntry(source) else null source: RandomAccessSource,
metadata: IndexMetadata,
shouldRead: Boolean
): IndexEntry? = if (shouldRead) readEntry(source, metadata) else null
fun trimMemory(level: Int) { fun trimMemory(level: Int) {
metadataCache.trim(level) metadataCache.trim(level)
resultsCache.trim(level) resultsCache.trim(level)
cursorResultsCache.trim(level)
} }
suspend fun isMetadataLoaded(pathOrUri: String): Boolean { suspend fun isMetadataLoaded(pathOrUri: String): Boolean {
@@ -252,7 +354,13 @@ class IndexSearcher(private val context: Context) {
val count = input.readInt() val count = input.readInt()
return@withContext if (version >= 2) { return@withContext if (version >= 2) {
loadMetadataV2(input, version, count) val headerBase = if (version >= 6) {
input.readInt()
12L
} else {
8L
}
loadMetadataV2(input, version, count, headerBase)
} else { } else {
loadMetadataV1(input, version, count) loadMetadataV1(input, version, count)
} }
@@ -272,13 +380,13 @@ class IndexSearcher(private val context: Context) {
// runs inside withContext(Dispatchers.IO). The dispatcher is not visible across the // runs inside withContext(Dispatchers.IO). The dispatcher is not visible across the
// suspend-call boundary, so the inspection is suppressed deliberately. // suspend-call boundary, so the inspection is suppressed deliberately.
@Suppress("BlockingMethodInNonBlockingContext") @Suppress("BlockingMethodInNonBlockingContext")
private suspend fun loadMetadataV2(input: DataInputStream, version: Int, count: Int): IndexMetadata { private suspend fun loadMetadataV2(input: DataInputStream, version: Int, count: Int, headerBase: Long): IndexMetadata {
val sparseCount = input.readInt() val sparseCount = input.readInt()
val sparseIndices = IntArray(sparseCount) val sparseIndices = IntArray(sparseCount)
val sparseOffsets = LongArray(sparseCount) val sparseOffsets = LongArray(sparseCount)
val sparseWords = Array(sparseCount) { "" } val sparseWords = Array(sparseCount) { "" }
var headerSize = 4 + 4 + 4L var headerSize = headerBase + 4L
for (i in 0 until sparseCount) { for (i in 0 until sparseCount) {
if (i % 1000 == 0) yield() if (i % 1000 == 0) yield()
@@ -337,6 +445,17 @@ class IndexSearcher(private val context: Context) {
dataStartOffset = 8L dataStartOffset = 8L
) )
} }
private suspend fun readPrefixMatches(
source: RandomAccessSource,
metadata: IndexMetadata,
query: String
): RangeScanResult {
val firstMatch = findFirstMatch(source, metadata, query)
?: return RangeScanResult(emptyList(), null)
return readMatchingEntries(source, metadata, firstMatch, query)
}
private suspend fun findFirstMatch(source: RandomAccessSource, metadata: IndexMetadata, query: String): Pair<Int, IndexEntry>? { private suspend fun findFirstMatch(source: RandomAccessSource, metadata: IndexMetadata, query: String): Pair<Int, IndexEntry>? {
val sparseIndices = metadata.sparseIndices val sparseIndices = metadata.sparseIndices
val sparseOffsets = metadata.sparseOffsets val sparseOffsets = metadata.sparseOffsets
@@ -359,7 +478,7 @@ class IndexSearcher(private val context: Context) {
while (currentAbsoluteIdx < metadata.totalCount) { while (currentAbsoluteIdx < metadata.totalCount) {
if (iterationCount % 1000 == 0) yield() if (iterationCount % 1000 == 0) yield()
val entry = readEntry(source) val entry = readEntry(source, metadata)
val entryWordLower = entry.word.lowercase() val entryWordLower = entry.word.lowercase()
if (entryWordLower >= query) { if (entryWordLower >= query) {
return if (entryWordLower.startsWith(query)) Pair(currentAbsoluteIdx, entry.copy(word = entryWordLower)) else null return if (entryWordLower.startsWith(query)) Pair(currentAbsoluteIdx, entry.copy(word = entryWordLower)) else null
@@ -394,53 +513,64 @@ class IndexSearcher(private val context: Context) {
} }
private suspend fun readMatchingEntries( private suspend fun readMatchingEntries(
source: RandomAccessSource, source: RandomAccessSource,
totalCount: Int, metadata: IndexMetadata,
firstMatch: Pair<Int, IndexEntry>, firstMatch: Pair<Int, IndexEntry>,
query: String query: String
): List<IndexEntry> { ): RangeScanResult {
val results = mutableListOf<IndexEntry>() val results = mutableListOf<IndexEntry>()
results.add(firstMatch.second) results.add(firstMatch.second)
val uniqueArticles = hashSetOf(firstMatch.second.articleKey())
var currentIdx = firstMatch.first + 1 var currentIdx = firstMatch.first + 1
var iterationCount = 0 var iterationCount = 0
while (currentIdx < totalCount && results.size < 100) { while (currentIdx < metadata.totalCount) {
if (uniqueArticles.size >= MAX_PREFIX_RESULTS) {
return RangeScanResult(results, RangeCursor(query, currentIdx, source.position()))
}
if (iterationCount % 1000 == 0) yield() if (iterationCount % 1000 == 0) yield()
val entry = readEntry(source) val entry = readEntry(source, metadata)
val entryWordLower = entry.word.lowercase() val entryWordLower = entry.word.lowercase()
if (!entryWordLower.startsWith(query)) break if (!entryWordLower.startsWith(query)) break
uniqueArticles.add(entry.articleKey())
results.add(entry.copy(word = entryWordLower)) results.add(entry.copy(word = entryWordLower))
currentIdx++ currentIdx++
iterationCount++ iterationCount++
} }
return results return RangeScanResult(results, null)
} }
private fun readEntry(source: RandomAccessSource): IndexEntry { private fun readPooledString(source: RandomAccessSource, length: Int): String {
val wordLen = source.readUnsignedShort() return if (length <= 4096) {
val word = if (wordLen <= 4096) {
val buffer = entryBuffer.get() ?: throw IllegalStateException("Entry buffer pool exhausted") val buffer = entryBuffer.get() ?: throw IllegalStateException("Entry buffer pool exhausted")
source.readFully(buffer, 0, wordLen) source.readFully(buffer, 0, length)
String(buffer, 0, wordLen, Charsets.UTF_8) String(buffer, 0, length, Charsets.UTF_8)
} else { } else {
val bytes = ByteArray(wordLen) val bytes = ByteArray(length)
source.readFully(bytes) source.readFully(bytes)
String(bytes, Charsets.UTF_8) String(bytes, Charsets.UTF_8)
} }
val origLen = source.readUnsignedShort()
val originalWord = if (origLen <= 4096) {
val buffer = entryBuffer.get() ?: throw IllegalStateException("Entry buffer pool exhausted")
source.readFully(buffer, 0, origLen)
String(buffer, 0, origLen, Charsets.UTF_8)
} else {
val bytes = ByteArray(origLen)
source.readFully(bytes)
String(bytes, Charsets.UTF_8)
} }
private fun readEntry(source: RandomAccessSource, metadata: IndexMetadata): IndexEntry {
val word = readPooledString(source, source.readUnsignedShort())
val originalWord = readPooledString(source, source.readUnsignedShort())
val offset = ArticleOffset(source.readLong()) val offset = ArticleOffset(source.readLong())
val length = ArticleLength(source.readInt()) val length = ArticleLength(source.readInt())
return IndexEntry(word, originalWord, offset, length) val isAlias = if (metadata.version >= 6) {
source.readUnsignedByte() and ENTRY_FLAG_ALIAS != 0
} else {
false
} }
return IndexEntry(word, originalWord, offset, length, isAlias = isAlias)
}
private data class ArticleKey(
val originalWord: String,
val offset: Long,
val length: Int
)
private fun IndexEntry.articleKey() = ArticleKey(originalWord, offset.value, length.value)
private fun openSource(pathOrUri: String): RandomAccessSource { private fun openSource(pathOrUri: String): RandomAccessSource {
val file = File(pathOrUri) val file = File(pathOrUri)
if (!file.exists()) throw DictionaryError.FileSystemError.NotFound(pathOrUri) if (!file.exists()) throw DictionaryError.FileSystemError.NotFound(pathOrUri)
@@ -455,20 +585,22 @@ class IndexSearcher(private val context: Context) {
private const val TAG = "IndexSearcher" private const val TAG = "IndexSearcher"
private const val SUBSTRING_SCAN_MAX_ENTRIES = 200_000 private const val SUBSTRING_SCAN_MAX_ENTRIES = 200_000
private const val SPARSE_INTERVAL_V1 = 128 private const val SPARSE_INTERVAL_V1 = 128
private const val MAX_PREFIX_RESULTS = 200
private const val MAX_SUBSTRING_RESULTS = 100
} }
} }
private class ResultsLruCache(private val maxEntries: Int) { private class ResultsLruCache<V : Any>(private val maxEntries: Int) {
private val mutex = Mutex() private val mutex = Mutex()
private val map = object : LinkedHashMap<String, List<IndexEntry>>(maxEntries, 0.75f, true) { private val map = object : LinkedHashMap<String, V>(maxEntries, 0.75f, true) {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, List<IndexEntry>>): Boolean { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, V>): Boolean {
return size > maxEntries return size > maxEntries
} }
} }
suspend fun get(key: String): List<IndexEntry>? = mutex.withLock { suspend fun get(key: String): V? = mutex.withLock {
map[key] map[key]
} }
suspend fun put(key: String, value: List<IndexEntry>) = mutex.withLock { suspend fun put(key: String, value: V) = mutex.withLock {
map[key] = value map[key] = value
} }
fun trim(level: Int) { fun trim(level: Int) {
@@ -548,6 +680,8 @@ private class MetadataLruCache(private val maxBytes: Long) {
} }
interface RandomAccessSource : Closeable { interface RandomAccessSource : Closeable {
fun seek(pos: Long) fun seek(pos: Long)
fun position(): Long
fun readUnsignedByte(): Int
fun readUnsignedShort(): Int fun readUnsignedShort(): Int
fun readInt(): Int fun readInt(): Int
fun readLong(): Long fun readLong(): Long
@@ -561,6 +695,10 @@ class FileSource(file: File) : RandomAccessSource {
raf.seek(pos) raf.seek(pos)
} }
override fun position(): Long = raf.filePointer
override fun readUnsignedByte(): Int = raf.readUnsignedByte()
override fun readUnsignedShort(): Int = raf.readUnsignedShort() override fun readUnsignedShort(): Int = raf.readUnsignedShort()
override fun readInt(): Int = raf.readInt() override fun readInt(): Int = raf.readInt()
@@ -1,15 +1,17 @@
package com.example.research.data.parser package com.example.research.data.parser
import com.example.research.core.domain.model.IndexEntry import com.example.research.core.domain.model.IndexEntry
import com.example.research.core.util.foldLatinDiacritics
internal data class SearchRankingContext( internal data class SearchRankingContext(
val normalizedQuery: String, val normalizedQuery: String,
val foldedQuery: String = normalizedQuery.foldLatinDiacritics(),
) )
private data class SortKey( private data class SortKey(
val charCount: Int, val matchTier: Int,
val exactMatch: Int, val exactMatch: Int,
val startsWithMatch: Int, val charCount: Int,
val word: String, val word: String,
val entry: IndexEntry, val entry: IndexEntry,
) )
@@ -26,13 +28,18 @@ internal fun List<IndexEntry>.rankBySearchRelevance(
.ifEmpty { entry.word.trim() } .ifEmpty { entry.word.trim() }
SortKey( SortKey(
charCount = displayKey.codePointCount(0, displayKey.length), matchTier = when {
!entry.isAlias && word.startsWith(ranking.normalizedQuery) -> 0
entry.isAlias && word.startsWith(ranking.foldedQuery) -> 1
!entry.isAlias && word.foldLatinDiacritics().startsWith(ranking.foldedQuery) -> 1
else -> 2
},
exactMatch = if (word == ranking.normalizedQuery) 0 else 1, exactMatch = if (word == ranking.normalizedQuery) 0 else 1,
startsWithMatch = if (word.startsWith(ranking.normalizedQuery)) 0 else 1, charCount = displayKey.codePointCount(0, displayKey.length),
word = word, word = word,
entry = entry, entry = entry,
) )
} }
.sortedWith(compareBy({ it.charCount }, { it.exactMatch }, { it.startsWithMatch }, { it.word })) .sortedWith(compareBy({ it.matchTier }, { it.exactMatch }, { it.charCount }, { it.word }))
.map { it.entry } .map { it.entry }
} }
@@ -14,6 +14,7 @@ import com.example.research.core.util.extractDictionaryPrefix
import com.example.research.core.util.sanitizeQuery import com.example.research.core.util.sanitizeQuery
import com.example.research.core.util.toDictionaryError import com.example.research.core.util.toDictionaryError
import com.example.research.data.dictzip.DictZipRandomAccessFile import com.example.research.data.dictzip.DictZipRandomAccessFile
import com.example.research.data.local.preferences.PreferencesManager
import com.example.research.data.parser.DslCharsetDetector import com.example.research.data.parser.DslCharsetDetector
import com.example.research.data.parser.DslHeaderParser import com.example.research.data.parser.DslHeaderParser
import com.example.research.data.parser.DslIndexer import com.example.research.data.parser.DslIndexer
@@ -26,6 +27,7 @@ import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.ensureActive import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.update import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.supervisorScope import kotlinx.coroutines.supervisorScope
@@ -85,15 +87,11 @@ private class ProgressTracker(files: List<LocalDictionaryRepository.DiscoveredFi
class LocalDictionaryRepository( class LocalDictionaryRepository(
private val context: Context, private val context: Context,
private val engine: KotlinDictionaryEngine, private val engine: KotlinDictionaryEngine = KotlinDictionaryEngine(context),
private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO, private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO,
private val defaultDispatcher: CoroutineDispatcher = Dispatchers.Default, private val defaultDispatcher: CoroutineDispatcher = Dispatchers.Default,
private val preferencesManager: PreferencesManager = PreferencesManager(context),
) : CoroutineScope { ) : CoroutineScope {
constructor(context: Context) : this(
context,
KotlinDictionaryEngine(context)
)
override val coroutineContext = Job() + ioDispatcher override val coroutineContext = Job() + ioDispatcher
private val mutableDictionaries = MutableStateFlow<List<Dictionary>>(emptyList()) private val mutableDictionaries = MutableStateFlow<List<Dictionary>>(emptyList())
@@ -107,6 +105,7 @@ class LocalDictionaryRepository(
private val indexingMutex = Mutex() private val indexingMutex = Mutex()
private val indexingSemaphore = Semaphore(1) private val indexingSemaphore = Semaphore(1)
private val warmupMutex = Mutex() private val warmupMutex = Mutex()
private val dictionaryStateMutex = Mutex()
@Volatile @Volatile
private var currentIndexingJob: Job? = null private var currentIndexingJob: Job? = null
@@ -124,6 +123,24 @@ class LocalDictionaryRepository(
fun isIndexingInProgress(): Boolean = currentIndexingJob?.isActive == true fun isIndexingInProgress(): Boolean = currentIndexingJob?.isActive == true
suspend fun listDictionaryPayloadFileNames(path: String): List<String>? =
withContext(ioDispatcher) {
try {
val directory = File(path)
when {
!directory.exists() -> {
if (directory.parentFile?.isDirectory == true) emptyList() else null
}
!directory.isDirectory -> null
else -> directory.listFiles { file -> isDictionaryPayloadFile(file) }
?.map(File::getName)
}
} catch (e: SecurityException) {
Log.w(TAG, "Failed to list dictionary files: ${e.message}")
null
}
}
suspend fun scanDirectory(pathOrUri: String): OperationResult<Int> = withContext(ioDispatcher) { suspend fun scanDirectory(pathOrUri: String): OperationResult<Int> = withContext(ioDispatcher) {
if (isIndexingInProgress()) { if (isIndexingInProgress()) {
return@withContext OperationResult.Error( return@withContext OperationResult.Error(
@@ -140,7 +157,7 @@ class LocalDictionaryRepository(
val filesToScan = scanLocalDirectory(pathOrUri) val filesToScan = scanLocalDirectory(pathOrUri)
if (filesToScan.isEmpty()) { if (filesToScan.isEmpty()) {
mutableDictionaries.value = emptyList() replaceDictionaries(emptyList())
mutableIndexingProgress.value = IndexingProgress(isIndexing = false) mutableIndexingProgress.value = IndexingProgress(isIndexing = false)
return@withContext OperationResult.Success(0) return@withContext OperationResult.Success(0)
} }
@@ -159,12 +176,12 @@ class LocalDictionaryRepository(
} }
if (filesToIndex.isEmpty()) { if (filesToIndex.isEmpty()) {
mutableDictionaries.value = existingDictionaries.sortedBy { it.name } replaceDictionaries(existingDictionaries)
engine.getIndexSearcher().trimMemory(80) engine.getIndexSearcher().trimMemory(80)
return@withContext OperationResult.Success(existingDictionaries.size) return@withContext OperationResult.Success(existingDictionaries.size)
} }
mutableDictionaries.value = existingDictionaries.sortedBy { it.name } replaceDictionaries(existingDictionaries)
val totalArticlesIndexed = AtomicInteger(existingDictionaries.sumOf { it.articleCount }) val totalArticlesIndexed = AtomicInteger(existingDictionaries.sumOf { it.articleCount })
@@ -233,12 +250,9 @@ class LocalDictionaryRepository(
if (res is OperationResult.Success) { if (res is OperationResult.Success) {
progressTracker?.updateFileProgress(file.name, 1.0f) progressTracker?.updateFileProgress(file.name, 1.0f)
totalArticlesIndexed.addAndGet(res.data.articleCount) totalArticlesIndexed.addAndGet(res.data.articleCount)
mutableDictionaries.update { current -> val dictionary = res.data
(current + res.data) addDictionary(dictionary)
.distinctBy { it.path } dictionary
.sortedBy { it.name }
}
res.data
} else null } else null
} }
} }
@@ -266,17 +280,37 @@ class LocalDictionaryRepository(
} }
} }
private suspend fun replaceDictionaries(dictionaries: List<Dictionary>) {
dictionaryStateMutex.withLock { applyDictionaries(dictionaries) }
}
private suspend fun addDictionary(dictionary: Dictionary) {
dictionaryStateMutex.withLock {
applyDictionaries((mutableDictionaries.value + dictionary).distinctBy { it.path })
}
}
private suspend fun applyDictionaries(dictionaries: List<Dictionary>) {
val disabledPaths = preferencesManager.disabledDictionaryPaths.first()
mutableDictionaries.value = dictionaries
.map { it.withActiveStatus(it.path !in disabledPaths) }
.sortedBy { it.name }
}
private suspend fun tryLoadExistingIndex(file: DiscoveredFile): Dictionary? { private suspend fun tryLoadExistingIndex(file: DiscoveredFile): Dictionary? {
val dictFile = File(file.localPath) val dictFile = File(file.localPath)
val indexFile = File(file.indexPath) val indexFile = File(file.indexPath)
if (!indexFile.exists() || indexFile.length() == 0L) return null if (!indexFile.exists() || indexFile.length() == 0L) {
deleteIndexFiles(indexFile)
return null
}
if (!dictFile.exists()) return null if (!dictFile.exists()) return null
val header = readIndexHeader(indexFile) val header = readIndexHeader(indexFile)
if (header == null || header.version != DslIndexer.INDEX_VERSION) { if (header == null || header.version != DslIndexer.INDEX_VERSION) {
try { indexFile.delete() } catch (_: Exception) {} deleteIndexFiles(indexFile)
return null return null
} }
@@ -326,23 +360,36 @@ class LocalDictionaryRepository(
) )
OperationResult.Success(dict) OperationResult.Success(dict)
} else { } else {
indexFile.delete() deleteIndexFiles(indexFile)
OperationResult.Error(result.errorMsg, null) OperationResult.Error(result.errorMsg, null)
} }
} catch(e: Exception) { } catch(e: Exception) {
indexFile.delete() deleteIndexFiles(indexFile)
if (e is CancellationException) throw e if (e is CancellationException) throw e
OperationResult.Error(e.message ?: context.getString(R.string.error_indexing), e) OperationResult.Error(e.message ?: context.getString(R.string.error_indexing), e)
} }
} }
private fun deleteIndexFiles(indexFile: File, errors: MutableList<String>? = null) {
deleteFileTracked(indexFile, "index file", errors)
deleteFileTracked(DslIndexer.foldedIndexFile(indexFile), "legacy folded index file", errors)
}
private fun deleteFileTracked(file: File, label: String, errors: MutableList<String>?) {
try {
if (file.exists() && !file.delete()) {
errors?.add("Failed to delete $label: ${file.absolutePath}")
}
} catch (e: Exception) {
errors?.add("Failed to delete $label: ${file.absolutePath} (${e.message})")
}
}
private fun scanLocalDirectory(path: String): List<DiscoveredFile> { private fun scanLocalDirectory(path: String): List<DiscoveredFile> {
val dir = File(path) val dir = File(path)
if (!dir.exists() || !dir.isDirectory) return emptyList() if (!dir.exists() || !dir.isDirectory) return emptyList()
return dir.listFiles { f -> return dir.listFiles { file -> isDictionaryPayloadFile(file) }?.map { file ->
f.isFile && (f.name.endsWith(".dsl") || f.name.endsWith(".dsl.dz") || f.name.endsWith(".dsl.gz"))
}?.map { file ->
DiscoveredFile( DiscoveredFile(
name = file.name, name = file.name,
localPath = file.absolutePath, localPath = file.absolutePath,
@@ -351,10 +398,17 @@ class LocalDictionaryRepository(
} ?: emptyList() } ?: emptyList()
} }
private fun isDictionaryPayloadFile(file: File): Boolean =
file.isFile && (
file.name.endsWith(".dsl") ||
file.name.endsWith(".dsl.dz") ||
file.name.endsWith(".dsl.gz")
)
suspend fun search(query: String): OperationResult<List<IndexEntry>> = withContext(defaultDispatcher) { suspend fun search(query: String): OperationResult<List<IndexEntry>> = withContext(defaultDispatcher) {
if (query.isBlank()) return@withContext OperationResult.Success(emptyList()) if (query.isBlank()) return@withContext OperationResult.Success(emptyList())
val activeDicts = mutableDictionaries.value val activeDicts = mutableDictionaries.value.filter { it.isActive }
val sanitizedQuery = query.sanitizeQuery() val sanitizedQuery = query.sanitizeQuery()
ReSearchTrace.asyncSection(ReSearchTrace.SEARCH_DIRECT) { ReSearchTrace.asyncSection(ReSearchTrace.SEARCH_DIRECT) {
@@ -404,45 +458,63 @@ class LocalDictionaryRepository(
return engine.getIndexSearcher() return engine.getIndexSearcher()
} }
fun toggleDictionaryActive(dictionaryPath: String) { suspend fun toggleDictionaryActive(dictionaryPath: String) {
dictionaryStateMutex.withLock {
val isActive = mutableDictionaries.value
.firstOrNull { it.path == dictionaryPath }
?.isActive
?.not()
?: return
preferencesManager.setDictionaryActive(dictionaryPath, isActive)
mutableDictionaries.update { currentList -> mutableDictionaries.update { currentList ->
currentList.map { dict -> currentList.map { dict ->
if (dict.path == dictionaryPath) { if (dict.path == dictionaryPath) {
dict.withActiveStatus(!dict.isActive) dict.withActiveStatus(isActive)
} else { } else {
dict dict
} }
} }
} }
} }
fun deleteDictionary(dictionary: Dictionary): OperationResult<Unit> {
try {
mutableDictionaries.update { current ->
current.filter { it.path != dictionary.path }
} }
launch(ioDispatcher) { private fun removeDictionaryFromState(path: String) {
try { mutableDictionaries.update { current -> current.filter { it.path != path } }
val deleteErrors = mutableListOf<String>() }
suspend fun deleteDictionary(dictionary: Dictionary): OperationResult<Unit> {
return try {
val (failure, cleanupErrors) = withContext(ioDispatcher) {
val dictFile = File(dictionary.path) val dictFile = File(dictionary.path)
if (dictFile.exists()) { if (dictFile.exists() && !dictFile.delete()) {
if (!dictFile.delete()) { return@withContext Pair(
deleteErrors.add("Failed to delete dictionary file: ${dictFile.absolutePath}") "Failed to delete dictionary file: ${dictFile.absolutePath}",
} emptyList<String>(),
)
} }
val deleteErrors = mutableListOf<String>()
val indexFile = if (dictionary.indexPath.endsWith(".idx", ignoreCase = true)) { val indexFile = if (dictionary.indexPath.endsWith(".idx", ignoreCase = true)) {
File(dictionary.indexPath) File(dictionary.indexPath)
} else { } else {
File("${dictionary.indexPath}.idx") File("${dictionary.indexPath}.idx")
} }
if (indexFile.exists()) { deleteIndexFiles(indexFile, deleteErrors)
if (!indexFile.delete()) { Pair<String?, List<String>>(null, deleteErrors)
deleteErrors.add("Failed to delete index file: ${indexFile.absolutePath}")
} }
if (failure != null) {
return OperationResult.Error(failure)
}
dictionaryStateMutex.withLock {
try {
preferencesManager.removeDictionaryActiveState(dictionary.path)
} catch (e: Exception) {
Log.w(TAG, "Failed to clear dictionary active state: ${e.message}")
}
removeDictionaryFromState(dictionary.path)
} }
try { try {
@@ -451,18 +523,16 @@ class LocalDictionaryRepository(
Log.w(TAG, "Failed to trim memory after deletion: ${e.message}") Log.w(TAG, "Failed to trim memory after deletion: ${e.message}")
} }
if (deleteErrors.isNotEmpty()) { if (cleanupErrors.isNotEmpty()) {
val message = deleteErrors.joinToString("; ") Log.w(
Log.w(TAG, "Dictionary deletion completed with errors: $message") TAG,
} "Dictionary deletion completed with errors: ${cleanupErrors.joinToString("; ")}"
} catch (e: Exception) { )
Log.e(TAG, "Background deletion failed: ${e.message}", e)
}
} }
return OperationResult.Success(Unit) OperationResult.Success(Unit)
} catch (e: Exception) { } catch (e: Exception) {
return OperationResult.Error("Failed to delete dictionary", e) OperationResult.Error("Failed to delete dictionary", e)
} }
} }
@@ -507,13 +577,22 @@ class LocalDictionaryRepository(
if (count < 0) return null if (count < 0) return null
var articleCount = count
if (version >= 6) {
if (indexFile.length() < INDEX_V6_HEADER_BYTES) return null
articleCount = dataInput.readInt()
if (articleCount < 0) return null
}
if (version >= 2) { if (version >= 2) {
if (indexFile.length() < INDEX_V2_HEADER_BYTES) return null val fixedHeaderBytes =
if (version >= 6) INDEX_V6_HEADER_BYTES else INDEX_V2_HEADER_BYTES
if (indexFile.length() < fixedHeaderBytes) return null
val sparseCount = dataInput.readInt() val sparseCount = dataInput.readInt()
if (sparseCount < 0) return null if (sparseCount < 0) return null
var headerBytes = INDEX_V2_HEADER_BYTES var headerBytes = fixedHeaderBytes
repeat(sparseCount) { repeat(sparseCount) {
dataInput.readInt() dataInput.readInt()
dataInput.readLong() dataInput.readLong()
@@ -524,7 +603,7 @@ class LocalDictionaryRepository(
} }
} }
IndexHeader(version, count) IndexHeader(version, articleCount)
} }
} }
} catch (e: Exception) { } catch (e: Exception) {
@@ -602,6 +681,7 @@ class LocalDictionaryRepository(
const val MIN_PROGRESS_EMIT_INTERVAL_MS = 200L const val MIN_PROGRESS_EMIT_INTERVAL_MS = 200L
const val LEGACY_INDEX_HEADER_BYTES = 8L const val LEGACY_INDEX_HEADER_BYTES = 8L
const val INDEX_V2_HEADER_BYTES = 12 const val INDEX_V2_HEADER_BYTES = 12
const val INDEX_V6_HEADER_BYTES = 16
const val SPARSE_ENTRY_FIXED_BYTES = 14 const val SPARSE_ENTRY_FIXED_BYTES = 14
} }
} }
@@ -24,6 +24,7 @@ class DownloadManager(
mutableDownloadProgressState.asStateFlow() mutableDownloadProgressState.asStateFlow()
private val mutex = Mutex() private val mutex = Mutex()
private var downloadJob: Job? = null private var downloadJob: Job? = null
private var cancelCleanupJob: Job? = null
companion object { companion object {
private const val TAG = "DownloadManager" private const val TAG = "DownloadManager"
@@ -44,6 +45,7 @@ class DownloadManager(
fun startDownload() { fun startDownload() {
if (downloadState.value is DownloadState.Loading) return if (downloadState.value is DownloadState.Loading) return
if (!downloadScope.isActive) return if (!downloadScope.isActive) return
cancelCleanupJob = null
downloadJob = downloadScope.launch { downloadJob = downloadScope.launch {
mutex.withLock { mutex.withLock {
try { try {
@@ -94,7 +96,7 @@ class DownloadManager(
} catch (e: Exception) { } catch (e: Exception) {
if (e is CancellationException) { if (e is CancellationException) {
updateDownloadProgressState(DownloadState.Cancelled, 0f) updateDownloadProgressState(DownloadState.Cancelled, 0f)
downloadScope.launch { cancelCleanupJob = downloadScope.launch {
try { try {
dictionaryRepository.deleteUnprocessedFiles() dictionaryRepository.deleteUnprocessedFiles()
} catch (cleanupException: Exception) { } catch (cleanupException: Exception) {
@@ -124,6 +126,7 @@ class DownloadManager(
fun startDownloadForSources(sourceUrls: List<String>) { fun startDownloadForSources(sourceUrls: List<String>) {
if (downloadState.value is DownloadState.Loading) return if (downloadState.value is DownloadState.Loading) return
if (!downloadScope.isActive) return if (!downloadScope.isActive) return
cancelCleanupJob = null
downloadJob = downloadScope.launch { downloadJob = downloadScope.launch {
mutex.withLock { mutex.withLock {
try { try {
@@ -168,7 +171,7 @@ class DownloadManager(
} catch (e: Exception) { } catch (e: Exception) {
if (e is CancellationException) { if (e is CancellationException) {
updateDownloadProgressState(DownloadState.Cancelled, 0f) updateDownloadProgressState(DownloadState.Cancelled, 0f)
downloadScope.launch { cancelCleanupJob = downloadScope.launch {
try { try {
dictionaryRepository.deleteUnprocessedFiles() dictionaryRepository.deleteUnprocessedFiles()
} catch (cleanupException: Exception) { } catch (cleanupException: Exception) {
@@ -195,10 +198,12 @@ class DownloadManager(
} }
} }
fun cancelDownload() { fun cancelDownload() {
downloadJob?.cancel() val jobToCancel = downloadJob
jobToCancel?.cancel()
updateDownloadProgressState(DownloadState.Cancelled, 0f) updateDownloadProgressState(DownloadState.Cancelled, 0f)
downloadScope.launch { downloadScope.launch {
delay(TERMINAL_STATE_DURATION) jobToCancel?.join()
cancelCleanupJob?.join()
if (downloadState.value == DownloadState.Cancelled) { if (downloadState.value == DownloadState.Cancelled) {
updateDownloadProgressState(DownloadState.Idle, 0f) updateDownloadProgressState(DownloadState.Idle, 0f)
} }
@@ -59,11 +59,11 @@ class DictionaryCleaner(
toDelete.add(item.name) toDelete.add(item.name)
continue continue
} }
if ((item.ext == "idx" || item.ext == "dsl.idx") && item.date != latestDsl) { if (item.ext in indexExtensions && item.date != latestDsl) {
toDelete.add(item.name) toDelete.add(item.name)
continue continue
} }
if ((item.ext == "idx" || item.ext == "dsl.idx") && item.date !in dslDates) { if (item.ext in indexExtensions && item.date !in dslDates) {
toDelete.add(item.name) toDelete.add(item.name)
} }
} }
@@ -76,10 +76,13 @@ class DictionaryCleaner(
val date: String, val date: String,
val ext: String, val ext: String,
) )
private val validExtensions = setOf("dsl", "dsl.dz", "dsl.gz", "gz", "idx", "dsl.idx") private val indexExtensions = setOf("idx", "dsl.idx", "idx.fold", "dsl.idx.fold")
private val validExtensions = setOf("dsl", "dsl.dz", "dsl.gz", "gz") + indexExtensions
private fun parseManagedName(name: String, prefixes: Set<String>): ManagedFileName? { private fun parseManagedName(name: String, prefixes: Set<String>): ManagedFileName? {
val ext = when { val ext = when {
name.endsWith(".dsl.idx.fold") -> "dsl.idx.fold"
name.endsWith(".idx.fold") -> "idx.fold"
name.endsWith(".dsl.dz") -> "dsl.dz" name.endsWith(".dsl.dz") -> "dsl.dz"
name.endsWith(".dsl.gz") -> "dsl.gz" name.endsWith(".dsl.gz") -> "dsl.gz"
name.endsWith(".dsl.idx") -> "dsl.idx" name.endsWith(".dsl.idx") -> "dsl.idx"
@@ -343,6 +343,8 @@ class DictionaryDownloader(
context.getString(R.string.download_connection_reset) context.getString(R.string.download_connection_reset)
e is SecurityException -> e is SecurityException ->
context.getString(R.string.download_no_write_permission) context.getString(R.string.download_no_write_permission)
e.isNetworkError() ->
context.getString(R.string.download_network_failed)
else -> else ->
context.getString(R.string.download_error) context.getString(R.string.download_error)
} }
@@ -98,8 +98,7 @@ class SearchViewModel(
pageSize = 12, pageSize = 12,
prefetchDistance = 1, prefetchDistance = 1,
enablePlaceholders = false, enablePlaceholders = false,
initialLoadSize = 12, initialLoadSize = 12
jumpThreshold = 1
), ),
pagingSourceFactory = { pagingSourceFactory = {
IndexEntryPagingSource( IndexEntryPagingSource(
@@ -4,6 +4,7 @@ import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.offset
import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.windowInsetsPadding import androidx.compose.foundation.layout.windowInsetsPadding
@@ -135,7 +136,9 @@ private fun SearchField(
Icon( Icon(
painter = searchIcon, painter = searchIcon,
contentDescription = null, contentDescription = null,
modifier = Modifier.size(24.dp) modifier = Modifier
.offset(x = 4.dp)
.size(24.dp)
) )
}, },
trailingIcon = if (searchState.text.isNotEmpty()) { trailingIcon = if (searchState.text.isNotEmpty()) {
@@ -146,6 +149,7 @@ private fun SearchField(
onClearQuery() onClearQuery()
}, },
modifier = Modifier modifier = Modifier
.padding(end = 12.dp)
.testTag("clear_search") .testTag("clear_search")
.semantics { testTagsAsResourceId = true } .semantics { testTagsAsResourceId = true }
) { ) {
@@ -41,7 +41,6 @@ fun SearchResultsList(
) { ) {
items( items(
count = results.itemCount, count = results.itemCount,
key = results.itemKey { entry -> "${entry.dictionaryPath}_${entry.offset.value}_${entry.word}" },
contentType = results.itemContentType { "search_result" } contentType = results.itemContentType { "search_result" }
) { index -> ) { index ->
results[index]?.let { entry -> results[index]?.let { entry ->
@@ -11,6 +11,7 @@ import com.example.research.core.domain.model.AppTheme
import com.example.research.core.domain.model.Dictionary import com.example.research.core.domain.model.Dictionary
import com.example.research.core.domain.model.DictionarySource import com.example.research.core.domain.model.DictionarySource
import com.example.research.core.domain.model.IndexingProgress import com.example.research.core.domain.model.IndexingProgress
import com.example.research.core.domain.usecase.DictionarySourceFileMatcher
import com.example.research.core.domain.usecase.DictionarySourceValidator import com.example.research.core.domain.usecase.DictionarySourceValidator
import com.example.research.core.domain.usecase.ManageDictionarySourcesUseCase import com.example.research.core.domain.usecase.ManageDictionarySourcesUseCase
import com.example.research.core.util.OperationResult import com.example.research.core.util.OperationResult
@@ -30,7 +31,7 @@ import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import java.io.File import java.util.concurrent.ConcurrentHashMap
private data class DictionaryStateInputs( private data class DictionaryStateInputs(
val theme: AppTheme, val theme: AppTheme,
@@ -79,7 +80,7 @@ class SettingsViewModel(
private val language = MutableStateFlow("system") private val language = MutableStateFlow("system")
private val hasCompletedStartupScan = MutableStateFlow(false) private val hasCompletedStartupScan = MutableStateFlow(false)
private val pendingSourceUrls = mutableSetOf<String>() private val pendingSourceUrls = ConcurrentHashMap.newKeySet<String>()
private var isDownloadInProgress = false private var isDownloadInProgress = false
private var cancelRefreshPending = false private var cancelRefreshPending = false
@@ -158,19 +159,14 @@ class SettingsViewModel(
DictionaryStatus.UpToDate DictionaryStatus.UpToDate
} }
if (pendingSourceUrls.isNotEmpty()) { if (pendingSourceUrls.isNotEmpty()) {
pendingSourceUrls.forEach { urlTemplate -> pendingSourceUrls.forEach { urlTemplate ->
val source = dictionaryState.dictionarySources.find { val source = dictionaryState.dictionarySources.find {
it.urlTemplate == urlTemplate it.urlTemplate == urlTemplate
} }
if (source != null) { if (source != null) {
val hasDictionary = dictionaryState.dictionaries.any { dict -> val hasDictionary = dictionaryState.dictionaries.any { dictionary ->
DictionarySource.matchesDictionaryFile( DictionarySourceFileMatcher.matches(source, dictionary)
urlTemplate,
File(dict.path).name
)
} }
if (!hasDictionary) { if (!hasDictionary) {
manageDictionarySourcesUseCase.removeSource(source.id) manageDictionarySourcesUseCase.removeSource(source.id)
} }
@@ -264,18 +260,21 @@ class SettingsViewModel(
private suspend fun evaluateDictionaryStatus(): DictionaryStatus { private suspend fun evaluateDictionaryStatus(): DictionaryStatus {
return try { return try {
val actualDictionaries = localDictionaryRepository.dictionaries.first() val actualDictionaries = localDictionaryRepository.dictionaries.first()
val hasDictionaries = actualDictionaries.isNotEmpty()
if (!hasDictionaries) {
return DictionaryStatus.Empty
}
var sources = preferencesManager.dictionarySources.first() var sources = preferencesManager.dictionarySources.first()
if (downloadManager.downloadState.value !is DownloadState.Loading && if (pendingSourceUrls.isEmpty() &&
downloadManager.downloadState.value !is DownloadState.Loading &&
downloadManager.downloadState.value !is DownloadState.Extracting downloadManager.downloadState.value !is DownloadState.Extracting
) { ) {
val installedSources = installedSources(sources, actualDictionaries) val dictionaryFileNames = localDictionaryRepository.listDictionaryPayloadFileNames(
val installedSourceIds = installedSources.mapTo(mutableSetOf(), DictionarySource::id) preferencesManager.dictionaryPath
)
if (dictionaryFileNames != null) {
val installedSources = DictionarySourceFileMatcher.installedSourcesForFileNames(
sources,
dictionaryFileNames,
)
val installedSourceIds = installedSources
.mapTo(mutableSetOf(), DictionarySource::id)
val staleSourceIds = sources val staleSourceIds = sources
.filterNot { it.id in installedSourceIds } .filterNot { it.id in installedSourceIds }
.map(DictionarySource::id) .map(DictionarySource::id)
@@ -284,6 +283,12 @@ class SettingsViewModel(
sources = installedSources sources = installedSources
} }
} }
}
if (actualDictionaries.isEmpty()) {
return DictionaryStatus.Empty
}
val enabledSources = installedEnabledSources(sources, actualDictionaries) val enabledSources = installedEnabledSources(sources, actualDictionaries)
if (enabledSources.isEmpty()) { if (enabledSources.isEmpty()) {
@@ -331,26 +336,38 @@ class SettingsViewModel(
} }
private fun toggleDictionaryActive(dictionaryPath: String) { private fun toggleDictionaryActive(dictionaryPath: String) {
viewModelScope.launch {
localDictionaryRepository.toggleDictionaryActive(dictionaryPath) localDictionaryRepository.toggleDictionaryActive(dictionaryPath)
} }
}
private fun deleteDictionary(dictionary: Dictionary) { private fun deleteDictionary(dictionary: Dictionary) {
viewModelScope.launch { viewModelScope.launch {
localDictionaryRepository.deleteDictionary(dictionary) when (localDictionaryRepository.deleteDictionary(dictionary)) {
is OperationResult.Success -> {
manageDictionarySourcesUseCase.removeSourceForDictionary(dictionary) val remainingDictionaryFileNames =
localDictionaryRepository.listDictionaryPayloadFileNames(
val remainingDictionaries = localDictionaryRepository.dictionaries.first() preferencesManager.dictionaryPath
)
if (remainingDictionaries.isEmpty()) { manageDictionarySourcesUseCase.removeSourceForDictionary(
dictionaryStatus.value = DictionaryStatus.Empty dictionary = dictionary,
remainingDictionaryFileNames = remainingDictionaryFileNames,
)
dictionaryStatus.value = withContext(Dispatchers.IO) {
evaluateDictionaryStatus()
}
}
is OperationResult.Error -> {
effectChannel.send(
getApplication<Application>().getString(R.string.error_delete_dictionary)
)
}
} }
} }
} }
private fun addDictionarySources(urlTemplates: List<String>) { private fun addDictionarySources(urlTemplates: List<String>) {
viewModelScope.launch { viewModelScope.launch {
try {
val validUrls = mutableListOf<String>() val validUrls = mutableListOf<String>()
urlTemplates.forEach { urlTemplate -> urlTemplates.forEach { urlTemplate ->
@@ -359,23 +376,30 @@ class SettingsViewModel(
return@forEach return@forEach
} }
val normalizedUrl = DictionarySource.normalizeTemplate(trimmed)
if (!pendingSourceUrls.add(normalizedUrl)) {
return@forEach
}
try {
when (manageDictionarySourcesUseCase.addSource(trimmed)) { when (manageDictionarySourcesUseCase.addSource(trimmed)) {
is ManageDictionarySourcesUseCase.AddSourceResult.Success -> { is ManageDictionarySourcesUseCase.AddSourceResult.Success -> {
pendingSourceUrls.add(DictionarySource.normalizeTemplate(trimmed))
validUrls.add(trimmed) validUrls.add(trimmed)
} }
is ManageDictionarySourcesUseCase.AddSourceResult.ValidationFailed -> { is ManageDictionarySourcesUseCase.AddSourceResult.ValidationFailed -> {
pendingSourceUrls.remove(normalizedUrl)
// Validation failed, skip this source // Validation failed, skip this source
} }
} }
} catch (e: Exception) {
pendingSourceUrls.remove(normalizedUrl)
android.util.Log.e("SettingsViewModel", "Error adding dictionary source", e)
}
} }
if (validUrls.isNotEmpty()) { if (validUrls.isNotEmpty()) {
startDownloadForSources(validUrls) startDownloadForSources(validUrls)
} }
} catch (e: Exception) {
android.util.Log.e("SettingsViewModel", "Error adding dictionary sources", e)
}
} }
} }
@@ -446,14 +470,8 @@ class SettingsViewModel(
private fun installedSources( private fun installedSources(
sources: List<DictionarySource>, sources: List<DictionarySource>,
dictionaries: List<Dictionary> dictionaries: List<Dictionary>
): List<DictionarySource> { ): List<DictionarySource> =
val installedFileNames = dictionaries.map { File(it.path).name } DictionarySourceFileMatcher.installedSources(sources, dictionaries)
return sources.filter { source ->
installedFileNames.any { fileName ->
DictionarySource.matchesDictionaryFile(source.urlTemplate, fileName)
}
}
}
private fun cancelDownload() { private fun cancelDownload() {
// Cancel the ViewModel-side scan job so no further status recomputes // Cancel the ViewModel-side scan job so no further status recomputes
@@ -499,10 +517,11 @@ class SettingsViewModel(
is OperationResult.Success -> { is OperationResult.Success -> {
val dictionaries = localDictionaryRepository.dictionaries.first() val dictionaries = localDictionaryRepository.dictionaries.first()
if (dictionaries.isEmpty()) { if (dictionaries.isEmpty()) {
dictionaryStatus.value = DictionaryStatus.Empty dictionaryStatus.value = withContext(Dispatchers.IO) {
evaluateDictionaryStatus()
}
isDownloadInProgress = false isDownloadInProgress = false
} else { } else if (isDownloadInProgress) {
if (isDownloadInProgress) {
dictionaryStatus.value = DictionaryStatus.UpToDate dictionaryStatus.value = DictionaryStatus.UpToDate
isDownloadInProgress = false isDownloadInProgress = false
} else { } else {
@@ -510,7 +529,6 @@ class SettingsViewModel(
val status = withContext(Dispatchers.IO) { evaluateDictionaryStatus() } val status = withContext(Dispatchers.IO) { evaluateDictionaryStatus() }
dictionaryStatus.value = status dictionaryStatus.value = status
} }
}
if (result.data > 0) { if (result.data > 0) {
viewModelScope.launch(Dispatchers.Default) { viewModelScope.launch(Dispatchers.Default) {
@@ -30,6 +30,7 @@ import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clipToBounds
import androidx.compose.ui.draw.scale import androidx.compose.ui.draw.scale
import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.graphics.graphicsLayer
@@ -81,7 +82,8 @@ fun DictionaryListItem(
dictionary: Dictionary, dictionary: Dictionary,
onToggle: () -> Unit, onToggle: () -> Unit,
onDelete: () -> Unit, onDelete: () -> Unit,
modifier: Modifier = Modifier modifier: Modifier = Modifier,
canDelete: Boolean = true
) { ) {
val haptic = LocalHapticFeedback.current val haptic = LocalHapticFeedback.current
val density = LocalDensity.current val density = LocalDensity.current
@@ -93,7 +95,9 @@ fun DictionaryListItem(
val offsetAnim = remember { Animatable(0f) } val offsetAnim = remember { Animatable(0f) }
var rawOffset by remember { mutableFloatStateOf(0f) } var rawOffset by remember { mutableFloatStateOf(0f) }
val isDeleteRevealed by remember { derivedStateOf { rawOffset <= -swipeThresholdPx } } val isDeleteRevealed by remember(canDelete) {
derivedStateOf { canDelete && rawOffset <= -swipeThresholdPx }
}
LaunchedEffect(isDeleteRevealed) { LaunchedEffect(isDeleteRevealed) {
if (isDeleteRevealed) { if (isDeleteRevealed) {
@@ -101,6 +105,13 @@ fun DictionaryListItem(
} }
} }
LaunchedEffect(canDelete) {
if (!canDelete) {
rawOffset = 0f
offsetAnim.snapTo(0f)
}
}
val currentOnDelete by rememberUpdatedState(onDelete) val currentOnDelete by rememberUpdatedState(onDelete)
val currentOnToggle by rememberUpdatedState(onToggle) val currentOnToggle by rememberUpdatedState(onToggle)
@@ -121,6 +132,8 @@ fun DictionaryListItem(
Box( Box(
modifier = modifier modifier = modifier
.fillMaxWidth() .fillMaxWidth()
.height(DictionaryItemHeight)
.clipToBounds()
.testTag("dictionary_item") .testTag("dictionary_item")
) { ) {
Row( Row(
@@ -129,7 +142,8 @@ fun DictionaryListItem(
.height(DictionaryItemHeight) .height(DictionaryItemHeight)
.background(MaterialTheme.colorScheme.surfaceVariant) .background(MaterialTheme.colorScheme.surfaceVariant)
.graphicsLayer { translationX = offsetAnim.value } .graphicsLayer { translationX = offsetAnim.value }
.pointerInput(maxSwipePx, swipeThresholdPx) { .pointerInput(canDelete, maxSwipePx, swipeThresholdPx) {
if (!canDelete) return@pointerInput
detectHorizontalDragGestures( detectHorizontalDragGestures(
onDragStart = { onDragStart = {
haptic.performHapticFeedback(HapticFeedbackType.TextHandleMove) haptic.performHapticFeedback(HapticFeedbackType.TextHandleMove)
@@ -222,25 +236,11 @@ fun DictionaryListItem(
} }
if (isDeleteRevealed) { if (isDeleteRevealed) {
IconButton( Box(
onClick = {
haptic.performHapticFeedback(HapticFeedbackType.TextHandleMove)
currentOnDelete()
scope.launch { offsetAnim.snapTo(0f) }
rawOffset = 0f
},
modifier = Modifier modifier = Modifier
.padding(end = ItemHorizontalPadding) .padding(end = ItemHorizontalPadding)
.graphicsLayer { translationX = -offsetAnim.value }
.size(IconButtonSize) .size(IconButtonSize)
) {
Icon(
painter = painterResource(R.drawable.ic_delete),
contentDescription = stringResource(R.string.dictionary_delete),
tint = DeleteIconColor,
modifier = Modifier.size(IconSize)
) )
}
} else { } else {
Switch( Switch(
checked = dictionary.isActive, checked = dictionary.isActive,
@@ -257,5 +257,27 @@ fun DictionaryListItem(
) )
} }
} }
if (isDeleteRevealed) {
IconButton(
onClick = {
haptic.performHapticFeedback(HapticFeedbackType.TextHandleMove)
currentOnDelete()
scope.launch { offsetAnim.snapTo(0f) }
rawOffset = 0f
},
modifier = Modifier
.align(Alignment.CenterEnd)
.padding(end = ItemHorizontalPadding)
.size(IconButtonSize)
) {
Icon(
painter = painterResource(R.drawable.ic_delete),
contentDescription = stringResource(R.string.dictionary_delete),
tint = DeleteIconColor,
modifier = Modifier.size(IconSize)
)
}
}
} }
} }
@@ -166,7 +166,8 @@ fun DictionaryManagement(
onDeleteDictionary = onDeleteDictionary, onDeleteDictionary = onDeleteDictionary,
modifier = Modifier modifier = Modifier
.fillMaxWidth() .fillMaxWidth()
.then(backgroundModifier) .then(backgroundModifier),
canDelete = !isInProgress
) )
if (!isInProgress) { if (!isInProgress) {
@@ -246,7 +247,8 @@ private fun DictionaryListSection(
dictionaries: List<Dictionary>, dictionaries: List<Dictionary>,
onToggleDictionary: (String) -> Unit, onToggleDictionary: (String) -> Unit,
onDeleteDictionary: (Dictionary) -> Unit, onDeleteDictionary: (Dictionary) -> Unit,
modifier: Modifier = Modifier modifier: Modifier = Modifier,
canDelete: Boolean = true
) { ) {
Column( Column(
modifier = modifier, modifier = modifier,
@@ -256,7 +258,8 @@ private fun DictionaryListSection(
DictionaryListItem( DictionaryListItem(
dictionary = dictionary, dictionary = dictionary,
onToggle = { onToggleDictionary(dictionary.path) }, onToggle = { onToggleDictionary(dictionary.path) },
onDelete = { onDeleteDictionary(dictionary) } onDelete = { onDeleteDictionary(dictionary) },
canDelete = canDelete
) )
} }
} }
+2
View File
@@ -7,6 +7,7 @@
<string name="button_cancel_download">Отмена</string> <string name="button_cancel_download">Отмена</string>
<string name="action_ok">ОК</string> <string name="action_ok">ОК</string>
<string name="download_error">Ошибка загрузки словарей</string> <string name="download_error">Ошибка загрузки словарей</string>
<string name="download_network_failed">Нет связи с сервером словарей. Проверьте сеть и попробуйте снова</string>
<string name="language">Язык</string> <string name="language">Язык</string>
<string name="language_english">Английский</string> <string name="language_english">Английский</string>
<string name="language_russian">Русский</string> <string name="language_russian">Русский</string>
@@ -95,6 +96,7 @@
</plurals> </plurals>
<string name="dictionary_not_indexed">• Не индексирован</string> <string name="dictionary_not_indexed">• Не индексирован</string>
<string name="dictionary_delete">Удалить словарь</string> <string name="dictionary_delete">Удалить словарь</string>
<string name="error_delete_dictionary">Не удалось удалить словарь</string>
<string name="dictionaries_tap_to_update">Нажмите для обновления</string> <string name="dictionaries_tap_to_update">Нажмите для обновления</string>
<string name="notification_import_title">Импорт словарей</string> <string name="notification_import_title">Импорт словарей</string>
<string name="notification_import_success_title">Импорт завершён</string> <string name="notification_import_success_title">Импорт завершён</string>
+2
View File
@@ -7,6 +7,7 @@
<string name="button_cancel_download">Cancel</string> <string name="button_cancel_download">Cancel</string>
<string name="action_ok">OK</string> <string name="action_ok">OK</string>
<string name="download_error">Dictionary download failed</string> <string name="download_error">Dictionary download failed</string>
<string name="download_network_failed">No connection to the dictionary server. Check your network and try again</string>
<string name="language">Language</string> <string name="language">Language</string>
<string name="language_english">English</string> <string name="language_english">English</string>
<string name="language_russian">Russian</string> <string name="language_russian">Russian</string>
@@ -91,6 +92,7 @@
</plurals> </plurals>
<string name="dictionary_not_indexed">• Not indexed</string> <string name="dictionary_not_indexed">• Not indexed</string>
<string name="dictionary_delete">Delete dictionary</string> <string name="dictionary_delete">Delete dictionary</string>
<string name="error_delete_dictionary">Failed to delete dictionary</string>
<string name="dictionaries_tap_to_update">Tap to update all dictionaries</string> <string name="dictionaries_tap_to_update">Tap to update all dictionaries</string>
<string name="notification_import_title">Importing dictionaries</string> <string name="notification_import_title">Importing dictionaries</string>
<string name="notification_import_success_title">Import completed</string> <string name="notification_import_success_title">Import completed</string>
@@ -0,0 +1,4 @@
- Fixed a crash and lost dictionary state around search results
- Added diacritic-insensitive search, so accented and plain spellings match each other
- Fixed indexing failures on dictionaries with dense non-Latin text
- Search results now keep loading as you scroll instead of stopping early
@@ -0,0 +1 @@
- Fixed the dictionary update card taking too long to reappear after cancelling a download
@@ -0,0 +1,2 @@
- Dictionaries can no longer be deleted while an update is running, which used to lose their update source
- Fixed the dictionary icon sliding outside the row while swiping to delete
@@ -0,0 +1 @@
- Fixed matching source URLs to installed dictionary files, so deleted dictionaries can be downloaded again from the same URL
Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

@@ -0,0 +1,4 @@
- Исправлен сбой и потеря состояния словаря в результатах поиска
- Добавлен поиск без учёта диакритических знаков — совпадают варианты с акцентами и без
- Исправлена ошибка индексации словарей с плотным нелатинским текстом
- Результаты поиска теперь полностью подгружаются при прокрутке, а не обрываются на первых совпадениях
@@ -0,0 +1 @@
- Исправлена лишняя задержка перед повторным появлением карточки обновления словаря после отмены загрузки
@@ -0,0 +1,2 @@
- Словари больше нельзя удалить во время обновления — раньше при этом терялся источник обновления
- Исправлен выход иконки словаря за границы строки при свайпе
@@ -0,0 +1 @@
- Исправлено сопоставление URL-источников с установленными файлами — удалённый словарь теперь можно повторно скачать по тому же URL
Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

+3 -3
View File
@@ -1,12 +1,12 @@
[versions] [versions]
aboutlibraries = "15.0.3" aboutlibraries = "15.0.4"
activity_compose = "1.13.0" activity_compose = "1.13.0"
agp = "9.2.1" agp = "9.3.1"
compose_bom = "2026.06.01" compose_bom = "2026.06.01"
core = "1.19.0" core = "1.19.0"
datastore_preferences = "1.2.1" datastore_preferences = "1.2.1"
documentfile = "1.1.0" documentfile = "1.1.0"
kotlin = "2.4.0" kotlin = "2.4.10"
lifecycle_runtime_ktx = "2.11.0" lifecycle_runtime_ktx = "2.11.0"
okhttp = "5.4.0" okhttp = "5.4.0"
paging = "3.5.0" paging = "3.5.0"
+1 -1
View File
@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.1-bin.zip
networkTimeout=10000 networkTimeout=10000
retries=0 retries=0
retryBackOffMs=500 retryBackOffMs=500