fix(indexing): keep parsing progress moving

The parse phase estimated progress from a line-count formula that hit
its ceiling early, so the percentage froze on large dictionaries.
Estimate it from the position in the decompressed stream against the
uncompressed size instead.
This commit is contained in:
OneWay committed 2026-10-11 22:25:23 +08:00
1 parent 2fdcae6ae3
commit c6ec14da3e
2 files changed
+36 -3

No files matched your search

@@ -49,6 +49,7 @@ class DslIndexer(context: Context? = null) {
suspend fun createIndex( suspend fun createIndex(
dslInputStream: java.io.InputStream, dslInputStream: java.io.InputStream,
indexFile: File, indexFile: File,
totalInputBytes: Long = -1L,
onProgress: (Int) -> Unit = {}, onProgress: (Int) -> Unit = {},
onOperationChange: (String) -> Unit = {}, onOperationChange: (String) -> Unit = {},
onFileProgress: (Float) -> Unit = {}, onFileProgress: (Float) -> Unit = {},
@@ -65,7 +66,7 @@ class DslIndexer(context: Context? = null) {
try { foldedIndexFile(indexFile).delete() } catch (_: Throwable) { /* Legacy v5 leftover */ } try { foldedIndexFile(indexFile).delete() } catch (_: Throwable) { /* Legacy v5 leftover */ }
onOperationChange("Parsing file") onOperationChange("Parsing file")
onFileProgress(0.0f) onFileProgress(0.0f)
counts = parseDslToTempFile(bufferedStream, tempFile, counts = parseDslToTempFile(bufferedStream, tempFile, totalInputBytes,
onProgress, onFileProgress, onCharsetDetected) onProgress, onFileProgress, onCharsetDetected)
onFileProgress(0.5f) onFileProgress(0.5f)
onOperationChange("Sorting index") onOperationChange("Sorting index")
@@ -88,6 +89,7 @@ class DslIndexer(context: Context? = null) {
private suspend fun parseDslToTempFile( private suspend fun parseDslToTempFile(
inputStream: java.io.InputStream, inputStream: java.io.InputStream,
tempFile: File, tempFile: File,
totalInputBytes: Long,
onProgress: (Int) -> Unit, onProgress: (Int) -> Unit,
onFileProgress: (Float) -> Unit = {}, onFileProgress: (Float) -> Unit = {},
onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {} onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {}
@@ -230,8 +232,12 @@ class DslIndexer(context: Context? = null) {
val yieldThreshold = adaptiveConfig.yieldInterval val yieldThreshold = adaptiveConfig.yieldInterval
if (yieldCounter >= yieldThreshold) { if (yieldCounter >= yieldThreshold) {
onProgress(entryCount) onProgress(entryCount)
val estimatedProgress = (lineCount * 10).toFloat() / (Int.MAX_VALUE / 1000) val parseProgress = if (totalInputBytes > 0L) {
onFileProgress(estimatedProgress.coerceIn(0f, 0.4f)) (reader.position.toFloat() / totalInputBytes.toFloat())
} else {
(lineCount * 10).toFloat() / (Int.MAX_VALUE / 1000)
}
onFileProgress(parseProgress.coerceIn(0f, 0.4f))
if (resourceManager != null) { if (resourceManager != null) {
if (resourceManager.shouldYield(adaptiveConfig)) { if (resourceManager.shouldYield(adaptiveConfig)) {
@@ -22,6 +22,7 @@ import java.io.BufferedInputStream
import java.io.File import java.io.File
import java.io.FileInputStream import java.io.FileInputStream
import java.io.InputStream import java.io.InputStream
import java.io.RandomAccessFile
import java.util.concurrent.CancellationException import java.util.concurrent.CancellationException
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
@@ -51,12 +52,14 @@ class KotlinDictionaryEngine(private val context: Context) : ProgressIndexingEng
val indexFile = File(indexPath) val indexFile = File(indexPath)
val metadata = readMetadata(dictPath) val metadata = readMetadata(dictPath)
val totalInputBytes = uncompressedInputLength(dictFile)
val articleCount = withContext(Dispatchers.IO) { val articleCount = withContext(Dispatchers.IO) {
openDslInputStream(dictPath).use { inputStream -> openDslInputStream(dictPath).use { inputStream ->
dslIndexer.createIndex( dslIndexer.createIndex(
dslInputStream = inputStream, dslInputStream = inputStream,
indexFile = indexFile, indexFile = indexFile,
totalInputBytes = totalInputBytes,
onProgress = { count -> onProgress = { count ->
onArticlesIndexed(count) onArticlesIndexed(count)
}, },
@@ -207,6 +210,30 @@ class KotlinDictionaryEngine(private val context: Context) : ProgressIndexingEng
} }
} }
private fun uncompressedInputLength(file: File): Long {
val name = file.name.lowercase()
if (!name.endsWith(".dz") && !name.endsWith(".gz")) return file.length()
return readGzipUncompressedLength(file).takeIf { it > 0L } ?: file.length()
}
private fun readGzipUncompressedLength(file: File): Long = try {
RandomAccessFile(file, "r").use { raf ->
val fileLength = raf.length()
if (fileLength < 18L) {
-1L
} else {
raf.seek(fileLength - 4L)
val b0 = raf.readUnsignedByte()
val b1 = raf.readUnsignedByte()
val b2 = raf.readUnsignedByte()
val b3 = raf.readUnsignedByte()
(b0 or (b1 shl 8) or (b2 shl 16) or (b3 shl 24)).toLong() and 0xFFFFFFFFL
}
}
} catch (_: Exception) {
-1L
}
private fun resolveIndexPath(indexArg: String): String { private fun resolveIndexPath(indexArg: String): String {
val candidate = File(indexArg) val candidate = File(indexArg)
if (candidate.exists()) return candidate.absolutePath if (candidate.exists()) return candidate.absolutePath