fix(indexing): keep parsing progress moving
The parse phase estimated progress from a line-count formula that hit its ceiling early, so the percentage froze on large dictionaries. Estimate it from the position in the decompressed stream against the uncompressed size instead.
This commit is contained in:
1 parent
2fdcae6ae3
commit
c6ec14da3e
2 files changed
+36
-3
No files matched your search
@@ -49,6 +49,7 @@ class DslIndexer(context: Context? = null) {
|
||||
suspend fun createIndex(
|
||||
dslInputStream: java.io.InputStream,
|
||||
indexFile: File,
|
||||
totalInputBytes: Long = -1L,
|
||||
onProgress: (Int) -> Unit = {},
|
||||
onOperationChange: (String) -> Unit = {},
|
||||
onFileProgress: (Float) -> Unit = {},
|
||||
@@ -65,7 +66,7 @@ class DslIndexer(context: Context? = null) {
|
||||
try { foldedIndexFile(indexFile).delete() } catch (_: Throwable) { /* Legacy v5 leftover */ }
|
||||
onOperationChange("Parsing file")
|
||||
onFileProgress(0.0f)
|
||||
counts = parseDslToTempFile(bufferedStream, tempFile,
|
||||
counts = parseDslToTempFile(bufferedStream, tempFile, totalInputBytes,
|
||||
onProgress, onFileProgress, onCharsetDetected)
|
||||
onFileProgress(0.5f)
|
||||
onOperationChange("Sorting index")
|
||||
@@ -88,6 +89,7 @@ class DslIndexer(context: Context? = null) {
|
||||
private suspend fun parseDslToTempFile(
|
||||
inputStream: java.io.InputStream,
|
||||
tempFile: File,
|
||||
totalInputBytes: Long,
|
||||
onProgress: (Int) -> Unit,
|
||||
onFileProgress: (Float) -> Unit = {},
|
||||
onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {}
|
||||
@@ -230,8 +232,12 @@ class DslIndexer(context: Context? = null) {
|
||||
val yieldThreshold = adaptiveConfig.yieldInterval
|
||||
if (yieldCounter >= yieldThreshold) {
|
||||
onProgress(entryCount)
|
||||
val estimatedProgress = (lineCount * 10).toFloat() / (Int.MAX_VALUE / 1000)
|
||||
onFileProgress(estimatedProgress.coerceIn(0f, 0.4f))
|
||||
val parseProgress = if (totalInputBytes > 0L) {
|
||||
(reader.position.toFloat() / totalInputBytes.toFloat())
|
||||
} else {
|
||||
(lineCount * 10).toFloat() / (Int.MAX_VALUE / 1000)
|
||||
}
|
||||
onFileProgress(parseProgress.coerceIn(0f, 0.4f))
|
||||
|
||||
if (resourceManager != null) {
|
||||
if (resourceManager.shouldYield(adaptiveConfig)) {
|
||||
|
||||
@@ -22,6 +22,7 @@ import java.io.BufferedInputStream
|
||||
import java.io.File
|
||||
import java.io.FileInputStream
|
||||
import java.io.InputStream
|
||||
import java.io.RandomAccessFile
|
||||
import java.util.concurrent.CancellationException
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
@@ -51,12 +52,14 @@ class KotlinDictionaryEngine(private val context: Context) : ProgressIndexingEng
|
||||
|
||||
val indexFile = File(indexPath)
|
||||
val metadata = readMetadata(dictPath)
|
||||
val totalInputBytes = uncompressedInputLength(dictFile)
|
||||
|
||||
val articleCount = withContext(Dispatchers.IO) {
|
||||
openDslInputStream(dictPath).use { inputStream ->
|
||||
dslIndexer.createIndex(
|
||||
dslInputStream = inputStream,
|
||||
indexFile = indexFile,
|
||||
totalInputBytes = totalInputBytes,
|
||||
onProgress = { count ->
|
||||
onArticlesIndexed(count)
|
||||
},
|
||||
@@ -207,6 +210,30 @@ class KotlinDictionaryEngine(private val context: Context) : ProgressIndexingEng
|
||||
}
|
||||
}
|
||||
|
||||
private fun uncompressedInputLength(file: File): Long {
|
||||
val name = file.name.lowercase()
|
||||
if (!name.endsWith(".dz") && !name.endsWith(".gz")) return file.length()
|
||||
return readGzipUncompressedLength(file).takeIf { it > 0L } ?: file.length()
|
||||
}
|
||||
|
||||
private fun readGzipUncompressedLength(file: File): Long = try {
|
||||
RandomAccessFile(file, "r").use { raf ->
|
||||
val fileLength = raf.length()
|
||||
if (fileLength < 18L) {
|
||||
-1L
|
||||
} else {
|
||||
raf.seek(fileLength - 4L)
|
||||
val b0 = raf.readUnsignedByte()
|
||||
val b1 = raf.readUnsignedByte()
|
||||
val b2 = raf.readUnsignedByte()
|
||||
val b3 = raf.readUnsignedByte()
|
||||
(b0 or (b1 shl 8) or (b2 shl 16) or (b3 shl 24)).toLong() and 0xFFFFFFFFL
|
||||
}
|
||||
}
|
||||
} catch (_: Exception) {
|
||||
-1L
|
||||
}
|
||||
|
||||
private fun resolveIndexPath(indexArg: String): String {
|
||||
val candidate = File(indexArg)
|
||||
if (candidate.exists()) return candidate.absolutePath
|
||||
|
||||
Reference in new issue
Block a user