fix(indexing): keep parsing progress moving
The parse phase estimated progress from a line-count formula that hit its ceiling early, so the percentage froze on large dictionaries. Estimate it from the position in the decompressed stream against the uncompressed size instead.
This commit is contained in:
1 parent
2fdcae6ae3
commit
c6ec14da3e
2 files changed
+36
-3
No files matched your search
@@ -49,6 +49,7 @@ class DslIndexer(context: Context? = null) {
|
|||||||
suspend fun createIndex(
|
suspend fun createIndex(
|
||||||
dslInputStream: java.io.InputStream,
|
dslInputStream: java.io.InputStream,
|
||||||
indexFile: File,
|
indexFile: File,
|
||||||
|
totalInputBytes: Long = -1L,
|
||||||
onProgress: (Int) -> Unit = {},
|
onProgress: (Int) -> Unit = {},
|
||||||
onOperationChange: (String) -> Unit = {},
|
onOperationChange: (String) -> Unit = {},
|
||||||
onFileProgress: (Float) -> Unit = {},
|
onFileProgress: (Float) -> Unit = {},
|
||||||
@@ -65,7 +66,7 @@ class DslIndexer(context: Context? = null) {
|
|||||||
try { foldedIndexFile(indexFile).delete() } catch (_: Throwable) { /* Legacy v5 leftover */ }
|
try { foldedIndexFile(indexFile).delete() } catch (_: Throwable) { /* Legacy v5 leftover */ }
|
||||||
onOperationChange("Parsing file")
|
onOperationChange("Parsing file")
|
||||||
onFileProgress(0.0f)
|
onFileProgress(0.0f)
|
||||||
counts = parseDslToTempFile(bufferedStream, tempFile,
|
counts = parseDslToTempFile(bufferedStream, tempFile, totalInputBytes,
|
||||||
onProgress, onFileProgress, onCharsetDetected)
|
onProgress, onFileProgress, onCharsetDetected)
|
||||||
onFileProgress(0.5f)
|
onFileProgress(0.5f)
|
||||||
onOperationChange("Sorting index")
|
onOperationChange("Sorting index")
|
||||||
@@ -88,6 +89,7 @@ class DslIndexer(context: Context? = null) {
|
|||||||
private suspend fun parseDslToTempFile(
|
private suspend fun parseDslToTempFile(
|
||||||
inputStream: java.io.InputStream,
|
inputStream: java.io.InputStream,
|
||||||
tempFile: File,
|
tempFile: File,
|
||||||
|
totalInputBytes: Long,
|
||||||
onProgress: (Int) -> Unit,
|
onProgress: (Int) -> Unit,
|
||||||
onFileProgress: (Float) -> Unit = {},
|
onFileProgress: (Float) -> Unit = {},
|
||||||
onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {}
|
onCharsetDetected: (DslCharsetDetector.DetectedCharset) -> Unit = {}
|
||||||
@@ -230,8 +232,12 @@ class DslIndexer(context: Context? = null) {
|
|||||||
val yieldThreshold = adaptiveConfig.yieldInterval
|
val yieldThreshold = adaptiveConfig.yieldInterval
|
||||||
if (yieldCounter >= yieldThreshold) {
|
if (yieldCounter >= yieldThreshold) {
|
||||||
onProgress(entryCount)
|
onProgress(entryCount)
|
||||||
val estimatedProgress = (lineCount * 10).toFloat() / (Int.MAX_VALUE / 1000)
|
val parseProgress = if (totalInputBytes > 0L) {
|
||||||
onFileProgress(estimatedProgress.coerceIn(0f, 0.4f))
|
(reader.position.toFloat() / totalInputBytes.toFloat())
|
||||||
|
} else {
|
||||||
|
(lineCount * 10).toFloat() / (Int.MAX_VALUE / 1000)
|
||||||
|
}
|
||||||
|
onFileProgress(parseProgress.coerceIn(0f, 0.4f))
|
||||||
|
|
||||||
if (resourceManager != null) {
|
if (resourceManager != null) {
|
||||||
if (resourceManager.shouldYield(adaptiveConfig)) {
|
if (resourceManager.shouldYield(adaptiveConfig)) {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import java.io.BufferedInputStream
|
|||||||
import java.io.File
|
import java.io.File
|
||||||
import java.io.FileInputStream
|
import java.io.FileInputStream
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
|
import java.io.RandomAccessFile
|
||||||
import java.util.concurrent.CancellationException
|
import java.util.concurrent.CancellationException
|
||||||
import java.util.zip.GZIPInputStream
|
import java.util.zip.GZIPInputStream
|
||||||
|
|
||||||
@@ -51,12 +52,14 @@ class KotlinDictionaryEngine(private val context: Context) : ProgressIndexingEng
|
|||||||
|
|
||||||
val indexFile = File(indexPath)
|
val indexFile = File(indexPath)
|
||||||
val metadata = readMetadata(dictPath)
|
val metadata = readMetadata(dictPath)
|
||||||
|
val totalInputBytes = uncompressedInputLength(dictFile)
|
||||||
|
|
||||||
val articleCount = withContext(Dispatchers.IO) {
|
val articleCount = withContext(Dispatchers.IO) {
|
||||||
openDslInputStream(dictPath).use { inputStream ->
|
openDslInputStream(dictPath).use { inputStream ->
|
||||||
dslIndexer.createIndex(
|
dslIndexer.createIndex(
|
||||||
dslInputStream = inputStream,
|
dslInputStream = inputStream,
|
||||||
indexFile = indexFile,
|
indexFile = indexFile,
|
||||||
|
totalInputBytes = totalInputBytes,
|
||||||
onProgress = { count ->
|
onProgress = { count ->
|
||||||
onArticlesIndexed(count)
|
onArticlesIndexed(count)
|
||||||
},
|
},
|
||||||
@@ -207,6 +210,30 @@ class KotlinDictionaryEngine(private val context: Context) : ProgressIndexingEng
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun uncompressedInputLength(file: File): Long {
|
||||||
|
val name = file.name.lowercase()
|
||||||
|
if (!name.endsWith(".dz") && !name.endsWith(".gz")) return file.length()
|
||||||
|
return readGzipUncompressedLength(file).takeIf { it > 0L } ?: file.length()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun readGzipUncompressedLength(file: File): Long = try {
|
||||||
|
RandomAccessFile(file, "r").use { raf ->
|
||||||
|
val fileLength = raf.length()
|
||||||
|
if (fileLength < 18L) {
|
||||||
|
-1L
|
||||||
|
} else {
|
||||||
|
raf.seek(fileLength - 4L)
|
||||||
|
val b0 = raf.readUnsignedByte()
|
||||||
|
val b1 = raf.readUnsignedByte()
|
||||||
|
val b2 = raf.readUnsignedByte()
|
||||||
|
val b3 = raf.readUnsignedByte()
|
||||||
|
(b0 or (b1 shl 8) or (b2 shl 16) or (b3 shl 24)).toLong() and 0xFFFFFFFFL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (_: Exception) {
|
||||||
|
-1L
|
||||||
|
}
|
||||||
|
|
||||||
private fun resolveIndexPath(indexArg: String): String {
|
private fun resolveIndexPath(indexArg: String): String {
|
||||||
val candidate = File(indexArg)
|
val candidate = File(indexArg)
|
||||||
if (candidate.exists()) return candidate.absolutePath
|
if (candidate.exists()) return candidate.absolutePath
|
||||||
|
|||||||
Reference in new issue
Block a user