diff --git a/JVM_RUNTIME.md b/JVM_RUNTIME.md new file mode 100644 index 00000000..1974ef1c --- /dev/null +++ b/JVM_RUNTIME.md @@ -0,0 +1,75 @@ +# Lucene on a JVM, on every platform + +Lucene needs a JVM, and keeps up with the newest Java (21 for Lucene 10, 25 for Lucene 11): ART can't, so the search +runs Lucene on a real JVM everywhere, and everything else (query building, dictionary, ranking, snippets, highlights, +fusion, the semantic model) in common Kotlin. The bridge between both is +[Nucleus JVM Access](https://github.com/NucleusFramework/NucleusJvmAccess) (NJA). + +``` +search (Kotlin Multiplatform) search-facade (Kotlin/JVM): what runs on the JVM +────────────────────────────── ───────────────────────────────────────────────── +LexicalSearchEngine, MagicDictionaryIndex, snippets, +highlights... use the façade's API as if it were LuceneIndex(indexDir, cacheQueries): analyze, search, +local: io.github...search.index.LuceneIndex, ──▶ cursors, facets, collectLongs, count, KNN +IndexQuery, IndexPage, KnnRequest... (generated by IndexQuery (a sealed value), IndexPage, KnnRequest... +NJA from the façade's sources) LuceneIndexBackend (internal): Lucene 10 +``` + +| | Desktop (JVM) | Android 10+ | iOS | +|---|---|---|---| +| The JVM | the app's own | HotSpot (server VM, JIT), in the app's process next to ART | none: the façade compiled ahead of time into a GraalVM native image, linked into the app (iOS forbids generated code) | +| A call | a plain call into NJA's dispatcher, on the calling thread | handed to a HotSpot thread paired with the calling one | a call into the image's entry point, on the calling thread | + +- `search-facade` holds the façade: its public API (`LuceneIndex`, `IndexCursor` and the value types: data classes, + enums, the sealed `IndexQuery`) is what NJA exports; `LuceneIndexBackend` is internal. +- NJA generates, from the façade's sources, the proxies and value types `search`'s common code uses, in + `io.github.kdroidfilter.seforimlibrary.search.index` (`njaFacade.clientPackage`: apart from the façade's own package, + as the desktop has both on its class path). Query helpers built in the app (`bool { }`) stay in common code. +- `LuceneIndex(path)` is the same constructor on every platform; `close()` closes the JVM's index. + +## The runtimes + +`nucleusJvm.iosRuntime` is `GRAALVM`: Zero, jdk-mobile's interpreter, searched about 8× slower than the native image +(a first page 1.2 s against 140 ms in the simulator, on real data). Android keeps HotSpot: an Android native image +builds on Linux only, and HotSpot is the fastest once warm. + +- **iOS**: NJA downloads [graalvm-mobile](https://github.com/NucleusFramework/graalvm-mobile)'s latest kit (GraalVM + CE's static libraries, for iOS) and the GraalVM CE release it names, for the build host, and compiles the façade + into `search/build/nja/nativeImage//nja.o` (`njaNativeImage`). The binaries link it with the kit's + libraries (`-export_dynamic`: NJA's client finds the image's entry points at run time): nothing to bundle. Lucene's + reflection and resources are in the façade's jar (`META-INF/native-image/.../reachability-metadata.json`), traced on + the JVM tests with the real index: `./gradlew :search:jvmTest -PindexDir= + -PnativeImageAgent=` (then drop what the tests alone reach). The image reads the index through NIO. + For `search`'s own tests NJA does it all; an app links as Zayit's `searchkit` shows (`assembleIosJavaRuntimeDevice`: + `static/`, the object and the kit's libraries). + +Android's JVM is jdk-mobile's: a stable OpenJDK (27, `openjdk/jdk27u`'s GA releases) with openjdk/mobile's port, +built by the CI of [NucleusFramework/openjdk-mobile](https://github.com/NucleusFramework/openjdk-mobile) and published +as releases named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.3`: `jdk-27-ga`, the +JDK's build 35). NJA downloads the latest one (the runtimes `search` needs: arm64-v8a and x86_64 Android), +checks them against their `.sha256`, keeps them in Gradle's cache and extracts them into `search/build/nja/jdkMobile`; +`nucleusJvm.jdkMobileRelease` pins one. + +- **Android**: NJA links HotSpot and the JDK's natives into `libnja.so` (one per ABI: arm64-v8a, x86_64) and puts the + runtime (`java.home` per ABI, the façade's class path, the VM's options) in `search`'s assets, extracted once per + version on the device: an app depending on `search` gets both through the library, with nothing to configure. +- **The desktop**: the façade is on the app's class path; nothing else. +- The VM's options are `nucleusJvm.jvmOptions` (a 128 MB heap: about 140 MB of memory on Android; the native image + keeps the heap's maximum, about 60 MB of memory on iOS for Lucene alone). + +## Tests + +```bash +./gradlew :search:jvmTest -PindexDir= -PluceneQueryCache=off # build/parity/jvm.txt +./gradlew :search:iosSimulatorArm64Test -PindexDir= -PluceneQueryCache=off # build/parity/ios.txt +``` + +They dump what the search answers on a real index (`SearchDump`). `-PluceneQueryCache=off` turns off Lucene's query +cache: it changes the last digits of the scores from one search to the next. Two runs differ anyway by Lucene's +process-dependent clause order (`BooleanQuery` rewrites through a `HashMap` keyed by class identity hashes), so the +comparison allows score differences: facets, snippets and highlights must be identical. On Android, Zayit's +`searchkit-check` app runs the whole search on a device or an emulator. + +Verified on the real index (7M documents): the desktop, the iOS simulator and Android 11 emulator give the same facets, +snippets, highlights, titles and find-in-book candidates (Zayit's `searchkit`, hybrid with the semantic search: the +same dump as the desktop, but for a hit total Lucene counts as a lower bound). diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts new file mode 100644 index 00000000..b4003baa --- /dev/null +++ b/cache/build.gradle.kts @@ -0,0 +1,68 @@ + +plugins { + alias(libs.plugins.multiplatform) + alias(libs.plugins.android.library) + alias(libs.plugins.maven.publish) +} + +group = "io.github.kdroidfilter.seforimlibrary" + +/** Caches read and written from several threads at once, on every platform (ConcurrentCache.kt). */ +kotlin { + jvmToolchain(libs.versions.jvmToolchain.get().toInt()) + + android { + namespace = "io.github.kdroidfilter.seforimlibrary.cache" + compileSdk = 35 + minSdk = 21 + } + jvm() + iosArm64() + iosSimulatorArm64() + + sourceSets { + commonTest.dependencies { + implementation(kotlin("test")) + implementation(libs.kotlinx.coroutines.core) + } + + // No java.util.concurrent on iOS: cachemap's map, lock-free reads + iosMain.dependencies { + implementation(libs.cachemap) + } + } + +} + +//Publishing your Kotlin Multiplatform library to Maven Central +//https://www.jetbrains.com/help/kotlin-multiplatform-dev/multiplatform-publish-libraries.html +mavenPublishing { + publishToMavenCentral() + coordinates("io.github.kdroidfilter.seforimlibrary", "cache", "1.0.0") + + pom { + name = "SeforimLibrary" + description = "Kotlin Multiplatform library" + url = "github url" //todo + + licenses { + license { + name = "MIT" + url = "https://opensource.org/licenses/MIT" + } + } + + developers { + developer { + id = "" //todo + name = "" //todo + email = "" //todo + } + } + + scm { + url = "github url" //todo + } + } + if (project.hasProperty("signing.keyId")) signAllPublications() +} diff --git a/cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.android.kt b/cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.android.kt new file mode 100644 index 00000000..1af2f411 --- /dev/null +++ b/cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.android.kt @@ -0,0 +1,17 @@ +package io.github.kdroidfilter.seforimlibrary.cache + +import java.util.concurrent.ConcurrentHashMap + +actual fun concurrentCache(): ConcurrentCache = + object : ConcurrentCache { + private val map = ConcurrentHashMap() + + override fun get(key: K): V? = map[key] + + override fun set( + key: K, + value: V, + ) { + map[key] = value + } + } diff --git a/cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.kt b/cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.kt new file mode 100644 index 00000000..333ffe59 --- /dev/null +++ b/cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.kt @@ -0,0 +1,14 @@ +package io.github.kdroidfilter.seforimlibrary.cache + +/** A cache read and written from several threads at once: concurrent lookups of the same key may both load it. */ +interface ConcurrentCache { + operator fun get(key: K): V? + + operator fun set( + key: K, + value: V, + ) +} + +/** On the JVM and Android, ConcurrentHashMap; on iOS (no java.util.concurrent), cachemap's lock-free reads. */ +expect fun concurrentCache(): ConcurrentCache diff --git a/cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt b/cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt new file mode 100644 index 00000000..bf375002 --- /dev/null +++ b/cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt @@ -0,0 +1,31 @@ +package io.github.kdroidfilter.seforimlibrary.cache + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class ConcurrentCacheTest { + @Test + fun storesAndReplacesValues() { + val cache = concurrentCache() + assertNull(cache["a"]) + cache["a"] = 1 + cache["a"] = 2 + assertEquals(2, cache["a"]) + } + + @Test + fun keepsEveryValueWrittenFromSeveralThreads() = + runBlocking { + val cache = concurrentCache() + (0 until 8) + .map { worker -> + async(Dispatchers.Default) { for (i in 0 until 1_000) cache[worker * 1_000 + i] = i } + }.awaitAll() + for (key in 0 until 8_000) assertEquals(key % 1_000, cache[key]) + } +} diff --git a/cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.ios.kt b/cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.ios.kt new file mode 100644 index 00000000..09694e10 --- /dev/null +++ b/cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.ios.kt @@ -0,0 +1,17 @@ +package io.github.kdroidfilter.seforimlibrary.cache + +import io.github.charlietap.cachemap.cacheMapOf + +actual fun concurrentCache(): ConcurrentCache = + object : ConcurrentCache { + private val map = cacheMapOf() + + override fun get(key: K): V? = map[key] + + override fun set( + key: K, + value: V, + ) { + map[key] = value + } + } diff --git a/cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.jvm.kt b/cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.jvm.kt new file mode 100644 index 00000000..1af2f411 --- /dev/null +++ b/cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.jvm.kt @@ -0,0 +1,17 @@ +package io.github.kdroidfilter.seforimlibrary.cache + +import java.util.concurrent.ConcurrentHashMap + +actual fun concurrentCache(): ConcurrentCache = + object : ConcurrentCache { + private val map = ConcurrentHashMap() + + override fun get(key: K): V? = map[key] + + override fun set( + key: K, + value: V, + ) { + map[key] = value + } + } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 56376722..a3edcbef 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -12,11 +12,13 @@ kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) android { - namespace = "io.github.kdroidfilter.seforimlibrary" + namespace = "io.github.kdroidfilter.seforimlibrary.core" compileSdk = 35 minSdk = 21 } jvm() + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { diff --git a/dao/build.gradle.kts b/dao/build.gradle.kts index c8e42a0a..2a5b5cb4 100644 --- a/dao/build.gradle.kts +++ b/dao/build.gradle.kts @@ -14,20 +14,24 @@ kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) android { - namespace = "io.github.kdroidfilter.seforimlibrary" + namespace = "io.github.kdroidfilter.seforimlibrary.dao" compileSdk = 35 minSdk = 21 } jvm() + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { api(project(":core")) + implementation(project(":cache")) implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.serialization.json) implementation(libs.kotlinx.serialization.protobuf) implementation(libs.kotlinx.datetime) implementation(libs.kermit) + implementation(libs.kotlinx.io.core) } commonTest.dependencies { @@ -48,6 +52,10 @@ kotlin { implementation(libs.sqlDelight.driver.sqlite) } + iosMain.dependencies { + implementation(libs.sqlDelight.driver.native) + } + } } diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt index cd9990a3..94eb1e11 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt @@ -1,18 +1,19 @@ package io.github.kdroidfilter.seforimlibrary.dao import io.github.kdroidfilter.seforimlibrary.core.models.PrecomputedCatalog +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.readByteArray import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.protobuf.ProtoBuf -import java.io.File -import java.nio.file.Path -import kotlin.io.path.exists -import kotlin.io.path.readBytes /** * Loader for the precomputed catalog tree. * Provides fast loading of the entire book/category tree from a binary file. */ object CatalogLoader { + private const val CATALOG_FILE = "catalog.pb" /** * Loads the precomputed catalog from a file next to the database. @@ -20,53 +21,15 @@ object CatalogLoader { * @param dbPath Path to the database file. The catalog file should be in the same directory. * @return The precomputed catalog, or null if the file doesn't exist or can't be loaded */ - @OptIn(ExperimentalSerializationApi::class) - fun loadCatalog(dbPath: String): PrecomputedCatalog? { - return try { - val dbFile = File(dbPath) - val catalogFile = File(dbFile.parentFile, "catalog.pb") - - if (!catalogFile.exists()) { - println("Catalog file not found: ${catalogFile.absolutePath}") - return null - } - - val bytes = catalogFile.readBytes() - val catalog = ProtoBuf.decodeFromByteArray(PrecomputedCatalog.serializer(), bytes) - - println("Loaded precomputed catalog: ${catalog.totalCategories} categories, ${catalog.totalBooks} books") - catalog - } catch (e: Exception) { - println("Failed to load precomputed catalog: ${e.message}") - e.printStackTrace() - null - } - } + fun loadCatalog(dbPath: String): PrecomputedCatalog? = load(catalogBeside(dbPath)) /** - * Loads the precomputed catalog from a specific Path. + * Loads the precomputed catalog from a specific path. * * @param catalogPath Direct path to the catalog.pb file * @return The precomputed catalog, or null if the file doesn't exist or can't be loaded */ - fun loadCatalogFromPath(catalogPath: Path): PrecomputedCatalog? { - return try { - if (!catalogPath.exists()) { - println("Catalog file not found: $catalogPath") - return null - } - - val bytes = catalogPath.readBytes() - val catalog = ProtoBuf.decodeFromByteArray(PrecomputedCatalog.serializer(), bytes) - - println("Loaded precomputed catalog: ${catalog.totalCategories} categories, ${catalog.totalBooks} books") - catalog - } catch (e: Exception) { - println("Failed to load precomputed catalog: ${e.message}") - e.printStackTrace() - null - } - } + fun loadCatalogFromPath(catalogPath: String): PrecomputedCatalog? = load(Path(catalogPath)) /** * Checks if a catalog file exists next to the database. @@ -74,9 +37,28 @@ object CatalogLoader { * @param dbPath Path to the database file * @return true if catalog.pb exists, false otherwise */ - fun catalogExists(dbPath: String): Boolean { - val dbFile = File(dbPath) - val catalogFile = File(dbFile.parentFile, "catalog.pb") - return catalogFile.exists() + fun catalogExists(dbPath: String): Boolean = SystemFileSystem.exists(catalogBeside(dbPath)) + + private fun catalogBeside(dbPath: String): Path { + val parent = Path(dbPath).parent + return if (parent != null) Path(parent, CATALOG_FILE) else Path(CATALOG_FILE) } + + @OptIn(ExperimentalSerializationApi::class) + private fun load(catalogFile: Path): PrecomputedCatalog? = + try { + if (!SystemFileSystem.exists(catalogFile)) { + println("Catalog file not found: $catalogFile") + null + } else { + val bytes = SystemFileSystem.source(catalogFile).buffered().use { it.readByteArray() } + ProtoBuf.decodeFromByteArray(PrecomputedCatalog.serializer(), bytes).also { + println("Loaded precomputed catalog: ${it.totalCategories} categories, ${it.totalBooks} books") + } + } + } catch (e: Exception) { + println("Failed to load precomputed catalog: ${e.message}") + e.printStackTrace() + null + } } diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt index 75136b5e..e02f7ba2 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt @@ -7,6 +7,7 @@ import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.db.SqlDriver import co.touchlab.kermit.Logger import co.touchlab.kermit.Severity +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.core.models.AltTocEntry import io.github.kdroidfilter.seforimlibrary.core.models.AltTocStructure import io.github.kdroidfilter.seforimlibrary.core.models.Author @@ -26,6 +27,7 @@ import io.github.kdroidfilter.seforimlibrary.dao.extensions.toModel import io.github.kdroidfilter.seforimlibrary.db.SeforimDb import io.github.kdroidfilter.seforimlibrary.env.getEnvironmentVariable import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json @@ -44,19 +46,18 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L // Category rows are immutable at runtime, so a plain read-through cache avoids // paying a SQLite prepare + connection round-trip for every breadcrumb/tree walk. // Profiling (see JFR 2026-04-23) showed 70 `sqlite3_prepare` calls in 20 s all - // coming from `getCategory`. Both KMP targets (JVM + Android) are JVM-based so - // `ConcurrentHashMap` is safe to use directly from commonMain. - private val categoryCache = java.util.concurrent.ConcurrentHashMap() + // coming from `getCategory`. + private val categoryCache = concurrentCache() // book.orderIndex is denormalized onto each link row at insert time (targetBookOrderIndex) // so the commentaries path can sort without JOINing book. Books are immutable once inserted // and exist before any link references them, so a read-through cache is safe during import. - private val bookOrderIndexCache = java.util.concurrent.ConcurrentHashMap() + private val bookOrderIndexCache = concurrentCache() private fun resolveBookOrderIndex(bookId: Long): Long = - bookOrderIndexCache.getOrPut(bookId) { - database.bookQueriesQueries.selectOrderIndexById(bookId).executeAsOneOrNull() ?: 0L - } + bookOrderIndexCache[bookId] + ?: (database.bookQueriesQueries.selectOrderIndexById(bookId).executeAsOneOrNull() ?: 0L) + .also { bookOrderIndexCache[bookId] = it } init { val repositoryLoggingEnv = getEnvironmentVariable("SEFORIMAPP_REPOSITORY_LOGGING")?.lowercase() @@ -69,11 +70,11 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L // Create the database schema (fresh builds only; no runtime migrations needed) SeforimDb.Schema.create(driver) // SQLite optimizations (balanced mode for desktop: 256MB cache + 512MB mmap) - driver.execute(null, "PRAGMA journal_mode=WAL", 0) - driver.execute(null, "PRAGMA synchronous=NORMAL", 0) - driver.execute(null, "PRAGMA cache_size=-256000", 0) // 256MB cache (negative = KiB) - driver.execute(null, "PRAGMA mmap_size=536870912", 0) // 512MB memory-mapped I/O - driver.execute(null, "PRAGMA temp_store=MEMORY", 0) + pragma("journal_mode=WAL") + pragma("synchronous=NORMAL") + pragma("cache_size=-256000") // 256MB cache (negative = KiB) + pragma("mmap_size=536870912") // 512MB memory-mapped I/O + pragma("temp_store=MEMORY") // Check if the database is empty try { @@ -96,6 +97,17 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L + /** + * Sets a PRAGMA on every driver: some answer with a row (journal_mode, mmap_size), which the Android and native + * drivers only accept as a query, and the JDBC one only as a statement. + */ + private fun pragma(assignment: String) { + val sql = "PRAGMA $assignment" + runCatching { driver.execute(null, sql, 0) } + .recoverCatching { driver.executeQuery(null, sql, { cursor -> QueryResult.Value(cursor.next().value) }, 0) } + .onFailure { logger.w(it) { "Couldn't set $sql" } } + } + // --- Line ⇄ TOC mapping --- /** @@ -275,7 +287,9 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L val loaded = withContext(Dispatchers.IO) { database.categoryQueriesQueries.selectById(id).executeAsOneOrNull()?.toModel() } ?: return null - return categoryCache.putIfAbsent(id, loaded) ?: loaded + // Rows are immutable: a concurrent load writes the same value + categoryCache[id] = loaded + return loaded } /** diff --git a/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt b/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt new file mode 100644 index 00000000..134d1b20 --- /dev/null +++ b/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt @@ -0,0 +1,8 @@ +package io.github.kdroidfilter.seforimlibrary.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.toKString +import platform.posix.getenv + +@OptIn(ExperimentalForeignApi::class) +actual fun getEnvironmentVariable(name: String): String? = getenv(name)?.toKString() diff --git a/gradle.properties b/gradle.properties index a6460d48..e0b5e18d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,3 +14,4 @@ kotlin.incremental.wasm=true #Android android.useAndroidX=true android.nonTransitiveRClass=true +kotlin.mpp.enableCInteropCommonization=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cb3b201b..15dbada8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,7 +1,9 @@ [versions] filekitCore = "0.13.0" -kotlin = "2.3.20" +kotlin = "2.4.20" +# Nucleus JVM Access: Lucene on a JVM on every platform (the search's façade, search-facade) +nja = "0.4.0" agp = "9.1.1" jvmToolchain = "25" lucene = "10.5.2" @@ -12,6 +14,10 @@ kotlinx-coroutines = "1.10.2" kotlinx-serialization = "1.10.0" kotlinx-datetime = "0.7.1" kermit = "2.1.0" +kotlinx-io = "0.9.1" +cachemap = "0.3.0" +atomicfu = "0.33.0" +ksoup = "0.2.6" sqlDelight = "2.3.2" compose = "1.10.2" androidx-activityCompose = "1.13.0" @@ -32,6 +38,10 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa kotlinx-serialization-protobuf = { module = "org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref = "kotlinx-serialization" } kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime" } kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } +kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } +cachemap = { module = "io.github.charlietap:cachemap", version.ref = "cachemap" } +atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "atomicfu" } +ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.ref = "lucene" } lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "lucene" } lucene-analysis-common = { module = "org.apache.lucene:lucene-analysis-common", version.ref = "lucene" } @@ -54,6 +64,9 @@ compose-runtime = { module = "org.jetbrains.compose.runtime:runtime", version.re [plugins] multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } +kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } +nja = { id = "dev.nucleusframework.nja", version.ref = "nja" } +nja-facade = { id = "dev.nucleusframework.nja.facade", version.ref = "nja" } android-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } maven-publish = { id = "com.vanniktech.maven.publish", version.ref = "maven-publish" } kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } diff --git a/search-facade/build.gradle.kts b/search-facade/build.gradle.kts new file mode 100644 index 00000000..458a6c82 --- /dev/null +++ b/search-facade/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { + id("org.jetbrains.kotlin.jvm") + alias(libs.plugins.nja.facade) +} + +group = "io.github.kdroidfilter.seforimlibrary" + +// The search's Lucene side, its façade for Nucleus JVM Access: it runs on a JVM (the desktop's own, the mobile apps' +// embedded one), and search's common code uses it through the proxies NJA generates (see JVM_RUNTIME.md) +kotlin { + jvmToolchain(libs.versions.jvmToolchain.get().toInt()) +} + +njaFacade { + libName = "search" + // The package the common code knows these types by (the façade's own is ...search.lucene: the desktop has both) + clientPackage = "io.github.kdroidfilter.seforimlibrary.search.index" +} + +dependencies { + implementation(libs.lucene.core) + testImplementation(kotlin("test")) +} diff --git a/search-facade/native-image/caller-filter.json b/search-facade/native-image/caller-filter.json new file mode 100644 index 00000000..b0bbcf9a --- /dev/null +++ b/search-facade/native-image/caller-filter.json @@ -0,0 +1,7 @@ +{ + "rules": [ + {"excludeClasses": "**"}, + {"includeClasses": "org.apache.lucene.**"}, + {"includeClasses": "io.github.kdroidfilter.seforimlibrary.search.lucene.**"} + ] +} diff --git a/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt new file mode 100644 index 00000000..7da91c71 --- /dev/null +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt @@ -0,0 +1,124 @@ +package io.github.kdroidfilter.seforimlibrary.search.lucene + +/** + * A Lucene query as plain data: the engines build it in common code, a [LuceneIndex] turns it into Lucene's own + * query, on a JVM (the desktop's own, the mobile apps' embedded one). + */ +sealed interface IndexQuery { + /** Documents whose [field] holds the term [text]. */ + data class Term( + val field: String, + val text: String, + ) : IndexQuery + + /** Documents whose [field] holds a term starting with [text]. */ + data class Prefix( + val field: String, + val text: String, + ) : IndexQuery + + /** Documents whose [field] holds a term within [maxEdits] edits of [text]. */ + data class Fuzzy( + val field: String, + val text: String, + val maxEdits: Int, + ) : IndexQuery + + data class Bool( + val clauses: List, + ) : IndexQuery + + data class Boost( + val query: IndexQuery, + val boost: Float, + ) : IndexQuery + + /** Each position matches any of its terms, within [slop] moves. */ + data class MultiPhrase( + val field: String, + val positions: List>, + val slop: Int, + ) : IndexQuery + + /** + * [text] analyzed like the index (StandardAnalyzer) into a phrase within [slop] moves (a term query for one term), + * every term required when the analysis can't build a phrase, and nothing when it yields no term. + */ + data class AnalyzedPhrase( + val field: String, + val text: String, + val slop: Int, + ) : IndexQuery + + /** Documents whose int point [field] is [value]. */ + data class IntExact( + val field: String, + val value: Int, + ) : IndexQuery + + /** Documents whose int point [field] is one of [values]. */ + data class IntSet( + val field: String, + val values: IntArray, + ) : IndexQuery + + /** Every document. */ + data object MatchAll : IndexQuery +} + +data class Clause( + val query: IndexQuery, + val occur: Occur, +) + +enum class Occur { MUST, SHOULD, FILTER, MUST_NOT } + +/** A document found: its score and the stored fields asked for, numeric ones in [longs], the others in [strings]. */ +data class IndexDoc( + val score: Float, + val longs: Map = emptyMap(), + val strings: Map = emptyMap(), +) + +/** Documents found, with the number of matches ([totalHits] may be a lower bound, as Lucene counts them). */ +data class IndexPage( + val docs: List, + val totalHits: Long, +) + +/** How the documents of a query spread over the books and categories (ancestors included). */ +data class IndexFacets( + val totalHits: Long, + val categoryCounts: Map, + val bookCounts: Map, +) + +enum class SortType { INT, LONG } + +data class SortField( + val field: String, + val type: SortType, + val reverse: Boolean = false, +) + +enum class VectorEncoding { FLOAT32, BYTE } + +/** + * A nearest neighbors search of [field]: with [byteVector] on a byte field, else with [floatVector]; on a float field + * quantized by Lucene, [oversample] times more candidates are gathered, then the best [k] kept by their exact + * similarity. [filter] restricts the documents considered. + */ +data class KnnRequest( + val field: String, + val k: Int, + val fields: Set, + val floatVector: FloatArray? = null, + val byteVector: ByteArray? = null, + val filter: IndexQuery? = null, + val oversample: Int = 1, +) { + init { + // A NaN or infinite component means a broken embedding: its neighbors would be meaningless + require(floatVector?.all { it.isFinite() } != false) { "KNN vector with a NaN or infinite component" } + } +} diff --git a/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt new file mode 100644 index 00000000..d7a487db --- /dev/null +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt @@ -0,0 +1,67 @@ +package io.github.kdroidfilter.seforimlibrary.search.lucene + +import java.nio.file.Path + +/** + * The Lucene index of a search, read only: the only part of the search that runs Lucene, on a JVM (the desktop's own, + * the mobile apps' embedded one, through Nucleus JVM Access). Everything else (query building, dictionary, ranking, + * snippets, highlighting, fusion) is common code. + * + * Thread-safe; an [IndexCursor] is used by one caller at a time. + * + * @param indexDir the index directory + * @param cacheQueries whether to cache any query seen often (small indexes: the lookup index); else only the filters + * every search repeats + */ +class LuceneIndex( + indexDir: String, + cacheQueries: Boolean = false, +) : AutoCloseable { + private val backend = LuceneIndexBackend(Path.of(indexDir), cacheQueries) + + /** The terms of [text] as the index's analyzer (StandardAnalyzer, the `text` field) splits it. */ + fun analyze(text: String): List = backend.analyze(text) + + /** The best [limit] documents for [query] (by score, or by [sort]), with their stored [fields]. */ + fun search( + query: IndexQuery, + limit: Int, + fields: Set, + sort: List? = null, + ): IndexPage = backend.search(query, limit, fields, sort) + + /** Pages through the documents of [query] by score, on the index as it is now, with their stored [fields]. */ + fun openCursor( + query: IndexQuery, + fields: Set, + ): IndexCursor = IndexCursor(backend.openCursor(query, fields)) + + /** How the documents of [query] spread over the books and categories (`book_id`, `ancestor_category_ids`). */ + fun facets(query: IndexQuery): IndexFacets = backend.facets(query) + + /** The stored numeric [field] of every document of [query], unscored and in no particular order. */ + fun collectLongs( + query: IndexQuery, + field: String, + ): LongArray = backend.collectLongs(query, field) + + fun count(query: IndexQuery): Int = backend.count(query) + + /** How the vectors of [field] are stored, null when no segment has the field. */ + fun vectorEncoding(field: String): VectorEncoding? = backend.vectorEncoding(field) + + /** The nearest documents to a vector (see [KnnRequest]), best first, with their stored [KnnRequest.fields]. */ + fun knn(request: KnnRequest): List = backend.knn(request) + + override fun close() = backend.close() +} + +/** A page of a [LuceneIndex.openCursor]. */ +class IndexCursor internal constructor( + private val cursor: BackendCursor, +) : AutoCloseable { + /** The next [limit] documents, none once all were read. */ + fun next(limit: Int): IndexPage = cursor.next(limit) + + override fun close() = cursor.close() +} diff --git a/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt new file mode 100644 index 00000000..f5d5ed92 --- /dev/null +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt @@ -0,0 +1,584 @@ +package io.github.kdroidfilter.seforimlibrary.search.lucene + +import org.apache.lucene.analysis.Analyzer +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute +import org.apache.lucene.document.IntPoint +import org.apache.lucene.index.DocValuesType +import org.apache.lucene.index.FieldInfo +import org.apache.lucene.index.IndexReader +import org.apache.lucene.index.LeafReaderContext +import org.apache.lucene.index.NumericDocValues +import org.apache.lucene.index.ReaderUtil +import org.apache.lucene.index.StoredFieldVisitor +import org.apache.lucene.index.StoredFields +import org.apache.lucene.index.Term +import org.apache.lucene.index.VectorSimilarityFunction +import org.apache.lucene.search.BooleanClause +import org.apache.lucene.search.BooleanQuery +import org.apache.lucene.search.BoostQuery +import org.apache.lucene.search.Collector +import org.apache.lucene.search.CollectorManager +import org.apache.lucene.search.FuzzyQuery +import org.apache.lucene.search.IndexSearcher +import org.apache.lucene.search.KnnByteVectorQuery +import org.apache.lucene.search.KnnFloatVectorQuery +import org.apache.lucene.search.LRUQueryCache +import org.apache.lucene.search.LeafCollector +import org.apache.lucene.search.MatchAllDocsQuery +import org.apache.lucene.search.MultiPhraseQuery +import org.apache.lucene.search.PointRangeQuery +import org.apache.lucene.search.PrefixQuery +import org.apache.lucene.search.Query +import org.apache.lucene.search.QueryCachingPolicy +import org.apache.lucene.search.ReferenceManager +import org.apache.lucene.search.Scorable +import org.apache.lucene.search.ScoreDoc +import org.apache.lucene.search.ScoreMode +import org.apache.lucene.search.SearcherFactory +import org.apache.lucene.search.SearcherManager +import org.apache.lucene.search.Sort +import org.apache.lucene.search.TermQuery +import org.apache.lucene.search.TopDocs +import org.apache.lucene.search.UsageTrackingQueryCachingPolicy +import org.apache.lucene.store.Directory +import org.apache.lucene.store.FSDirectory +import org.apache.lucene.store.NIOFSDirectory +import org.apache.lucene.util.QueryBuilder +import java.nio.file.Path +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import org.apache.lucene.index.VectorEncoding as LuceneVectorEncoding +import org.apache.lucene.search.SortField as LuceneSortField + +private const val FIELD_BOOK_ID = "book_id" +private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" + +// Smallest partition worth a task of its own when a segment is split across cores +private const val MIN_DOCS_PER_SLICE = 100_000 + +/** + * What [LuceneIndex] runs: a Lucene directory, read with Lucene in this JVM (the desktop's, the mobile apps' embedded + * one). + * + * @param indexDir the index directory + * @param cacheQueries whether to cache any query seen often (small indexes: the lookup index); else only the filters + * every search repeats + * @param analyzer the analyzer the index was written with + */ +internal class LuceneIndexBackend( + private val indexDir: Path, + private val cacheQueries: Boolean = false, + private val analyzer: Analyzer = StandardAnalyzer(), +) : AutoCloseable { + // Open lazily to avoid any I/O at app startup + private val dir: Directory by lazy { openDirectory(indexDir) } + + // One reader shared by every search: opening one per search re-reads the term index of each segment and maps, + // then unmaps, all the files. Refreshed when the index changes on disk (delta updates write into it). + private val searcherManagerLazy = + lazy { + SearcherManager(dir, searcherFactory()).apply { + addListener( + object : ReferenceManager.RefreshListener { + override fun beforeRefresh() = Unit + + override fun afterRefresh(didRefresh: Boolean) { + if (didRefresh) bookAncestors.clear() + } + }, + ) + } + } + private val searcherManager: SearcherManager by searcherManagerLazy + + // Ancestor category ids per book, for the facets: every line of a book shares them + private val bookAncestors = ConcurrentHashMap() + + private fun searcherFactory(): SearcherFactory = + if (!queryCacheEnabled) { + // Lucene's query cache changes how the filters run, and with it the last digits of the scores: off, the + // same search ranks the same every time (the parity tests across platforms) + object : SearcherFactory() { + override fun newSearcher( + reader: IndexReader, + previousReader: IndexReader?, + ): IndexSearcher = PartitionedSearcherFactory.newSearcher(reader, previousReader).apply { queryCache = null } + } + } else if (!cacheQueries) { + PartitionedSearcherFactory + } else { + object : SearcherFactory() { + override fun newSearcher( + reader: IndexReader, + previousReader: IndexReader?, + ): IndexSearcher = + IndexSearcher(reader).apply { + queryCache = LRUQueryCache(1024, 64L * 1024 * 1024) // up to 64MiB cache + queryCachingPolicy = UsageTrackingQueryCachingPolicy() + } + } + } + + private fun acquire(): IndexSearcher { + val manager = searcherManager + manager.maybeRefresh() + return manager.acquire() + } + + private inline fun withSearcher(block: (IndexSearcher) -> T): T { + val searcher = acquire() + try { + return block(searcher) + } finally { + searcherManager.release(searcher) + } + } + + fun analyze(text: String): List = + try { + val out = mutableListOf() + analyzer.tokenStream("text", text).use { ts -> + val termAtt = ts.addAttribute(CharTermAttribute::class.java) + ts.reset() + while (ts.incrementToken()) { + val t = termAtt.toString() + if (t.isNotBlank()) out += t + } + ts.end() + } + out + } catch (_: Exception) { + emptyList() + } + + fun search( + query: IndexQuery, + limit: Int, + fields: Set, + sort: List?, + ): IndexPage = + withSearcher { searcher -> + val luceneQuery = toLucene(query) + val top = + if (sort.isNullOrEmpty()) { + searcher.search(luceneQuery, limit) + } else { + searcher.search(luceneQuery, limit, Sort(*sort.map { it.toLucene() }.toTypedArray())) + } + top.toPage(searcher.storedFields(), fields) + } + + fun openCursor( + query: IndexQuery, + fields: Set, + ): BackendCursor = Cursor(toLucene(query), fields, acquire()) + + private inner class Cursor( + private val query: Query, + private val fields: Set, + private val searcher: IndexSearcher, + ) : BackendCursor { + private var after: ScoreDoc? = null + private var totalHits: Long? = null + private var closed = false + + override fun next(limit: Int): IndexPage { + val top = searcher.searchAfter(after, query, limit) + if (totalHits == null) totalHits = top.totalHits.value() + top.scoreDocs.lastOrNull()?.let { after = it } + return IndexPage(readDocs(searcher.storedFields(), top.scoreDocs.asList(), fields), totalHits ?: 0) + } + + override fun close() { + if (closed) return + closed = true + searcherManager.release(searcher) + } + } + + fun facets(query: IndexQuery): IndexFacets = + withSearcher { searcher -> + // Only book ids are read per hit (doc values when the index has them, else the first stored field); + // category counts follow from the book counts, as every line of a book shares its ancestors. One + // collector per slice, as the slices may run concurrently + class FacetCollector : Collector { + val bookCounts = HashMap() + var totalHits = 0L + + override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { + val reader = leafContext.reader() + val storedFields = reader.storedFields() + val bookIdValues: NumericDocValues? = + reader.fieldInfos + .fieldInfo(FIELD_BOOK_ID) + ?.takeIf { it.docValuesType == DocValuesType.NUMERIC } + ?.let { reader.getNumericDocValues(FIELD_BOOK_ID) } + // Lines of a book are mostly contiguous: count runs, then fold them into the map + var runBook = -1L + var runCount = 0 + + fun flushRun() { + if (runCount > 0) bookCounts.merge(runBook, runCount, Int::plus) + runCount = 0 + } + + return object : LeafCollector { + override fun setScorer(scorer: Scorable) = Unit + + override fun collect(doc: Int) { + totalHits++ + val bookId = + if (bookIdValues != null && bookIdValues.advanceExact(doc)) { + bookIdValues.longValue() + } else { + readStoredBookId(storedFields, doc) ?: return + } + if (bookId != runBook) { + flushRun() + runBook = bookId + if (!bookAncestors.containsKey(bookId)) loadAncestors(storedFields, doc, bookId) + } + runCount++ + } + + override fun finish() = flushRun() + } + } + + override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES + } + + searcher.search( + toLucene(query), + object : CollectorManager { + override fun newCollector() = FacetCollector() + + override fun reduce(collectors: Collection): IndexFacets { + val bookCounts = HashMap() + collectors.forEach { c -> c.bookCounts.forEach { (book, n) -> bookCounts.merge(book, n, Int::plus) } } + val categoryCounts = HashMap() + for ((bookId, count) in bookCounts) { + bookAncestors[bookId]?.forEach { categoryCounts.merge(it, count, Int::plus) } + } + return IndexFacets( + totalHits = collectors.sumOf { it.totalHits }, + categoryCounts = categoryCounts, + bookCounts = bookCounts, + ) + } + }, + ) + } + + fun collectLongs( + query: IndexQuery, + field: String, + ): LongArray = + withSearcher { searcher -> + // One collector per slice, as the slices may run concurrently + class LongCollector : Collector { + var values = LongArray(256) + var count = 0 + + override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { + val storedFields = leafContext.reader().storedFields() + val wanted = setOf(field) + return object : LeafCollector { + override fun setScorer(scorer: Scorable) = Unit + + override fun collect(doc: Int) { + val value = + storedFields + .document(doc, wanted) + .getField(field) + ?.numericValue() + ?.toLong() ?: return + if (count == values.size) values = values.copyOf(count * 2) + values[count++] = value + } + } + } + + override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES + } + searcher.search( + toLucene(query), + object : CollectorManager { + override fun newCollector() = LongCollector() + + override fun reduce(collectors: Collection): LongArray { + val out = LongArray(collectors.sumOf { it.count }) + var at = 0 + for (c in collectors) { + c.values.copyInto(out, at, 0, c.count) + at += c.count + } + return out + } + }, + ) + } + + fun count(query: IndexQuery): Int = withSearcher { it.count(toLucene(query)) } + + fun vectorEncoding(field: String): VectorEncoding? = + withSearcher { searcher -> + searcher.indexReader + .leaves() + .firstNotNullOfOrNull { it.reader().fieldInfos.fieldInfo(field) } + ?.let { if (it.vectorEncoding == LuceneVectorEncoding.BYTE) VectorEncoding.BYTE else VectorEncoding.FLOAT32 } + } + + fun knn(request: KnnRequest): List = + withSearcher { searcher -> + val filter = request.filter?.let { toLucene(it) } + val byteVector = request.byteVector + val floatVector = request.floatVector + val top: List = + when { + byteVector != null -> + searcher.search(KnnByteVectorQuery(request.field, byteVector, request.k, filter), request.k).scoreDocs.asList() + floatVector != null && request.oversample > 1 -> + rescoredKnn(searcher, request.field, floatVector, request.k, request.oversample, filter) + floatVector != null -> + searcher.search(KnnFloatVectorQuery(request.field, floatVector, request.k, filter), request.k).scoreDocs.asList() + else -> error("KnnRequest without a vector") + } + readDocs(searcher.storedFields(), top, request.fields) + } + + /** + * KNN over a field quantized by Lucene (float vectors stored next to their int8 copy): the graph search ranks on the + * approximate int8 similarity, so it gathers [oversample] times more candidates and keeps the [k] best by their + * exact float similarity. Empty when the field holds no eligible document. + */ + private fun rescoredKnn( + searcher: IndexSearcher, + field: String, + query: FloatArray, + k: Int, + oversample: Int, + filter: Query?, + ): List { + val candidates = searcher.search(KnnFloatVectorQuery(field, query, k * oversample, filter), k * oversample).scoreDocs + if (candidates.isEmpty()) return emptyList() + val leaves = searcher.indexReader.leaves() + val rescored = ArrayList(candidates.size) + // By segment, in doc order: the vector iterators only move forward + for ((leafIndex, docs) in candidates.groupBy { ReaderUtil.subIndex(it.doc, leaves) }) { + val leaf = leaves[leafIndex] + val values = leaf.reader().getFloatVectorValues(field) ?: continue + val iterator = values.iterator() + for (doc in docs.map { it.doc }.sorted()) { + if (iterator.advance(doc - leaf.docBase) != doc - leaf.docBase) continue + rescored += ScoreDoc(doc, VectorSimilarityFunction.COSINE.compare(query, values.vectorValue(iterator.index()))) + } + } + return rescored.sortedByDescending { it.score }.take(k) + } + + override fun close() { + if (searcherManagerLazy.isInitialized()) runCatching { searcherManager.close() } + } + + // --- Lucene mapping --- + + private fun toLucene(query: IndexQuery): Query = + when (query) { + is IndexQuery.Term -> TermQuery(Term(query.field, query.text)) + is IndexQuery.Prefix -> PrefixQuery(Term(query.field, query.text)) + is IndexQuery.Fuzzy -> FuzzyQuery(Term(query.field, query.text), query.maxEdits) + is IndexQuery.Bool -> + BooleanQuery + .Builder() + .apply { query.clauses.forEach { add(toLucene(it.query), it.occur.toLucene()) } } + .build() + is IndexQuery.Boost -> BoostQuery(toLucene(query.query), query.boost) + is IndexQuery.MultiPhrase -> + MultiPhraseQuery + .Builder() + .apply { + setSlop(query.slop) + query.positions.forEachIndexed { position, alternatives -> + add(alternatives.map { Term(query.field, it) }.toTypedArray(), position) + } + }.build() + is IndexQuery.AnalyzedPhrase -> { + val builder = QueryBuilder(analyzer) + builder.createPhraseQuery(query.field, query.text, query.slop) + ?: builder.createBooleanQuery(query.field, query.text, BooleanClause.Occur.MUST) + ?: BooleanQuery.Builder().build() + } + is IndexQuery.IntExact -> IntPoint.newExactQuery(query.field, query.value) + is IndexQuery.IntSet -> IntPoint.newSetQuery(query.field, *query.values) + IndexQuery.MatchAll -> MatchAllDocsQuery() + } + + private fun Occur.toLucene(): BooleanClause.Occur = + when (this) { + Occur.MUST -> BooleanClause.Occur.MUST + Occur.SHOULD -> BooleanClause.Occur.SHOULD + Occur.FILTER -> BooleanClause.Occur.FILTER + Occur.MUST_NOT -> BooleanClause.Occur.MUST_NOT + } + + private fun SortField.toLucene(): LuceneSortField = + LuceneSortField( + field, + when (type) { + SortType.INT -> LuceneSortField.Type.INT + SortType.LONG -> LuceneSortField.Type.LONG + }, + reverse, + ) + + private fun TopDocs.toPage( + stored: StoredFields, + fields: Set, + ): IndexPage = IndexPage(readDocs(stored, scoreDocs.asList(), fields), totalHits.value()) + + private fun readDocs( + stored: StoredFields, + scoreDocs: List, + fields: Set, + ): List = + scoreDocs.map { sd -> + val longs = HashMap() + val strings = HashMap() + if (fields.isNotEmpty()) { + for (field in stored.document(sd.doc, fields)) { + val number = field.numericValue() + if (number != null) { + longs[field.name()] = number.toLong() + } else { + field.stringValue()?.let { strings[field.name()] = it } + } + } + } + // A sorted search has no score (NaN): 0 instead, the same on every platform + IndexDoc(if (sd.score.isNaN()) 0f else sd.score, longs, strings) + } + + private fun readStoredBookId( + storedFields: StoredFields, + doc: Int, + ): Long? { + var bookId: Long? = null + storedFields.document( + doc, + object : StoredFieldVisitor() { + override fun needsField(fieldInfo: FieldInfo): Status = + when { + bookId != null -> Status.STOP + fieldInfo.name == FIELD_BOOK_ID -> Status.YES + else -> Status.NO + } + + override fun longField( + fieldInfo: FieldInfo, + value: Long, + ) { + bookId = value + } + + override fun intField( + fieldInfo: FieldInfo, + value: Int, + ) { + bookId = value.toLong() + } + }, + ) + return bookId + } + + private fun loadAncestors( + storedFields: StoredFields, + doc: Int, + bookId: Long, + ) { + val ancestors = + storedFields + .document(doc, setOf(FIELD_ANCESTOR_CATEGORY_IDS)) + .getField(FIELD_ANCESTOR_CATEGORY_IDS) + ?.stringValue() + ?.split(",") + ?.mapNotNull { it.trim().toLongOrNull() } + .orEmpty() + // Lines added by delta updates lack the field: try again on the book's next run + if (ancestors.isNotEmpty()) bookAncestors[bookId] = ancestors.toLongArray() + } + + companion object { + /** + * The directory of [path]: memory-mapped on the desktop's JVM, read through NIO where Lucene's Panama-based + * mapping isn't available (GraalVM native image, the iOS Java runtime) or not wanted: on Android, the mapped + * index (the vectors above all) would count in the app's memory, which Android looks at when it kills apps. + */ + fun openDirectory(path: Path): Directory = if (mmapAvailable) FSDirectory.open(path) else NIOFSDirectory(path) + + private val queryCacheEnabled: Boolean = System.getProperty("seforim.lucene.queryCache") != "off" + + private val mmapAvailable: Boolean by lazy { + System.getProperty("org.graalvm.nativeimage.imagecode") == null && + // Android: HotSpot runs in the app's process, with its environment + System.getenv("ANDROID_ROOT") == null && + System.getProperty("os.name") != "Darwin" && + System.getProperty("java.vm.name")?.contains("Zero") != true + } + } +} + +/** + * Searchers that split the index, even a single segment, into one partition per core and search them concurrently: + * a merged index is one big segment, which a plain searcher scores on one thread. + */ +private object PartitionedSearcherFactory : SearcherFactory() { + private val cores = Runtime.getRuntime().availableProcessors() + private val executor: ExecutorService by lazy { + val ids = AtomicInteger() + Executors.newFixedThreadPool(cores) { task -> + Thread(task, "lucene-search-${ids.incrementAndGet()}").apply { isDaemon = true } + } + } + + override fun newSearcher( + reader: IndexReader, + previousReader: IndexReader?, + ): IndexSearcher { + val searcher = + if (cores == 1) { + IndexSearcher(reader) + } else { + val docsPerSlice = maxOf(MIN_DOCS_PER_SLICE, reader.maxDoc() / cores + 1) + object : IndexSearcher(reader, executor) { + override fun slices(leaves: List): Array = slices(leaves, docsPerSlice, Int.MAX_VALUE, true) + } + } + return searcher.apply { queryCachingPolicy = StableFiltersCachingPolicy } + } +} + +/** + * Caches only the filters every search repeats (line documents, base books). Lucene's default policy also caches any + * clause seen twice, and each search builds its word filters twice (facets, then results): it would then turn them + * into bitsets over the whole index for a query that won't come back. + */ +private object StableFiltersCachingPolicy : QueryCachingPolicy { + override fun onUse(query: Query) = Unit + + override fun shouldCache(query: Query): Boolean = + when (query) { + is TermQuery -> query.term.field() == "type" + is PointRangeQuery -> query.field == "is_base_book" + else -> false + } +} + +/** A cursor of the backend, behind an [IndexCursor]. */ +internal interface BackendCursor : AutoCloseable { + fun next(limit: Int): IndexPage +} diff --git a/search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json b/search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json new file mode 100644 index 00000000..d0b8b665 --- /dev/null +++ b/search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json @@ -0,0 +1,327 @@ +{ + "reflection": [ + { + "type": "byte[][]" + }, + { + "type": "com.sun.management.HotSpotDiagnosticMXBean", + "methods": [ + { + "name": "getVMOption", + "parameterTypes": [ + "java.lang.String" + ] + } + ] + }, + { + "type": "com.sun.management.VMOption", + "methods": [ + { + "name": "getValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Enum", + "serializable": true + }, + { + "type": "java.lang.Integer" + }, + { + "type": "java.lang.Long" + }, + { + "type": "java.lang.Number" + }, + { + "type": "java.lang.Object" + }, + { + "type": "java.lang.Object[]", + "serializable": true + }, + { + "type": "java.lang.String" + }, + { + "type": "java.lang.management.ManagementFactory", + "methods": [ + { + "name": "getPlatformMXBean", + "parameterTypes": [ + "java.lang.Class" + ] + } + ] + }, + { + "type": "java.util.HashSet", + "serializable": true + }, + { + "type": "java.util.LinkedHashSet", + "serializable": true + }, + { + "type": "java.util.concurrent.ArrayBlockingQueue", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.AbstractOwnableSynchronizer", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.AbstractQueuedSynchronizer", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.ReentrantLock", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.ReentrantLock$NonfairSync", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.ReentrantLock$Sync", + "serializable": true + }, + { + "type": "org.apache.lucene.analysis.LowerCaseFilter" + }, + { + "type": "org.apache.lucene.analysis.StopFilter" + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.codecs.KnnVectorsFormat" + }, + { + "type": "org.apache.lucene.codecs.hnsw.FlatVectorsFormat" + }, + { + "type": "org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat" + }, + { + "type": "org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat" + }, + { + "type": "org.apache.lucene.index.DocsWithFieldSet" + }, + { + "type": "org.apache.lucene.index.Term" + }, + { + "type": "org.apache.lucene.index.TermStates$PendingTermLookup[]" + }, + { + "type": "org.apache.lucene.index.TermStates[]" + }, + { + "type": "org.apache.lucene.index.Term[]" + }, + { + "type": "org.apache.lucene.internal.hppc.FloatArrayList" + }, + { + "type": "org.apache.lucene.internal.hppc.IntArrayList" + }, + { + "type": "org.apache.lucene.internal.hppc.IntHashSet" + }, + { + "type": "org.apache.lucene.internal.hppc.IntIntHashMap" + }, + { + "type": "org.apache.lucene.internal.hppc.IntObjectHashMap" + }, + { + "type": "org.apache.lucene.internal.hppc.LongArrayList" + }, + { + "type": "org.apache.lucene.internal.hppc.LongHashSet" + }, + { + "type": "org.apache.lucene.internal.hppc.LongObjectHashMap" + }, + { + "type": "org.apache.lucene.search.AutomatonQuery" + }, + { + "type": "org.apache.lucene.search.BoostAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.search.DisiWrapper[]" + }, + { + "type": "org.apache.lucene.search.DocIdSet" + }, + { + "type": "org.apache.lucene.search.LRUQueryCache$CacheAndCount" + }, + { + "type": "org.apache.lucene.search.MaxNonCompetitiveBoostAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.search.MultiTermQuery" + }, + { + "type": "org.apache.lucene.search.PointInSetQuery" + }, + { + "type": "org.apache.lucene.search.Query" + }, + { + "type": "org.apache.lucene.search.TermStatistics[]" + }, + { + "type": "org.apache.lucene.util.BitDocIdSet" + }, + { + "type": "org.apache.lucene.util.BitSet" + }, + { + "type": "org.apache.lucene.util.ByteBlockPool" + }, + { + "type": "org.apache.lucene.util.BytesRef" + }, + { + "type": "org.apache.lucene.util.BytesRefBlockPool" + }, + { + "type": "org.apache.lucene.util.BytesRefHash" + }, + { + "type": "org.apache.lucene.util.FixedBitSet" + }, + { + "type": "org.apache.lucene.util.FrequencyTrackingRingBuffer" + }, + { + "type": "org.apache.lucene.util.FrequencyTrackingRingBuffer$IntBag" + }, + { + "type": "org.apache.lucene.util.IntArrayDocIdSet" + }, + { + "type": "org.apache.lucene.util.LongValues" + }, + { + "type": "org.apache.lucene.util.PagedBytes" + }, + { + "type": "org.apache.lucene.util.PagedBytes$Reader" + }, + { + "type": "org.apache.lucene.util.RoaringDocIdSet" + }, + { + "type": "org.apache.lucene.util.RoaringDocIdSet$RangeDocIdSet" + }, + { + "type": "org.apache.lucene.util.automaton.CompiledAutomaton" + }, + { + "type": "org.apache.lucene.util.automaton.RunAutomaton" + }, + { + "type": "org.apache.lucene.util.automaton.Transition" + }, + { + "type": "org.apache.lucene.util.packed.DeltaPackedLongValues" + }, + { + "type": "org.apache.lucene.util.packed.DeltaPackedLongValues$Builder" + }, + { + "type": "org.apache.lucene.util.packed.MonotonicLongValues" + }, + { + "type": "org.apache.lucene.util.packed.MonotonicLongValues$Builder" + }, + { + "type": "org.apache.lucene.util.packed.PackedInts$Reader[]" + }, + { + "type": "org.apache.lucene.util.packed.PackedLongValues" + }, + { + "type": "org.apache.lucene.util.packed.PackedLongValues$Builder" + }, + { + "type": "org.gradle.api.internal.tasks.testing.filter.TestFilterSpec", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.junit.JUnitSpec", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.junit.JUnitTestDefinitionProcessorFactory", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.worker.TestWorker", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.worker.TestWorker$State", + "serializable": true + }, + { + "type": "org.junit.runner.Result$SerializedForm", + "serializable": true + } + ], + "resources": [ + { + "glob": "META-INF/services/dev.nucleusframework.nja.server.NjaModule" + }, + { + "glob": "META-INF/services/java.nio.channels.spi.SelectorProvider" + }, + { + "glob": "META-INF/services/java.time.zone.ZoneRulesProvider" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.Codec" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.DocValuesFormat" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.PostingsFormat" + } + ] +} diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 52c283fb..83756b5c 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -1,6 +1,11 @@ +import dev.nucleusframework.nja.plugin.NjaRuntimeType +import org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeSimulatorTest + plugins { alias(libs.plugins.multiplatform) + alias(libs.plugins.android.library) alias(libs.plugins.kotlinx.serialization) + alias(libs.plugins.nja) } group = "io.github.kdroidfilter.seforimlibrary" @@ -9,22 +14,104 @@ kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) jvm() + android { + namespace = "io.github.kdroidfilter.seforimlibrary.search" + compileSdk = 35 + // NJA's bridge runs from Android 10 + minSdk = 29 + } + listOf(iosArm64(), iosSimulatorArm64()).forEach { target -> + target.compilations.getByName("main").cinterops { + create("sqlite3") { + definitionFile.set(project.file("src/nativeInterop/cinterop/sqlite3.def")) + } + } + } sourceSets { - jvmMain.dependencies { + commonMain.dependencies { api(project(":core")) implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.json) - implementation(libs.lucene.core) - implementation(libs.lucene.analysis.common) - implementation(libs.sqlDelight.driver.sqlite) + // SearchFacets, which the apps keep + api(libs.kotlinx.serialization.json) + implementation(libs.kotlinx.io.core) + implementation(libs.atomicfu) implementation(libs.kermit) - implementation(libs.jsoup) } + // The JVM and Android share the HTML and coroutine helpers (Lucene runs behind the façade, on a JVM) + val androidJvmMain by creating { + dependsOn(commonMain.get()) + dependencies { + implementation(libs.jsoup) + } + } + jvmMain { + dependsOn(androidJvmMain) + dependencies { + implementation(libs.sqlite.jdbc) + } + } + androidMain { dependsOn(androidJvmMain) } + + // The manual source set above turns off the default hierarchy: iOS shares its own one by hand + val iosMain by creating { + dependsOn(commonMain.get()) + dependencies { + implementation(libs.ksoup) + } + } + iosArm64Main { dependsOn(iosMain) } + iosSimulatorArm64Main { dependsOn(iosMain) } + jvmTest.dependencies { implementation(kotlin("test")) implementation(libs.kotlinx.coroutines.test) + // The test indexes are written with Lucene directly + implementation(libs.lucene.core) + implementation(libs.lucene.analysis.common) } } } + +// Lucene, through Nucleus JVM Access: the façade (search-facade) runs on the desktop's own JVM, compiled into a GraalVM +// native image on iOS (Zero, the JVM's interpreter, searches 8× slower), on HotSpot in the app's process on Android (an +// Android native image builds on Linux only), and common code uses its proxies +nucleusJvm { + facade(project(":search-facade")) + // NJA downloads the runtimes: graalvm-mobile's latest release for iOS, jdk-mobile's for Android (see JVM_RUNTIME.md) + iosRuntime.set(NjaRuntimeType.GRAALVM) + androidAbis.set(setOf("arm64-v8a", "x86_64")) + // Lucene maps its files with Panama (madvise): native access, without the warning (a native image keeps -Xmx only) + jvmOptions.addAll("-Xmx128m", "--enable-native-access=ALL-UNNAMED") + // Lucene's query cache off: the same search ranks the same every time (the parity tests across platforms) + providers.gradleProperty("luceneQueryCache").orNull?.let { jvmOptions.add("-Dseforim.lucene.queryCache=$it") } +} + +tasks.withType().configureEach { + device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) + val indexDir = providers.gradleProperty("indexDir") + val parityOut = layout.buildDirectory.file("parity/ios.txt") + doFirst { + // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix + indexDir.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_INDEX_DIR", it) } + environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", parityOut.get().asFile.path) + } +} + +tasks.named("jvmTest") { + // The parity dump on a real index: -PindexDir= + providers.gradleProperty("indexDir").orNull?.let { systemProperty("seforimIndexDir", it) } + systemProperty("parityOut", layout.buildDirectory.file("parity/jvm.txt").get().asFile.path) + providers.gradleProperty("parityCores").orNull?.let { jvmArgs("-XX:ActiveProcessorCount=$it") } + providers.gradleProperty("luceneQueryCache").orNull?.let { systemProperty("seforim.lucene.queryCache", it) } + providers.gradleProperty("probeOut").orNull?.let { systemProperty("probeOut", it) } + // The façade's native-image metadata, traced on these tests (with the real index): -PnativeImageAgent= + providers.gradleProperty("nativeImageAgent").orNull?.let { graalvm -> + val out = rootProject.layout.projectDirectory.dir("search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade") + val filter = rootProject.layout.projectDirectory.file("search-facade/native-image/caller-filter.json") + executable("$graalvm/bin/java") + jvmArgs("-agentlib:native-image-agent=config-merge-dir=${out.asFile.path},caller-filter-file=${filter.asFile.path}") + } + testLogging.showStandardStreams = true +} diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt new file mode 100644 index 00000000..5d6c3649 --- /dev/null +++ b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt @@ -0,0 +1,10 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.runBlocking +import org.jsoup.Jsoup +import org.jsoup.safety.Safelist + +internal actual fun escapedText(html: String): String = Jsoup.clean(html, Safelist.none()) + +internal actual fun runBlockingIo(block: suspend CoroutineScope.() -> T): T = runBlocking(block = block) diff --git a/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt b/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt new file mode 100644 index 00000000..891591d2 --- /dev/null +++ b/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt @@ -0,0 +1,27 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import android.database.sqlite.SQLiteDatabase + +/** Through Android's SQLite: one connection, which serializes the queries of every thread. */ +actual fun openReadOnlySqlite(path: String): SqliteReader = AndroidSqliteReader(path) + +private class AndroidSqliteReader( + path: String, +) : SqliteReader { + private val db = SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY or SQLiteDatabase.NO_LOCALIZED_COLLATORS) + + override fun query( + sql: String, + args: List, + ): List> = + db.rawQuery(sql, args.toTypedArray()).use { cursor -> + val columns = cursor.columnNames + val rows = ArrayList>(cursor.count) + while (cursor.moveToNext()) rows += columns.indices.associate { columns[it] to cursor.getString(it) } + rows + } + + override fun close() { + if (db.isOpen) db.close() + } +} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt new file mode 100644 index 00000000..1a9240fc --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt @@ -0,0 +1,56 @@ +package io.github.kdroidfilter.seforimlibrary.search + +/** + * Blacklist of hallucinated dictionary mappings: LLM-generated mappings that are incorrect and should be ignored. + * Key: normalized token, Value: set of incorrect base forms to reject. + * + * TODO: This is a temporary workaround. The proper fix is to correct the + * hallucinated mappings directly in the lexical dictionary (lexical.db) + * so this blacklist becomes unnecessary. + */ +internal object HallucinationBlacklist { + // token to base + private val ENTRIES = + listOf( + "לחתוכ" to "כנ", + "לחתוך" to "כנ", + "לבקש" to "רצה", + "מתפלל" to "בעה", + "לשבת" to "נתנ", + "לעבוד" to "עשה", + "להעמיד" to "קומ", + "כנ" to "כנ", + "בנ" to "בנ", + "לשימ" to "שמ", + "לשים" to "שמ", + "לדונ" to "יד", + "לדון" to "יד", + "אמ" to "אמ", + "להרגיע" to "יושב", + "לחייכ" to "חוה", + "לחייך" to "חוה", + "שנ" to "שנ", + "עצ" to "עצ", + "דנ" to "דנ", + "להיכנס" to "עלה", + "לשקול" to "לקח", + "לבנות" to "בנ", + "לשחרר" to "שרה", + "להתיר" to "שרה", + ) + + private val bases: Map> by lazy { + ENTRIES + .groupBy({ HebrewTextUtils.normalizeHebrew(it.first) }, { HebrewTextUtils.normalizeHebrew(it.second) }) + .mapValues { it.value.toSet() } + } + + /** Whether [expansion] reaches [token] only through a blacklisted base. */ + fun rejects( + token: String, + expansion: MagicDictionaryIndex.Expansion, + ): Boolean { + val blacklisted = bases[HebrewTextUtils.normalizeHebrew(token)] ?: return false + return expansion.base.any { HebrewTextUtils.normalizeHebrew(it) in blacklisted } + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt new file mode 100644 index 00000000..d914392f --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt @@ -0,0 +1,733 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforimlibrary.search.index.BoolBuilder +import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor +import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.Occur +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.withContext +import kotlin.concurrent.Volatile + +private val logger = Logger.withTag("LexicalSearchEngine") + +// Hard cap on how many synonym/expansion terms we allow per token +private const val MAX_SYNONYM_TERMS_PER_TOKEN: Int = 32 + +// Global cap for boost queries built from dictionary expansions +private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 +private const val DEFAULT_NEAR = 5 +private const val PLAN_CACHE_SIZE = 16 + +// Searches run by warmUp(): one and several words, common and rarer ones, a quoted phrase, the divine name (whose +// dictionary entry is the largest) +val WARMUP_QUERIES = listOf("ברוך ה׳ לעולם", "אמר רבי יוחנן", "תפילין", "\"קריאת שמע\"") +const val WARMUP_ROUNDS = 2 +const val WARMUP_PAGE_SIZE = 25 + +// Single Hebrew letters, dropped from a query before its dictionary expansion +private val SINGLE_LETTERS = + setOf("א", "ב", "ג", "ד", "ה", "ו", "ז", "ח", "ט", "י", "כ", "ל", "מ", "נ", "ס", "ע", "פ", "צ", "ק", "ר", "ש", "ת") + +private val WHITESPACE = Regex("\\s+") + +// The stored fields a hit is read from +private val HIT_FIELDS = setOf("book_id", "book_title", "line_id", "line_index", "is_base_book", "order_index", "text_raw") + +/** + * The full-text search of Hebrew religious texts, with diacritics handling, dictionary expansion and fuzzy matching, + * over a Lucene [index] (see [LuceneIndex]: in process on the JVM and Android, through the embedded Java runtime on + * iOS). Hits are ranked by Lucene, then boosted for base books; their snippets come from [snippetProvider] (the + * indexed text when there is none). Inflections come from the lexical [dictionary] when it is available. + */ +class LexicalSearchEngine( + private val index: LuceneIndex, + private val snippetProvider: SnippetProvider? = null, + private val dictionary: Lazy = lazyOf(null), +) : SearchEngine { + @Volatile + private var closed = false + + // A dictionary that finished loading after close() is closed by its first reader: close() couldn't see it + private val magicDict: MagicDictionaryIndex? + get() { + val dict = dictionary.value + if (closed) { + dict?.close() + return null + } + return dict + } + + // --- SearchEngine interface implementation --- + + override fun openSession( + query: String, + near: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection?, + lineIds: Collection?, + baseBookOnly: Boolean, + ): SearchSession? { + val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) ?: return null + return LexicalSearchSession(context, index.openCursor(context.query, HIT_FIELDS)) + } + + override suspend fun attachSnippets( + hits: List, + query: String, + near: Int, + ): List { + if (hits.isEmpty()) return hits + val context = buildSearchContext(query, near, null, null, null, null) ?: return hits + val sources = withContext(Dispatchers.IO) { snippetSources(hits.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) } + return hits.map { hit -> + val source = sources[hit.lineId] ?: SnippetSource(hit.rawText, 0, hit.rawText.length) + hit.copy( + snippet = SnippetBuilder.build(source.text, context.highlightPlan, source.lineStart, source.lineEnd), + rawText = source.text, + lineStart = source.lineStart, + lineEnd = source.lineEnd, + ) + } + } + + override suspend fun warmUp() { + withContext(Dispatchers.IO) { hashemHighlightTerms } + // A few searches down the whole path (facets, ranking, snippets) load the index structures and, on the JVM, + // get the hot code compiled before the user's first search + repeat(WARMUP_ROUNDS) { + for (query in WARMUP_QUERIES) { + computeFacets(query, baseBookOnly = true) + openSession(query, baseBookOnly = true)?.use { it.nextPage(WARMUP_PAGE_SIZE) } + } + } + } + + override fun searchBooksByTitlePrefix( + query: String, + limit: Int, + ): List { + val q = HebrewTextUtils.normalizeHebrew(query) + if (q.isBlank()) return emptyList() + val tokens = q.split(WHITESPACE).map { it.trim() }.filter { it.isNotEmpty() } + if (tokens.isEmpty()) return emptyList() + + val titleQuery = + BoolBuilder() + // Restrict to book_title docs + .add(IndexQuery.Term("type", "book_title"), Occur.FILTER) + .apply { + // prefix on analyzed 'title' + tokens.forEach { tok -> add(IndexQuery.Prefix("title", tok), Occur.MUST) } + }.build() + val ids = LinkedHashSet() + for (doc in index.search(titleQuery, limit, setOf("book_id")).docs) { + doc.longs["book_id"]?.let { ids.add(it) } + } + return ids.toList() + } + + override fun buildSnippet( + rawText: String, + query: String, + near: Int, + ): String = SnippetBuilder.build(SnippetSources.clean(rawText), planFor(query, near)) + + override fun highlightRanges( + text: String, + query: String, + ): List = mergedRanges(text, planFor(query, DEFAULT_NEAR).highlights(text)) + + override fun rangeSnippet( + text: String, + ranges: List, + lineStart: Int, + lineEnd: Int, + ): String = SnippetBuilder.build(text, ranges.map { Occurrence(it.first, it.last + 1, 0, EvidenceLevel.LITERAL) }, lineStart, lineEnd) + + // The highlight plans of the latest queries: a results page and its preview ask for the same one many times + private val planCache = LruCache(PLAN_CACHE_SIZE) + + private fun planFor( + query: String, + near: Int, + ): HighlightPlan { + planCache[query]?.let { return it } + val plan = buildSearchContext(query, near, null, null, null, null)?.highlightPlan ?: HighlightPlan.EMPTY + planCache[query] = plan + return plan + } + + /** + * What a query highlights: each word with its inflections, from the dictionary entries it truly is a form of and + * that aren't blacklisted for it. With ה׳ in the query, the lone ה is the divine name. + */ + private fun buildHighlightPlan( + tokens: List, + hasHashem: Boolean, + exactPhrases: List>, + ): HighlightPlan { + val forms = + tokens.distinct().map { token -> + if (hasHashem && highlightKey(token) == "ה") { + return@map TokenForms(token, extraLiterals = hashemHighlightTerms) + } + val key = highlightKey(token) + val inflections = + magicDict + ?.expansionsFor(listOf(token)) + .orEmpty() + .filter { exp -> exp.inflections.any { highlightKey(it) == key } && !HallucinationBlacklist.rejects(token, exp) } + .flatMap { it.inflections } + TokenForms(token, inflections) + } + val phrases = + exactPhrases + .map { it.joinToString(" ") } + .filter { it.isNotBlank() } + return HighlightPlan.build(forms, phrases) + } + + override fun findInBookCandidates( + query: String, + bookId: Long, + ): LongArray? { + val norm = HebrewTextUtils.normalizeHebrew(query) + if (norm.isBlank()) return LongArray(0) + val bookLines = + BoolBuilder() + .add(IndexQuery.Term("type", "line"), Occur.FILTER) + .add(IndexQuery.IntExact("book_id", bookId.toInt()), Occur.FILTER) + .build() + // Nothing to filter on: every line would be a candidate, cheaper scanned from the DB + val clauses = literalPresenceClauses(index.analyze(norm)) + if (clauses.isEmpty()) return null + val builder = BoolBuilder().add(bookLines, Occur.FILTER) + clauses.forEach { builder.add(it, Occur.FILTER) } + + // Only line_id is read per hit: no scoring, no other stored field + val ids = index.collectLongs(builder.build(), "line_id") + // No candidate may mean a book the index lacks (older index, book added later): unknown, not "no match" + return if (ids.isEmpty() && index.count(bookLines) == 0) null else ids + } + + /** + * Filters every line holding the analyzed [tokens] as one literal substring satisfies. The + * substring may start and end mid-word: the first token is only a word suffix, the last a + * word prefix, the inner ones whole words. A first or lone token is matched anywhere in a + * word through its 4-grams, so one under 4 letters can't filter (a leading wildcard would + * scan the whole term dictionary); a last one under 3 neither, its prefix being too common. + */ + private fun literalPresenceClauses(tokens: List): List = + tokens.mapIndexedNotNull { i, token -> + when { + i in 1 until tokens.lastIndex -> IndexQuery.Term("text", token) + i > 0 && token.length >= 3 -> IndexQuery.Prefix("text", token) + else -> buildNgramPresenceForToken(token) + } + } + + override fun close() { + closed = true + index.close() + // The engine owns its dictionary (and its SQLite connections) once loaded; one still loading is closed by + // magicDict + if (dictionary.isInitialized()) dictionary.value?.close() + } + + override fun computeFacets( + query: String, + near: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection?, + lineIds: Collection?, + baseBookOnly: Boolean, + ): SearchFacets? { + val context = + buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) + ?: return null + return index.facets(context.matchQuery).let { SearchFacets(it.totalHits, it.categoryCounts, it.bookCounts) } + } + + // --- Inner SearchSession class --- + + private inner class LexicalSearchSession( + private val context: SearchContext, + private val cursor: IndexCursor, + ) : SearchSession { + private var finished = false + private var totalHitsValue: Long? = null + + override suspend fun nextPage( + limit: Int, + snippets: Boolean, + ): SearchPage? = + withContext(Dispatchers.IO) { + if (finished) return@withContext null + val page = cursor.next(limit) + if (totalHitsValue == null) totalHitsValue = page.totalHits + if (page.docs.isEmpty()) { + finished = true + return@withContext null + } + val hits = mapDocs(page.docs, context, snippets) + val isLast = page.docs.size < limit + if (isLast) finished = true + SearchPage( + hits = hits, + totalHits = totalHitsValue ?: hits.size.toLong(), + isLastPage = isLast, + ) + } + + override fun close() { + cursor.close() + } + } + + // --- Additional public search methods --- + + suspend fun searchAllText( + rawQuery: String, + near: Int = 5, + limit: Int, + offset: Int = 0, + ): List = doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = null) + + suspend fun searchInBook( + rawQuery: String, + near: Int, + bookId: Long, + limit: Int, + offset: Int = 0, + ): List = doSearch(rawQuery, near, limit, offset, bookFilter = bookId, categoryFilter = null) + + suspend fun searchInCategory( + rawQuery: String, + near: Int, + categoryId: Long, + limit: Int, + offset: Int = 0, + ): List = doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = categoryId) + + suspend fun searchInBooks( + rawQuery: String, + near: Int, + bookIds: Collection, + limit: Int, + offset: Int = 0, + ): List { + if (bookIds.isEmpty()) return emptyList() + return doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = null, bookIds = bookIds) + } + + // --- Private implementation --- + + private class SearchContext( + val query: IndexQuery, + // The required clauses only: the same matches as [query], without the ranking-only clauses (fuzzy, n-grams, + // synonym boosts) that cost a rewrite but can't change the hit set + val matchQuery: IndexQuery, + private val plan: Lazy, + ) { + val highlightPlan: HighlightPlan get() = plan.value + } + + private fun buildSearchContext( + rawQuery: String, + near: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection?, + lineIds: Collection?, + baseBookOnly: Boolean = false, + ): SearchContext? { + // Split the raw query into exact phrases (wrapped in ASCII double quotes) and free text. + // Quoted segments are matched verbatim WITHOUT dictionary expansion (Google-style). + val parsed = SearchQueryParser.parse(rawQuery) + val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) + val exactPhrasesNorm = + parsed.exactPhrases + .map { HebrewTextUtils.normalizeHebrew(it) } + .filter { it.isNotBlank() } + // Analyzed once, for both the phrase clauses and the highlights (a bridge call on iOS) + val exactPhraseTokens = exactPhrasesNorm.map { index.analyze(it) } + + if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null + + val analyzedRaw = if (norm.isBlank()) emptyList() else index.analyze(norm) + + // Check if the original query contained ה׳ (Hashem) before normalization + val hasHashem = rawQuery.contains("ה׳") || rawQuery.contains("ה'") + + // Filter out single Hebrew letters and stop words BEFORE dictionary expansion + // BUT preserve "ה" if the original query had "ה׳" (Hashem) + val analyzedStd = + analyzedRaw.filter { token -> + // Special case: if query has ה׳, keep "ה" token + if (token == "ה" && hasHashem) return@filter true + // Preserve numeric tokens (e.g., "6") so they can expand via MagicDictionary + if (token.any { it.isDigit() }) return@filter true + + token.length >= 2 && token !in SINGLE_LETTERS + } + + logger.d { "[DEBUG] Original query had Hashem (ה׳): $hasHashem" } + logger.d { "[DEBUG] Analyzed tokens: $analyzedStd" } + + // Get all possible expansions for each token (a token can belong to multiple bases) + // These expansions are used for SEARCH - we keep all of them for better recall + magicDict?.prefetch(analyzedStd) + val tokenExpansions: Map> = + analyzedStd.associateWith { token -> + // Get best expansion (prefers matching base, then largest) + val expansion = magicDict?.expansionFor(token) ?: return@associateWith emptyList() + listOf(expansion) + } + + // What the snippets highlight: built on first use (facets need none) + val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhraseTokens) } + + val builder = BoolBuilder() + // Mirrors every required clause of builder (the facets need only the hit set) + val matchBuilder = BoolBuilder() + + fun require( + query: IndexQuery, + occur: Occur, + ) { + builder.add(query, occur) + matchBuilder.add(query, occur) + } + require(IndexQuery.Term("type", "line"), Occur.FILTER) + if (bookFilter != null) require(IndexQuery.IntExact("book_id", bookFilter.toInt()), Occur.FILTER) + if (categoryFilter != null) require(IndexQuery.IntExact("category_id", categoryFilter.toInt()), Occur.FILTER) + // Filter by base books only (is_base_book = 1) when baseBookOnly is true + if (baseBookOnly) require(IndexQuery.IntExact("is_base_book", 1), Occur.FILTER) + val bookIdsArray = bookIds?.map { it.toInt() }?.toIntArray() + if (bookIdsArray != null && bookIdsArray.isNotEmpty()) { + require(IndexQuery.IntSet("book_id", bookIdsArray), Occur.FILTER) + } + val lineIdsArray = lineIds?.map { it.toInt() }?.toIntArray() + if (lineIdsArray != null && lineIdsArray.isNotEmpty()) { + require(IndexQuery.IntSet("line_id", lineIdsArray), Occur.FILTER) + } + // Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase. + var exactClausesAdded = 0 + for ((phrase, tokens) in exactPhrasesNorm.zip(exactPhraseTokens)) { + val exactQuery = buildExactPhraseQuery(phrase, tokens) ?: continue + require(exactQuery, Occur.MUST) + exactClausesAdded++ + logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" } + } + + // A fully-quoted query that produced no usable phrase clause (e.g. just punctuation) + // must not fall through to a match-all query. + if (norm.isBlank() && exactClausesAdded == 0) return null + + // Free (unquoted) text: dictionary-aware ranking + presence filtering. + if (norm.isNotBlank()) { + val rankedQuery = buildExpandedQuery(norm, near, analyzedRaw, analyzedStd, tokenExpansions) + val mustAllTokensQuery: IndexQuery? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions) + val phraseQuery: IndexQuery? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near) + if (mustAllTokensQuery != null) { + require(mustAllTokensQuery, Occur.FILTER) + } + if (phraseQuery != null && analyzedStd.size >= 2) { + if (near == 0) require(phraseQuery, Occur.MUST) else builder.add(phraseQuery, Occur.SHOULD) + } + builder.add(rankedQuery, Occur.SHOULD) + } + + return SearchContext( + query = builder.build(), + matchQuery = matchBuilder.build(), + plan = highlightPlan, + ) + } + + // suspend because it calls snippetProvider.getSnippetSources(); + // callers are responsible for providing Dispatchers.IO context. + private suspend fun mapDocs( + docs: List, + context: SearchContext, + snippets: Boolean = true, + ): List { + if (docs.isEmpty()) return emptyList() + + // First pass: extract metadata from index + class DocMeta( + val score: Float, + val bookId: Long, + val bookTitle: String, + val lineId: Long, + val lineIndex: Int, + val isBaseBook: Boolean, + val orderIndex: Int, + // from text_raw field, may be empty if not stored + val indexedRaw: String, + ) + + val docMetas = + docs.map { doc -> + DocMeta( + score = doc.score, + bookId = doc.longs.getValue("book_id"), + bookTitle = doc.strings["book_title"] ?: "", + lineId = doc.longs.getValue("line_id"), + lineIndex = doc.longs.getValue("line_index").toInt(), + isBaseBook = doc.longs["is_base_book"]?.toInt() == 1, + orderIndex = doc.longs["order_index"]?.toInt() ?: 999, + indexedRaw = doc.strings["text_raw"] ?: "", + ) + } + + val snippetSources: Map = + if (snippets) snippetSources(docMetas.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) else emptyMap() + + val hits = + docMetas.map { meta -> + val source = snippetSources[meta.lineId] ?: SnippetSource(meta.indexedRaw, 0, meta.indexedRaw.length) + val raw = source.text + val baseScore = meta.score + + // Calculate boost: lower orderIndex = higher boost (only for base books) + val boostedScore = + if (meta.isBaseBook) { + // Formula: boost = baseScore * (1 + (120 - orderIndex) / 60) + // orderIndex 1 gets ~3x boost, orderIndex 50 gets ~2.2x boost, orderIndex 100+ gets ~1.3x boost + val boostFactor = 1.0f + (120 - meta.orderIndex).coerceAtLeast(0) / 60.0f + baseScore * boostFactor + } else { + baseScore + } + + val snippet = if (snippets) SnippetBuilder.build(raw, context.highlightPlan, source.lineStart, source.lineEnd) else "" + LineHit( + bookId = meta.bookId, + bookTitle = meta.bookTitle, + lineId = meta.lineId, + lineIndex = meta.lineIndex, + snippet = snippet, + score = boostedScore, + rawText = raw, + isBaseBook = meta.isBaseBook, + ) + } + // Re-sort by boosted score (descending) + return hits.sortedByDescending { it.score } + } + + // Snippet sources: from the provider if available, otherwise none (the indexed text_raw fallback) + private suspend fun snippetSources(lines: List): Map = + snippetProvider?.getSnippetSources(lines) ?: emptyMap() + + private suspend fun doSearch( + rawQuery: String, + near: Int, + limit: Int, + offset: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection? = null, + ): List { + val context = buildSearchContext(rawQuery, near, bookFilter, categoryFilter, bookIds, null) ?: return emptyList() + return withContext(Dispatchers.IO) { + val top = index.search(context.query, offset + limit, HIT_FIELDS) + mapDocs(top.docs.drop(offset), context) + } + } + + private fun buildNgramPresenceForToken(token: String): IndexQuery? { + if (token.length < 4) return null + val grams = (0..token.length - 4).map { token.substring(it, it + 4) }.distinct() + if (grams.isEmpty()) return null + return BoolBuilder().apply { grams.forEach { add(IndexQuery.Term("text_ng4", it), Occur.MUST) } }.build() + } + + private fun buildPresenceFilterForTokens( + tokens: List, + near: Int, + expansionsByToken: Map>, + ): IndexQuery? { + if (tokens.isEmpty()) return null + val outer = BoolBuilder() + for (t in tokens) { + val expansions = expansionsByToken[t] ?: emptyList() + val synonymTerms = buildLimitedTermsForToken(t, expansions) + val ngram = if (near > 0) buildNgramPresenceForToken(t) else null + val clause = + BoolBuilder() + .apply { + add(IndexQuery.Term("text", t), Occur.SHOULD) + if (ngram != null) add(ngram, Occur.SHOULD) + for (term in synonymTerms) { + if (term != t) add(IndexQuery.Term("text", term), Occur.SHOULD) + } + }.build() + outer.add(clause, Occur.MUST) + } + return outer.build() + } + + /** + * Builds a strict, verbatim phrase query for a quoted segment: terms must appear in the + * given order and be adjacent (slop = 0), with NO dictionary expansion, fuzzy matching or + * n-gram substring matching. A single-token segment becomes an exact term query. + * + * Nikud and final letters are still normalised (the segment is normalised by the caller and + * tokenised like the index) so that the phrase matches the indexed form. + */ + private fun buildExactPhraseQuery( + phraseNorm: String, + tokens: List, + ): IndexQuery? { + if (phraseNorm.isBlank() || tokens.isEmpty()) return null + return IndexQuery.AnalyzedPhrase("text", phraseNorm, 0) + } + + private fun expansionTerms(expansions: List): Triple, List, List> { + val surfaceTerms = LinkedHashSet() + val variantTerms = LinkedHashSet() + val baseTerms = LinkedHashSet() + for (exp in expansions) { + surfaceTerms.addAll(exp.surface) + variantTerms.addAll(exp.variants) + baseTerms.addAll(exp.base) + } + return Triple( + surfaceTerms.take(MAX_SYNONYM_BOOST_TERMS), + variantTerms.take(MAX_SYNONYM_BOOST_TERMS), + baseTerms.take(MAX_SYNONYM_BOOST_TERMS), + ) + } + + private fun buildMagicBoostQuery(expansions: List): IndexQuery? { + if (expansions.isEmpty()) return null + val (surfaces, variants, bases) = expansionTerms(expansions) + val b = BoolBuilder() + for (s in surfaces) b.add(IndexQuery.Boost(IndexQuery.Term("text", s), 2.0f), Occur.SHOULD) + for (v in variants) b.add(IndexQuery.Boost(IndexQuery.Term("text", v), 1.5f), Occur.SHOULD) + for (ba in bases) b.add(IndexQuery.Boost(IndexQuery.Term("text", ba), 1.0f), Occur.SHOULD) + return b.build() + } + + private fun buildSynonymBoostQuery(expansions: List): IndexQuery? { + if (expansions.isEmpty()) return null + val (surfaces, variants, bases) = expansionTerms(expansions) + val b = BoolBuilder() + for (term in surfaces + variants + bases) b.add(IndexQuery.Term("text", term), Occur.SHOULD) + return b.build() + } + + private fun buildSynonymPhrases( + tokens: List, + expansionsByToken: Map>, + ): List> { + if (tokens.isEmpty()) return emptyList() + val termExpansions = buildTermAlternativesForTokens(tokens, expansionsByToken) + return listOf( + IndexQuery.MultiPhrase("text", termExpansions, 0) to 50.0f, + IndexQuery.MultiPhrase("text", termExpansions, 3) to 20.0f, + IndexQuery.MultiPhrase("text", termExpansions, 8) to 5.0f, + ) + } + + private fun buildSynonymPhraseQuery( + tokens: List, + expansionsByToken: Map>, + near: Int, + ): IndexQuery? { + if (tokens.isEmpty()) return null + return IndexQuery.MultiPhrase("text", buildTermAlternativesForTokens(tokens, expansionsByToken), near) + } + + private fun buildNgram4Query(norm: String): IndexQuery? { + val tokens = norm.split(WHITESPACE).map { it.trim() }.filter { it.length >= 4 } + if (tokens.isEmpty()) return null + val uniq = tokens.flatMap { t -> (0..t.length - 4).map { t.substring(it, it + 4) } }.distinct() + if (uniq.isEmpty()) return null + return BoolBuilder().apply { uniq.forEach { add(IndexQuery.Term("text_ng4", it), Occur.MUST) } }.build() + } + + private fun buildExpandedQuery( + norm: String, + near: Int, + analyzed: List, + tokens: List, + expansionsByToken: Map>, + ): IndexQuery { + val base = IndexQuery.AnalyzedPhrase("text", norm, near) + val allExpansions = expansionsByToken.values.flatten() + val builder = BoolBuilder() + builder.add(base, Occur.SHOULD) + for ((query, boost) in buildSynonymPhrases(tokens, expansionsByToken)) { + builder.add(IndexQuery.Boost(query, boost), Occur.SHOULD) + } + buildNgram4Query(norm)?.let { builder.add(it, Occur.SHOULD) } + buildFuzzyQuery(norm, near, analyzed)?.let { builder.add(it, Occur.SHOULD) } + buildMagicBoostQuery(allExpansions)?.let { builder.add(it, Occur.SHOULD) } + buildSynonymBoostQuery(allExpansions)?.let { builder.add(it, Occur.SHOULD) } + return builder.build() + } + + private fun buildFuzzyQuery( + norm: String, + near: Int, + analyzed: List, + ): IndexQuery? { + if (near == 0) return null + if (norm.length < 4) return null + val tokens = analyzed.filter { it.length >= 4 } + if (tokens.isEmpty()) return null + return BoolBuilder().apply { tokens.distinct().forEach { add(IndexQuery.Fuzzy("text", it, 1), Occur.MUST) } }.build() + } + + private fun buildLimitedTermsForToken( + token: String, + expansions: List, + ): List { + if (expansions.isEmpty()) return listOf(token) + + val baseTerms = expansions.flatMap { it.base }.distinct() + val otherTerms = expansions.flatMap { it.surface + it.variants }.distinct() + + val ordered = LinkedHashSet() + if (token.isNotBlank()) ordered += token + baseTerms.forEach { ordered += it } + otherTerms.forEach { ordered += it } + return ordered.take(MAX_SYNONYM_TERMS_PER_TOKEN) + } + + private fun buildTermAlternativesForTokens( + tokens: List, + expansionsByToken: Map>, + ): List> = tokens.map { token -> buildLimitedTermsForToken(token, expansionsByToken[token] ?: emptyList()) } + + // The divine name's forms never change: read once from the dictionary, not on every search + private val hashemHighlightTerms: List by lazy { readHashemHighlightTerms() } + + private fun readHashemHighlightTerms(): List { + val dict = magicDict ?: return emptyList() + val raw = dict.loadHashemSurfaces() + if (raw.isEmpty()) return emptyList() + + val terms = linkedSetOf() + raw.forEach { value -> + val trimmed = value.trim() + if (trimmed.isEmpty()) return@forEach + terms += trimmed + val stripped = HebrewTextUtils.stripDiacritics(trimmed).trim() + if (stripped.isNotEmpty()) terms += stripped + val normalized = HebrewTextUtils.normalizeHebrew(trimmed).trim() + if (normalized.isNotEmpty()) terms += normalized + } + return terms.toList() + } +} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt new file mode 100644 index 00000000..a2a3da18 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt @@ -0,0 +1,48 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized + +/** A thread-safe cache of the [capacity] most recently used entries. */ +internal class LruCache( + private val capacity: Int, +) { + private val lock = SynchronizedObject() + + // Insertion order is kept: a hit moves its entry to the end, so the first entry is the least recently used + private val map = LinkedHashMap() + + operator fun get(key: K): V? = + synchronized(lock) { + map.remove(key)?.also { map[key] = it } + } + + operator fun set( + key: K, + value: V, + ) { + synchronized(lock) { putLocked(key, value) } + } + + /** The value of [key], else [create]'s, stored; [create] runs under the lock, so it must be cheap. */ + fun getOrPut( + key: K, + create: () -> V, + ): V = + synchronized(lock) { + map.remove(key)?.also { map[key] = it } ?: create().also { putLocked(key, it) } + } + + fun clear() { + synchronized(lock) { map.clear() } + } + + private fun putLocked( + key: K, + value: V, + ) { + map.remove(key) + map[key] = value + if (map.size > capacity) map.remove(map.keys.first()) + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt similarity index 51% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt index c39f5a99..96d3bee0 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt @@ -1,15 +1,14 @@ package io.github.kdroidfilter.seforimlibrary.search import co.touchlab.kermit.Logger -import java.nio.file.Files -import java.nio.file.Path -import java.sql.Connection -import java.sql.DriverManager -import java.sql.PreparedStatement -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem /** * Streaming dictionary index backed by SQLite (tables: surface, variant, base). @@ -19,8 +18,8 @@ import java.util.concurrent.atomic.AtomicInteger */ class MagicDictionaryIndex private constructor( private val norm: (String) -> String, - private val dbFile: Path -) { + private val db: SqliteReader, +) : AutoCloseable { /** * The forms a token expands to. [surface] holds every form (inflections, then the dictionary's variants, then * the base) as the ranking uses them; [inflections] only the base's inflected forms and the base itself: the @@ -30,54 +29,30 @@ class MagicDictionaryIndex private constructor( val surface: List, val variants: List, val base: List, - val inflections: List = emptyList() + val inflections: List = emptyList(), ) - private val url = "jdbc:sqlite:${dbFile.toAbsolutePath()}" - - /** - * Prepared statement per thread to avoid re-opening connections on every token. - */ - private val stmtProvider: ThreadLocal = ThreadLocal.withInitial { - val conn = DriverManager.getConnection(url).apply { - autoCommit = false - // Enforce read-only queries without altering connection flags post-open - createStatement().use { stmt -> stmt.execute("PRAGMA query_only=ON") } - } - // Dictionaries since v0.4.2 list the extra lemmas of ambiguous surfaces (פנינו: פנה and פנים) - val sql = if (conn.hasTable(EXTRA_BASE_TABLE)) LOOKUP_WITH_EXTRA_BASES_SQL else LOOKUP_SQL - LookupContext( - conn = conn, - stmt = conn.prepareStatement(sql), - paramCount = sql.count { it == '?' } - ) - } + // Dictionaries since v0.4.2 list the extra lemmas of ambiguous surfaces (פנינו: פנה and פנים) + private val lookupSql: String by lazy { if (db.hasTable(EXTRA_BASE_TABLE)) LOOKUP_WITH_EXTRA_BASES_SQL else LOOKUP_SQL } /** * Cache expansions per normalized token to avoid repeated DB hits. Lookups in progress are cached too, so * concurrent searches (facets and results start together) share them. */ - private val tokenCache = object : LinkedHashMap>>(TOKEN_CACHE_SIZE, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry>>?): Boolean = - size > TOKEN_CACHE_SIZE - } + private val tokenCache = LruCache>>(TOKEN_CACHE_SIZE) /** * Cache fully-normalized expansions per base id so repeated hits to the same base * avoid re-normalizing rows. */ - private val baseCache = object : LinkedHashMap(BASE_CACHE_SIZE, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = - size > BASE_CACHE_SIZE - } + private val baseCache = LruCache(BASE_CACHE_SIZE) /** Starts looking up [tokens] in parallel, so the following [expansionFor] calls find them ready. */ fun prefetch(tokens: Collection) { tokens.forEach { lookup(it, async = true) } } - fun expansionsFor(tokens: List): List = - tokens.flatMap { expansionsForToken(it) }.distinct() + fun expansionsFor(tokens: List): List = tokens.flatMap { expansionsForToken(it) }.distinct() fun expansionFor(token: String): Expansion? { val expansions = expansionsForToken(token) @@ -85,9 +60,10 @@ class MagicDictionaryIndex private constructor( val normalized = norm(token) // Strategy: prefer the expansion whose base matches the token - val matchingBase = expansions.firstOrNull { exp -> - exp.base.any { it == normalized } - } + val matchingBase = + expansions.firstOrNull { exp -> + exp.base.any { it == normalized } + } if (matchingBase != null) return matchingBase // Then the entries the token truly is a form of: one reached only through a dictionary variant (loose @@ -98,25 +74,35 @@ class MagicDictionaryIndex private constructor( return ownEntries.ifEmpty { expansions }.maxByOrNull { it.surface.size } } - private fun expansionsForToken(token: String): List = lookup(token, async = false).join() + private fun expansionsForToken(token: String): List { + val result = lookup(token, async = false) + @Suppress("OPT_IN_USAGE") + return if (result.isCompleted) result.getCompleted() else runBlockingIo { result.await() } + } - private fun lookup(token: String, async: Boolean): CompletableFuture> { + private fun lookup( + token: String, + async: Boolean, + ): CompletableDeferred> { val normalized = norm(token) - if (normalized.isEmpty()) return CompletableFuture.completedFuture(emptyList()) - val future = CompletableFuture>() - synchronized(tokenCache) { - tokenCache[normalized]?.let { return it } - tokenCache[normalized] = future - } - val fetch = Runnable { - // fetchExpansions logs and skips its failures: an empty result is cached like any other - future.complete(runCatching { fetchForToken(token, normalized) }.getOrDefault(emptyList())) - } - if (async) lookupExecutor.execute(fetch) else fetch.run() - return future + if (normalized.isEmpty()) return CompletableDeferred(emptyList()) + var created = false + val result = + tokenCache.getOrPut(normalized) { + created = true + CompletableDeferred() + } + if (!created) return result + // fetchExpansions logs and skips its failures: an empty result is cached like any other + val fetch = { result.complete(runCatching { fetchForToken(token, normalized) }.getOrDefault(emptyList())) } + if (async) lookupScope.launch { fetch() } else fetch() + return result } - private fun fetchForToken(token: String, normalized: String): List { + private fun fetchForToken( + token: String, + normalized: String, + ): List { // Try raw, normalized, and final-form variants to match DB values. val candidates = buildLookupCandidates(token, normalized) val mergedByBase = LinkedHashMap() @@ -142,57 +128,58 @@ class MagicDictionaryIndex private constructor( /** * Fetch expansions for a token. Returns a list of (baseId, Expansion) pairs so callers can merge by base id. */ - private fun fetchExpansions(rawToken: String, normalizedToken: String): List> { + private fun fetchExpansions( + rawToken: String, + normalizedToken: String, + ): List> { val expansions = mutableListOf>() - val ctx = stmtProvider.get() runCatching { - synchronized(ctx) { - repeat(ctx.paramCount) { idx -> ctx.stmt.setString(idx + 1, rawToken) } - val rs = ctx.stmt.executeQuery() - val accum = mutableMapOf() - while (rs.next()) { - val baseId = rs.getLong("base_id") - val bucket = accum.getOrPut(baseId) { + val sql = lookupSql + val rows = db.query(sql, List(sql.count { it == '?' }) { rawToken }) + val accum = LinkedHashMap() + for (row in rows) { + val baseId = row["base_id"]?.toLongOrNull() ?: continue + val bucket = + accum.getOrPut(baseId) { BaseBucket( - baseRaw = rs.getString("base") ?: "", + baseRaw = row["base"] ?: "", surfaces = linkedSetOf(), - variants = linkedSetOf() + variants = linkedSetOf(), ) } - rs.getString("surface")?.let { bucket.surfaces += it } - rs.getString("variant")?.let { bucket.variants += it } - } + row["surface"]?.let { bucket.surfaces += it } + row["variant"]?.let { bucket.variants += it } + } - for ((baseId, bucket) in accum) { - val cached = synchronized(baseCache) { baseCache[baseId] } - if (cached != null) { - expansions += baseId to cached - continue - } + for ((baseId, bucket) in accum) { + val cached = baseCache[baseId] + if (cached != null) { + expansions += baseId to cached + continue + } - val surfaceN = bucket.surfaces.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } - val variantsN = bucket.variants.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } - val baseN = norm(bucket.baseRaw).takeIf { it.isNotEmpty() } + val surfaceN = bucket.surfaces.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } + val variantsN = bucket.variants.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } + val baseN = + norm(bucket.baseRaw).takeIf { it.isNotEmpty() } ?: surfaceN.firstOrNull() ?: normalizedToken - val baseTerms = listOfNotNull(baseN.takeIf { it.isNotEmpty() }) - val allTerms = (surfaceN + variantsN + baseTerms).distinct() - if (allTerms.isEmpty()) continue + val baseTerms = listOfNotNull(baseN.takeIf { it.isNotEmpty() }) + val allTerms = (surfaceN + variantsN + baseTerms).distinct() + if (allTerms.isEmpty()) continue - val exp = Expansion( + val exp = + Expansion( surface = allTerms, variants = emptyList(), base = baseTerms, - inflections = (surfaceN + baseTerms).distinct() + inflections = (surfaceN + baseTerms).distinct(), ) - synchronized(baseCache) { - baseCache[baseId] = exp - } - expansions += baseId to exp - } + baseCache[baseId] = exp + expansions += baseId to exp } }.onFailure { logger.d { "[MagicDictionary] Failed to fetch expansions for '$rawToken' : ${it.message}" } @@ -201,76 +188,100 @@ class MagicDictionaryIndex private constructor( return expansions } + /** + * Load all surface forms whose base lemma directly from the underlying SQLite DB. + * This is used for snippet highlighting of Hashem names, independent of token-level expansions. + */ + fun loadHashemSurfaces(): List { + val terms = linkedSetOf() + runCatching { + val sql = + """ + SELECT s.value AS surface + FROM surface s + JOIN base b ON s.base_id = b.id + WHERE b.value = 'יהוה' + """.trimIndent() + for (row in db.query(sql)) { + val trimmed = row["surface"]?.trim() ?: continue + if (trimmed.isNotEmpty()) terms += trimmed + } + }.onFailure { + logger.d { "[MagicDictionary] Failed to load Hashem surfaces: ${it.message}" } + } + return terms.toList() + } + + override fun close() { + db.close() + } + companion object { private val logger = Logger.withTag("MagicDictionary") private const val TOKEN_CACHE_SIZE = 1024 - - // Parallel lookups of a query's words (each thread keeps its own connection) - private val lookupExecutor: ExecutorService by lazy { - val ids = AtomicInteger() - Executors.newFixedThreadPool(minOf(Runtime.getRuntime().availableProcessors(), 4)) { task -> - Thread(task, "magic-dictionary-${ids.incrementAndGet()}").apply { isDaemon = true } - } - } private const val BASE_CACHE_SIZE = 512 + private const val LOOKUP_PARALLELISM = 4 + + // Parallel lookups of a query's words + private val lookupScope = CoroutineScope(SupervisorJob() + Dispatchers.IO.limitedParallelism(LOOKUP_PARALLELISM)) + + private val FINALS = mapOf('כ' to 'ך', 'מ' to 'ם', 'נ' to 'ן', 'פ' to 'ף', 'צ' to 'ץ') + + private val REQUIRED_TABLES = listOf("surface", "variant", "base", "surface_variant") /** * Load from SQLite DB (expected tables: surface(value, base_id), variant(value, surface_id), base(value)). * Uses streaming lookup to avoid holding the entire dictionary in memory. */ - fun load(norm: (String) -> String, candidate: Path?): MagicDictionaryIndex? { - val file = candidate?.takeIf { Files.isRegularFile(it) && hasRequiredTables(it) } ?: run { - if (candidate != null) { - logger.d { "[MagicDictionary] Ignoring candidate $candidate because required tables are missing" } - } - return null - } + fun load( + norm: (String) -> String, + candidate: String?, + ): MagicDictionaryIndex? { + if (candidate == null || !isRegularFile(candidate)) return null return runCatching { - // Validate DB is reachable - DriverManager.getConnection("jdbc:sqlite:${file.toAbsolutePath()}").use { conn -> - conn.createStatement().use { stmt -> - stmt.execute("SELECT 1") - } + val db = openReadOnlySqlite(candidate) + if (!db.hasRequiredTables()) { + db.close() + logger.d { "[MagicDictionary] Ignoring candidate $candidate because required tables are missing" } + return null } - logger.d { "[MagicDictionary] Streaming lexical db from $file (lazy on-demand)" } - MagicDictionaryIndex(norm, file) + logger.d { "[MagicDictionary] Streaming lexical db from $candidate (lazy on-demand)" } + MagicDictionaryIndex(norm, db) }.onFailure { - logger.d { "[MagicDictionary] Failed to load from $file : ${it.message}" } + logger.d { "[MagicDictionary] Failed to load from $candidate : ${it.message}" } }.getOrNull() } /** * Find the first candidate path that exists and contains the required tables. */ - fun findValidDictionary(candidates: List): Path? { + fun findValidDictionary(candidates: List): String? { for (candidate in candidates) { - if (!Files.isRegularFile(candidate)) continue - if (hasRequiredTables(candidate)) { + if (!isRegularFile(candidate)) continue + val valid = runCatching { openReadOnlySqlite(candidate).use { it.hasRequiredTables() } }.getOrDefault(false) + if (valid) { logger.d { "[MagicDictionary] Using validated lexical db at $candidate" } return candidate - } else { - logger.d { - "[MagicDictionary] Candidate $candidate is present but missing required tables; skipping" - } } + logger.d { "[MagicDictionary] Candidate $candidate is present but missing required tables; skipping" } } return null } - private fun hasRequiredTables(file: Path): Boolean = runCatching { - DriverManager.getConnection("jdbc:sqlite:${file.toAbsolutePath()}").use { conn -> - val sql = """ - SELECT name FROM sqlite_master - WHERE type = 'table' AND name IN ('surface', 'variant', 'base', 'surface_variant') - """.trimIndent() - conn.createStatement().use { stmt -> - val rs = stmt.executeQuery(sql) - val names = mutableSetOf() - while (rs.next()) names += rs.getString("name") ?: "" - names.containsAll(listOf("surface", "variant", "base", "surface_variant")) - } - } - }.getOrElse { false } + private fun isRegularFile(path: String): Boolean = SystemFileSystem.metadataOrNull(Path(path))?.isRegularFile == true + + private fun SqliteReader.hasRequiredTables(): Boolean = + runCatching { + val names = + query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN " + + REQUIRED_TABLES.joinToString(prefix = "(", postfix = ")") { "'$it'" }, + ).mapNotNull { it["name"] } + names.containsAll(REQUIRED_TABLES) + }.getOrDefault(false) + + private fun SqliteReader.hasTable(name: String): Boolean = + query("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = ?", listOf(name)).isNotEmpty() private const val LOOKUP_SQL = """ WITH matches AS ( @@ -332,39 +343,22 @@ class MagicDictionaryIndex private constructor( LEFT JOIN surface_variant sv ON sv.surface_id = s.id AND mb.own = 1 LEFT JOIN variant v ON sv.variant_id = v.id """ - - private fun Connection.hasTable(name: String): Boolean = - prepareStatement("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").use { stmt -> - stmt.setString(1, name) - stmt.executeQuery().use { it.next() } - } } - private data class LookupContext( - val conn: Connection, - val stmt: PreparedStatement, - val paramCount: Int - ) - - private data class BaseBucket( + private class BaseBucket( val baseRaw: String, val surfaces: MutableSet, - val variants: MutableSet + val variants: MutableSet, ) - private fun buildLookupCandidates(rawToken: String, normalized: String): List { - val finalsMap = mapOf( - 'כ' to 'ך', - 'מ' to 'ם', - 'נ' to 'ן', - 'פ' to 'ף', - 'צ' to 'ץ' - ) - + private fun buildLookupCandidates( + rawToken: String, + normalized: String, + ): List { fun applyFinalForm(t: String): String { if (t.isEmpty()) return t val last = t.last() - val final = finalsMap[last] ?: last + val final = FINALS[last] ?: last return if (final == last) t else t.dropLast(1) + final } @@ -372,38 +366,7 @@ class MagicDictionaryIndex private constructor( rawToken, normalized, applyFinalForm(rawToken), - applyFinalForm(normalized) + applyFinalForm(normalized), ).filter { it.isNotBlank() }.distinct() } - - /** - * Load all surface forms whose base lemma directly from the underlying SQLite DB. - * This is used for snippet highlighting of Hashem names, independent of token-level expansions. - */ - fun loadHashemSurfaces(): List { - val terms = linkedSetOf() - runCatching { - DriverManager.getConnection(url).use { conn -> - val sql = """ - SELECT s.value AS surface - FROM surface s - JOIN base b ON s.base_id = b.id - WHERE b.value = 'יהוה' - """.trimIndent() - conn.createStatement().use { stmt -> - val rs = stmt.executeQuery(sql) - while (rs.next()) { - val v = rs.getString("surface") ?: continue - val trimmed = v.trim() - if (trimmed.isNotEmpty()) { - terms += trimmed - } - } - } - } - }.onFailure { - logger.d { "[MagicDictionary] Failed to load Hashem surfaces: ${it.message}" } - } - return terms.toList() - } } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt new file mode 100644 index 00000000..baae6173 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt @@ -0,0 +1,26 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.coroutines.CoroutineScope + +/** + * The text of [html] as snippet text is built from: tags dropped, special characters escaped as HTML entities (what + * `Jsoup.clean(html, Safelist.none())` returns, which the JVM and Android keep using). + */ +internal expect fun escapedText(html: String): String + +/** Runs [block] to completion on the calling thread, for the synchronous parts of the API that wait on I/O. */ +internal expect fun runBlockingIo(block: suspend CoroutineScope.() -> T): T + +/** + * A read-only SQLite database (the lexical dictionary). Safe to use from several threads at once. + */ +interface SqliteReader : AutoCloseable { + /** The rows [sql] returns with [args] bound in order, each as its column names to their text values. */ + fun query( + sql: String, + args: List = emptyList(), + ): List> +} + +/** Opens the SQLite database at [path] read-only; throws when it can't be opened. */ +expect fun openReadOnlySqlite(path: String): SqliteReader diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt similarity index 99% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt index 64cc5d0d..4445951f 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt @@ -1,6 +1,5 @@ package io.github.kdroidfilter.seforimlibrary.search -import java.io.Closeable /** * Main interface for full-text search operations on Hebrew religious texts. @@ -31,7 +30,7 @@ import java.io.Closeable * @see SearchSession for paginated result access * @see LineHit for individual search result structure */ -interface SearchEngine : Closeable { +interface SearchEngine : AutoCloseable { /** * Opens a search session for the given query with optional filters. diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt similarity index 97% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt index 2226885b..7d52443e 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt @@ -1,6 +1,7 @@ package io.github.kdroidfilter.seforimlibrary.search -import java.io.Closeable +import kotlinx.serialization.Serializable + /** * A stateful search session providing paginated access to search results. @@ -26,7 +27,7 @@ import java.io.Closeable * @see SearchEngine.openSession to create a session * @see SearchPage for the structure of returned pages */ -interface SearchSession : Closeable { +interface SearchSession : AutoCloseable { /** * Retrieves the next page of search results. * @@ -100,6 +101,7 @@ data class LineHit( * @property categoryCounts Map of categoryId to count (includes ancestor categories) * @property bookCounts Map of bookId to count */ +@Serializable data class SearchFacets( val totalHits: Long, val categoryCounts: Map, diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt similarity index 96% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt index 858dc31b..290184b8 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt @@ -1,8 +1,5 @@ package io.github.kdroidfilter.seforimlibrary.search -import org.jsoup.Jsoup -import org.jsoup.safety.Safelist - /** * Metadata about a line needed to fetch its snippet source text. * @@ -98,7 +95,7 @@ object SnippetSources { } /** [html] as snippet text: tags dropped, entities escaped, whitespace collapsed. */ - fun clean(html: String): String = Jsoup.clean(html, Safelist.none()).replace(WHITESPACE, " ").trim() + fun clean(html: String): String = escapedText(html).replace(WHITESPACE, " ").trim() private fun windows(lineIndexes: List): List { val out = ArrayList() diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt new file mode 100644 index 00000000..9dc2c981 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt @@ -0,0 +1,15 @@ +package io.github.kdroidfilter.seforimlibrary.search.index + +/** A [IndexQuery.Bool] built clause by clause. */ +class BoolBuilder { + private val clauses = ArrayList() + + fun add( + query: IndexQuery, + occur: Occur, + ): BoolBuilder = apply { clauses += Clause(query, occur) } + + fun build(): IndexQuery.Bool = IndexQuery.Bool(clauses.toList()) +} + +inline fun bool(block: BoolBuilder.() -> Unit): IndexQuery.Bool = BoolBuilder().apply(block).build() diff --git a/search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt b/search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt new file mode 100644 index 00000000..9575b929 --- /dev/null +++ b/search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt @@ -0,0 +1,72 @@ +package io.github.kdroidfilter.seforimlibrary.search + +/** + * A text dump of what a search engine answers for [PARITY_QUERIES]: facets, ranked hits with their snippets, + * highlights, title suggestions, find-in-book candidates. The same index gives the same dump on every platform (the + * JVM, Android, iOS through its Java runtime): the parity tests compare them. + */ +object SearchDump { + val PARITY_QUERIES = + WARMUP_QUERIES + + listOf( + "שבת", + "בראשית ברא", + "רבי עקיבא", + "\"ואהבת לרעך כמוך\"", + "תפילין של ראש", + "משה רבינו", + "רש״י", + "ה׳ אחד", + "קידוש השם", + "גמילות חסדים", + "וַיֹּאמֶר אֱלֹהִים", + ) + + private const val PAGE_SIZE = 20 + private const val TOP_FACETS = 5 + + suspend fun dump(engine: SearchEngine): String = + buildString { + for (query in PARITY_QUERIES) { + appendLine("## $query") + for (baseBookOnly in listOf(false, true)) { + val facets = engine.computeFacets(query, baseBookOnly = baseBookOnly) + appendLine("facets[base=$baseBookOnly] total=${facets?.totalHits}") + facets?.categoryCounts?.entries?.sortedWith(compareByDescending> { it.value }.thenBy { it.key }) + ?.take(TOP_FACETS) + ?.forEach { appendLine(" category ${it.key}=${it.value}") } + facets?.bookCounts?.entries?.sortedWith(compareByDescending> { it.value }.thenBy { it.key }) + ?.take(TOP_FACETS) + ?.forEach { appendLine(" book ${it.key}=${it.value}") } + engine.openSession(query, baseBookOnly = baseBookOnly)?.use { session -> + repeat(2) { pageIndex -> + val page = session.nextPage(PAGE_SIZE) ?: return@repeat + appendLine("page $pageIndex total=${page.totalHits} last=${page.isLastPage}") + for (hit in page.hits) { + appendLine(" ${hit.lineId} book=${hit.bookId} line=${hit.lineIndex} base=${hit.isBaseBook} score=${hit.score.format()}") + appendLine(" ${hit.snippet}") + } + } + } ?: appendLine("no session") + } + val sample = "בראשית ברא אלהים את השמים ואת הארץ, ויאמר ה׳ אחד ושבת קודש" + appendLine("highlight ${engine.highlightRanges(sample, query)}") + appendLine("snippet ${engine.buildSnippet(sample, query, 5)}") + } + appendLine("## titles") + for (prefix in listOf("בראש", "משנה ב", "שולחן")) appendLine("$prefix -> ${engine.searchBooksByTitlePrefix(prefix, 10)}") + appendLine("## find in book") + val firstBook = engine.openSession("שבת", baseBookOnly = true)?.use { it.nextPage(1)?.hits?.firstOrNull()?.bookId } + if (firstBook != null) { + for (query in listOf("שבת", "ויכלו השמים", "ים")) { + appendLine("$query in $firstBook -> ${engine.findInBookCandidates(query, firstBook)?.sorted()?.take(30)}") + } + } + } + + // Rounded: the scores are computed by the same Lucene everywhere, the rounding only keeps the dump readable + private fun Float.format(): String { + val scaled = kotlin.math.round(this * 1000).toLong() + return "${scaled / 1000}.${(scaled % 1000).toString().padStart(3, '0')}" + } +} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt new file mode 100644 index 00000000..4d127938 --- /dev/null +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt @@ -0,0 +1,11 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.safety.Safelist +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.runBlocking + +// Ksoup is jsoup's Kotlin port: the same cleaning as the JVM and Android +internal actual fun escapedText(html: String): String = Ksoup.clean(html, Safelist.none()) + +internal actual fun runBlockingIo(block: suspend CoroutineScope.() -> T): T = runBlocking(block = block) diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt new file mode 100644 index 00000000..bb270b48 --- /dev/null +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt @@ -0,0 +1,106 @@ +@file:OptIn(ExperimentalForeignApi::class) + +package io.github.kdroidfilter.seforimlibrary.search + +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_DONE +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OK +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OPEN_FULLMUTEX +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OPEN_READONLY +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_ROW +import cnames.structs.sqlite3 +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_bind_text +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_close_v2 +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_column_count +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_column_name +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_column_text +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_errmsg +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_finalize +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_open_v2 +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_prepare_v2 +import cnames.structs.sqlite3_stmt +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_step +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized +import kotlinx.cinterop.CFunction +import kotlinx.cinterop.COpaquePointer +import kotlinx.cinterop.CPointer +import kotlinx.cinterop.CPointerVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.toCPointer +import kotlinx.cinterop.toKString +import kotlinx.cinterop.value + +/** Through the system SQLite, one connection whose queries take turns: safe from any thread. */ +actual fun openReadOnlySqlite(path: String): SqliteReader = NativeSqliteReader(path) + +// SQLITE_TRANSIENT: SQLite copies the bound text (a C macro cinterop can't map) +private val TRANSIENT = (-1L).toCPointer Unit>>() + +private class NativeSqliteReader( + path: String, +) : SqliteReader { + private val db: CPointer = + memScoped { + val out = alloc>() + val status = sqlite3_open_v2(path, out.ptr, SQLITE_OPEN_READONLY or SQLITE_OPEN_FULLMUTEX, null) + val opened = out.value + if (status != SQLITE_OK || opened == null) { + val message = opened?.let { sqlite3_errmsg(it)?.toKString() } + sqlite3_close_v2(opened) + error("Can't open $path: ${message ?: status}") + } + opened + } + + // Queries take turns (the connection serializes them anyway): an error message is then the failing query's, and + // close never frees the connection under a running query + private val lock = SynchronizedObject() + private var closed = false + + override fun query( + sql: String, + args: List, + ): List> = + synchronized(lock) { + check(!closed) { "SQLite: the database is closed" } + queryLocked(sql, args) + } + + private fun queryLocked( + sql: String, + args: List, + ): List> = + memScoped { + val out = alloc>() + check(sqlite3_prepare_v2(db, sql, -1, out.ptr, null) == SQLITE_OK) { "SQLite: ${sqlite3_errmsg(db)?.toKString()}" } + val stmt = out.value!! + try { + args.forEachIndexed { i, arg -> sqlite3_bind_text(stmt, i + 1, arg, -1, TRANSIENT) } + val columns = (0 until sqlite3_column_count(stmt)).map { sqlite3_column_name(stmt, it)?.toKString() ?: "" } + val rows = ArrayList>() + var status = sqlite3_step(stmt) + while (status == SQLITE_ROW) { + rows += columns.indices.associate { columns[it] to sqlite3_column_text(stmt, it)?.reinterpret()?.toKString() } + status = sqlite3_step(stmt) + } + // Any other end is a failure (busy, I/O, corruption), not the last row: thrown, as the JVM and Android do + check(status == SQLITE_DONE) { "SQLite: ${sqlite3_errmsg(db)?.toKString()}" } + rows + } finally { + sqlite3_finalize(stmt) + } + } + + // Closing twice would free the connection twice + override fun close() { + synchronized(lock) { + if (closed) return + closed = true + sqlite3_close_v2(db) + } + } +} diff --git a/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt b/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt new file mode 100644 index 00000000..6ac8f7f8 --- /dev/null +++ b/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt @@ -0,0 +1,37 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import kotlinx.coroutines.runBlocking +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.writeString +import platform.Foundation.NSProcessInfo +import kotlin.time.measureTimedValue +import kotlin.test.Test + +/** + * The search on iOS, end to end in the simulator: Lucene in the embedded Java runtime (through NJA), everything else + * here. Dumps what it answers on a real index like RealIndexParityTest does on the JVM (SEFORIM_INDEX_DIR and + * SEFORIM_PARITY_OUT, passed by the iosSimulatorArm64Test task). Skipped without an index. + */ +class IosRealIndexParityTest { + @Test + fun dumpRealIndex() { + val env = NSProcessInfo.processInfo.environment + val indexDir = env["SEFORIM_INDEX_DIR"] as String? ?: return + val dictionary = indexDir.substringBeforeLast('/') + "/lexical.db" + val engine = + LexicalSearchEngine( + index = LuceneIndex(indexDir), + dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, dictionary) }, + ) + engine.use { + val (dump, took) = measureTimedValue { runBlocking { SearchDump.dump(it) } } + println("Parity dump in $took") + val out = Path(env["SEFORIM_PARITY_OUT"] as String) + SystemFileSystem.sink(out).buffered().use { sink -> sink.writeString(dump) } + println("Parity dump: $out") + } + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index bb21b9eb..97fbbd14 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -1,1176 +1,105 @@ package io.github.kdroidfilter.seforimlibrary.search import co.touchlab.kermit.Logger -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import org.apache.lucene.analysis.Analyzer -import org.apache.lucene.analysis.TokenStream -import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute -import org.apache.lucene.index.DocValuesType -import org.apache.lucene.index.FieldInfo -import org.apache.lucene.index.IndexReader -import org.apache.lucene.index.NumericDocValues -import org.apache.lucene.index.StoredFieldVisitor -import org.apache.lucene.index.StoredFields -import org.apache.lucene.index.Term -import org.apache.lucene.index.LeafReaderContext -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.BoostQuery -import org.apache.lucene.search.Collector -import org.apache.lucene.search.CollectorManager -import org.apache.lucene.search.FuzzyQuery -import org.apache.lucene.search.IndexSearcher -import org.apache.lucene.search.LeafCollector -import org.apache.lucene.search.PrefixQuery -import org.apache.lucene.search.PointRangeQuery -import org.apache.lucene.search.Query -import org.apache.lucene.search.QueryCachingPolicy -import org.apache.lucene.search.Scorable -import org.apache.lucene.search.ScoreDoc -import org.apache.lucene.search.ReferenceManager -import org.apache.lucene.search.ScoreMode -import org.apache.lucene.search.SearcherFactory -import org.apache.lucene.search.SearcherManager -import org.apache.lucene.search.TermQuery -import org.apache.lucene.util.QueryBuilder -import org.apache.lucene.store.FSDirectory -import org.apache.lucene.store.NIOFSDirectory -import org.apache.lucene.document.IntPoint +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex import java.nio.file.Path -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.atomic.AtomicInteger -/** - * Lucene-based implementation of SearchEngine for full-text search. - * Supports Hebrew text with diacritics handling, dictionary expansion, and fuzzy matching. - */ private val logger = Logger.withTag("LuceneSearchEngine") -// Hard cap on how many synonym/expansion terms we allow per token -private const val MAX_SYNONYM_TERMS_PER_TOKEN: Int = 32 -// Global cap for boost queries built from dictionary expansions -private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 -// Constants for snippet source building (must match indexer) -private const val SNIPPET_NEIGHBOR_WINDOW = 4 -private const val SNIPPET_MIN_LENGTH = 280 -private const val FIELD_BOOK_ID = "book_id" -private const val DEFAULT_NEAR = 5 -private const val PLAN_CACHE_SIZE = 16 -private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" -// Searches run by warmUp(): one and several words, common and rarer ones, a quoted phrase, the divine name (whose -// dictionary entry is the largest) -val WARMUP_QUERIES = listOf("ברוך ה׳ לעולם", "אמר רבי יוחנן", "תפילין", "\"קריאת שמע\"") -const val WARMUP_ROUNDS = 2 -const val WARMUP_PAGE_SIZE = 25 - -// Smallest partition worth a task of its own when a segment is split across cores -private const val MIN_DOCS_PER_SLICE = 100_000 - -/** - * Searchers that split the index, even a single segment, into one partition per core and search them concurrently: - * a merged index is one big segment, which a plain searcher scores on one thread. - */ -private object PartitionedSearcherFactory : SearcherFactory() { - private val cores = Runtime.getRuntime().availableProcessors() - private val executor: ExecutorService by lazy { - val ids = AtomicInteger() - Executors.newFixedThreadPool(cores) { task -> - Thread(task, "lucene-search-${ids.incrementAndGet()}").apply { isDaemon = true } - } - } - - override fun newSearcher(reader: IndexReader, previousReader: IndexReader?): IndexSearcher { - val searcher = if (cores == 1) { - IndexSearcher(reader) - } else { - val docsPerSlice = maxOf(MIN_DOCS_PER_SLICE, reader.maxDoc() / cores + 1) - object : IndexSearcher(reader, executor) { - override fun slices(leaves: List): Array = - slices(leaves, docsPerSlice, Int.MAX_VALUE, true) - } - } - return searcher.apply { queryCachingPolicy = StableFiltersCachingPolicy } - } -} - -/** - * Caches only the filters every search repeats (line documents, base books). Lucene's default policy also caches any - * clause seen twice, and each search builds its word filters twice (facets, then results): it would then turn them - * into bitsets over the whole index for a query that won't come back. - */ -private object StableFiltersCachingPolicy : QueryCachingPolicy { - override fun onUse(query: Query) = Unit - - override fun shouldCache(query: Query): Boolean = when (query) { - is TermQuery -> query.term.field() == "type" - is PointRangeQuery -> query.field == "is_base_book" - else -> false - } -} - -/** - * Blacklist of hallucinated dictionary mappings. - * Loaded from resources/hallucination_blacklist.tsv - * Key: normalized token, Value: set of incorrect base forms to reject - * - * TODO: This is a temporary workaround. The proper fix is to correct the - * hallucinated mappings directly in the lexical dictionary (lexical.db) - * so this blacklist becomes unnecessary. - */ -private val HALLUCINATION_BLACKLIST: Map> by lazy { - loadHallucinationBlacklist() -} - -private fun loadHallucinationBlacklist(): Map> { - val result = mutableMapOf>() - try { - val inputStream = LuceneSearchEngine::class.java.getResourceAsStream("/hallucination_blacklist.tsv") - if (inputStream == null) { - logger.w { "hallucination_blacklist.tsv not found in resources" } - return emptyMap() - } - inputStream.bufferedReader().useLines { lines -> - lines.forEach { line -> - val trimmed = line.trim() - if (trimmed.isEmpty() || trimmed.startsWith("#")) return@forEach - val parts = trimmed.split("\t") - if (parts.size >= 2) { - val token = HebrewTextUtils.normalizeHebrew(parts[0]) - val base = HebrewTextUtils.normalizeHebrew(parts[1]) - result.getOrPut(token) { mutableSetOf() }.add(base) - } - } - } - logger.d { "Loaded ${result.size} hallucination blacklist entries" } - } catch (e: Exception) { - logger.e(e) { "Failed to load hallucination blacklist" } - } - return result -} /** - * Check if an expansion should be rejected based on the hallucination blacklist. + * The desktop's [SearchEngine]: the common [LexicalSearchEngine] over the Lucene index at [indexDir] (the façade's, in + * this JVM), with the lexical dictionary found beside it (or at [dictionaryPath], `-DmagicDict`, `SEFORIM_MAGIC_DICT`). */ -private fun isHallucinatedExpansion(token: String, expansion: MagicDictionaryIndex.Expansion): Boolean { - val normalizedToken = HebrewTextUtils.normalizeHebrew(token) - val blacklistedBases = HALLUCINATION_BLACKLIST[normalizedToken] ?: return false - return expansion.base.any { base -> - val normalizedBase = HebrewTextUtils.normalizeHebrew(base) - blacklistedBases.contains(normalizedBase) - } -} - -class LuceneSearchEngine( - private val indexDir: Path, - private val snippetProvider: SnippetProvider? = null, - private val analyzer: Analyzer = StandardAnalyzer(), - private val dictionaryPath: Path? = null -) : SearchEngine { - - // Open Lucene directory lazily to avoid any I/O at app startup. - // GraalVM native image does not support MMapDirectory (Panama foreign downcalls), use NIOFSDirectory instead. - private val dir by lazy { - if (System.getProperty("org.graalvm.nativeimage.imagecode") != null) { - NIOFSDirectory(indexDir) - } else { - FSDirectory.open(indexDir) - } - } - - private val stdAnalyzer: Analyzer by lazy { analyzer } - private val magicDict: MagicDictionaryIndex? by lazy { - val candidates = listOfNotNull( - dictionaryPath, - System.getProperty("magicDict")?.let { Path.of(it) }, - System.getenv("SEFORIM_MAGIC_DICT")?.let { Path.of(it) }, - indexDir.resolveSibling("lexical.db"), - indexDir.resolveSibling("seforim.db").resolveSibling("lexical.db"), - Path.of("SeforimLibrary/SeforimMagicIndexer/magicindexer/build/db/lexical.db") - ).distinct() - val firstExisting = MagicDictionaryIndex.findValidDictionary(candidates) - if (firstExisting == null) { - logger.d { - "[MagicDictionary] Missing lexical.db; search will run without dictionary expansions. " + - "Provide -DmagicDict=/path/lexical.db or SEFORIM_MAGIC_DICT. Checked: " + - candidates.joinToString() - } - return@lazy null - } - logger.d { "[MagicDictionary] Loading lexical db from $firstExisting" } - val loaded = MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, firstExisting) - if (loaded == null) { - logger.d { - "[MagicDictionary] Failed to load lexical db at $firstExisting; " + - "continuing without dictionary expansions" - } - } - loaded - } - - // One reader shared by every search: opening one per search re-reads the term index of each segment and maps, - // then unmaps, all the files. Refreshed when the index changes on disk (delta updates write into it). - private val searcherManagerLazy = lazy { - SearcherManager(dir, PartitionedSearcherFactory).apply { - addListener( - object : ReferenceManager.RefreshListener { - override fun beforeRefresh() = Unit - - override fun afterRefresh(didRefresh: Boolean) { - if (didRefresh) bookAncestors.clear() - } - } - ) - } - } - - private val searcherManager: SearcherManager by searcherManagerLazy - - // Ancestor category ids per book, for the facets: every line of a book shares them - private val bookAncestors = ConcurrentHashMap() - - private fun acquireSearcher(): IndexSearcher { - val manager = searcherManager - manager.maybeRefresh() - return manager.acquire() - } - - private inline fun withSearcher(block: (IndexSearcher) -> T): T { - val searcher = acquireSearcher() - try { - return block(searcher) - } finally { - searcherManager.release(searcher) - } - } - - // --- SearchEngine interface implementation --- - - override fun openSession( - query: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean - ): SearchSession? { - val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) ?: return null - return LuceneSearchSession(context.query, context, acquireSearcher()) - } - - override suspend fun attachSnippets(hits: List, query: String, near: Int): List { - if (hits.isEmpty()) return hits - val context = buildSearchContext(query, near, null, null, null, null) ?: return hits - val sources = withContext(Dispatchers.IO) { snippetSources(hits.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) } - return hits.map { hit -> - val source = sources[hit.lineId] ?: SnippetSource(hit.rawText, 0, hit.rawText.length) - hit.copy( - snippet = SnippetBuilder.build(source.text, context.highlightPlan, source.lineStart, source.lineEnd), - rawText = source.text, - lineStart = source.lineStart, - lineEnd = source.lineEnd - ) - } - } - - override suspend fun warmUp() { - withContext(Dispatchers.IO) { hashemHighlightTerms } - // A few searches down the whole path (facets, ranking, snippets) load the index structures and, on the JVM, - // get the hot code compiled before the user's first search - repeat(WARMUP_ROUNDS) { - for (query in WARMUP_QUERIES) { - computeFacets(query, baseBookOnly = true) - openSession(query, baseBookOnly = true)?.use { it.nextPage(WARMUP_PAGE_SIZE) } - } - } - } - - override fun searchBooksByTitlePrefix(query: String, limit: Int): List { - val q = HebrewTextUtils.normalizeHebrew(query) - if (q.isBlank()) return emptyList() - val tokens = q.split("\\s+".toRegex()).map { it.trim() }.filter { it.isNotEmpty() } - if (tokens.isEmpty()) return emptyList() - - return withSearcher { searcher -> - val must = BooleanQuery.Builder() - // Restrict to book_title docs - must.add(TermQuery(Term("type", "book_title")), BooleanClause.Occur.FILTER) - tokens.forEach { tok -> - // prefix on analyzed 'title' - must.add(PrefixQuery(Term("title", tok)), BooleanClause.Occur.MUST) - } - val luceneQuery = must.build() - val top = searcher.search(luceneQuery, limit) - val stored: StoredFields = searcher.storedFields() - val ids = LinkedHashSet() - for (sd in top.scoreDocs) { - val doc = stored.document(sd.doc) - val id = doc.getField("book_id")?.numericValue()?.toLong() - if (id != null) ids.add(id) - } - ids.toList() - } - } - - override fun buildSnippet(rawText: String, query: String, near: Int): String = - SnippetBuilder.build(SnippetSources.clean(rawText), planFor(query, near)) - - override fun highlightRanges(text: String, query: String): List = - mergedRanges(text, planFor(query, DEFAULT_NEAR).highlights(text)) - - override fun rangeSnippet(text: String, ranges: List, lineStart: Int, lineEnd: Int): String = - SnippetBuilder.build( - text, - ranges.map { Occurrence(it.first, it.last + 1, 0, EvidenceLevel.LITERAL) }, - lineStart, - lineEnd - ) - - // The highlight plans of the latest queries: a results page and its preview ask for the same one many times - private val planCache = object : LinkedHashMap(PLAN_CACHE_SIZE, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = - size > PLAN_CACHE_SIZE - } - - private fun planFor(query: String, near: Int): HighlightPlan { - synchronized(planCache) { planCache[query]?.let { return it } } - val plan = buildSearchContext(query, near, null, null, null, null)?.highlightPlan ?: HighlightPlan.EMPTY - synchronized(planCache) { planCache[query] = plan } - return plan - } - - /** - * What a query highlights: each word with its inflections, from the dictionary entries it truly is a form of and - * that aren't blacklisted for it. With ה׳ in the query, the lone ה is the divine name. - */ - private fun buildHighlightPlan( - tokens: List, - hasHashem: Boolean, - exactPhrases: List - ): HighlightPlan { - val forms = tokens.distinct().map { token -> - if (hasHashem && highlightKey(token) == "ה") { - return@map TokenForms(token, extraLiterals = loadHashemHighlightTerms()) - } - val key = highlightKey(token) - val inflections = magicDict?.expansionsFor(listOf(token)).orEmpty() - .filter { exp -> exp.inflections.any { highlightKey(it) == key } && !isHallucinatedExpansion(token, exp) } - .flatMap { it.inflections } - TokenForms(token, inflections) - } - val phrases = exactPhrases - .map { analyzeToTerms(stdAnalyzer, it).orEmpty().joinToString(" ") } - .filter { it.isNotBlank() } - return HighlightPlan.build(forms, phrases) - } - - override fun findInBookCandidates(query: String, bookId: Long): LongArray? { - val norm = HebrewTextUtils.normalizeHebrew(query) - if (norm.isBlank()) return LongArray(0) - val bookLines = BooleanQuery.Builder() - .add(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER) - .add(IntPoint.newExactQuery("book_id", bookId.toInt()), BooleanClause.Occur.FILTER) - .build() - // Nothing to filter on: every line would be a candidate, cheaper scanned from the DB - val clauses = literalPresenceClauses(analyzeToTerms(stdAnalyzer, norm).orEmpty()) - if (clauses.isEmpty()) return null - val builder = BooleanQuery.Builder().add(bookLines, BooleanClause.Occur.FILTER) - clauses.forEach { builder.add(it, BooleanClause.Occur.FILTER) } - - return withSearcher { searcher -> - // Only line_id is read per hit: no scoring, no other stored field. One collector per slice, as the - // slices may run concurrently - class LineIdCollector : Collector { - var ids = LongArray(256) - var count = 0 - - override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { - val storedFields = leafContext.reader().storedFields() - return object : LeafCollector { - override fun setScorer(scorer: Scorable) = Unit - - override fun collect(doc: Int) { - val lineId = storedFields.document(doc, setOf("line_id")) - .getField("line_id")?.numericValue()?.toLong() ?: return - if (count == ids.size) ids = ids.copyOf(count * 2) - ids[count++] = lineId - } - } - } - - override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES - } - val ids = searcher.search( - builder.build(), - object : CollectorManager { - override fun newCollector() = LineIdCollector() - - override fun reduce(collectors: Collection): LongArray { - val out = LongArray(collectors.sumOf { it.count }) - var at = 0 - for (c in collectors) { - c.ids.copyInto(out, at, 0, c.count) - at += c.count - } - return out - } - } - ) - val count = ids.size - // No candidate may mean a book the index lacks (older index, book added later): unknown, not "no match" - if (count == 0 && searcher.count(bookLines) == 0) null else ids - } - } - - /** - * Filters every line holding the analyzed [tokens] as one literal substring satisfies. The - * substring may start and end mid-word: the first token is only a word suffix, the last a - * word prefix, the inner ones whole words. A first or lone token is matched anywhere in a - * word through its 4-grams, so one under 4 letters can't filter (a leading wildcard would - * scan the whole term dictionary); a last one under 3 neither, its prefix being too common. - */ - private fun literalPresenceClauses(tokens: List): List = - tokens.mapIndexedNotNull { i, token -> - when { - i in 1 until tokens.lastIndex -> TermQuery(Term("text", token)) - i > 0 && token.length >= 3 -> PrefixQuery(Term("text", token)) - else -> buildNgramPresenceForToken(token) - } - } - - override fun close() { - if (searcherManagerLazy.isInitialized()) runCatching { searcherManager.close() } - } - - override fun computeFacets( - query: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean - ): SearchFacets? { - val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) - ?: return null - - return withSearcher { searcher -> - // Only book ids are read per hit (doc values when the index has them, else the first stored field); - // category counts follow from the book counts, as every line of a book shares its ancestors. One - // collector per slice, as the slices may run concurrently - class FacetCollector : Collector { - val bookCounts = HashMap() - var totalHits = 0L - - override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { - val reader = leafContext.reader() - val storedFields = reader.storedFields() - val bookIdValues: NumericDocValues? = - reader.fieldInfos.fieldInfo(FIELD_BOOK_ID) - ?.takeIf { it.docValuesType == DocValuesType.NUMERIC } - ?.let { reader.getNumericDocValues(FIELD_BOOK_ID) } - // Lines of a book are mostly contiguous: count runs, then fold them into the map - var runBook = -1L - var runCount = 0 - - fun flushRun() { - if (runCount > 0) bookCounts.merge(runBook, runCount, Int::plus) - runCount = 0 - } - - return object : LeafCollector { - override fun setScorer(scorer: Scorable) = Unit - - override fun collect(doc: Int) { - totalHits++ - val bookId = - if (bookIdValues != null && bookIdValues.advanceExact(doc)) { - bookIdValues.longValue() - } else { - readStoredBookId(storedFields, doc) ?: return - } - if (bookId != runBook) { - flushRun() - runBook = bookId - if (!bookAncestors.containsKey(bookId)) loadAncestors(storedFields, doc, bookId) - } - runCount++ - } - - override fun finish() = flushRun() - } - } - - override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES - } - - searcher.search( - context.matchQuery, - object : CollectorManager { - override fun newCollector() = FacetCollector() - - override fun reduce(collectors: Collection): SearchFacets { - val bookCounts = HashMap() - collectors.forEach { c -> c.bookCounts.forEach { (book, n) -> bookCounts.merge(book, n, Int::plus) } } - val categoryCounts = HashMap() - for ((bookId, count) in bookCounts) { - bookAncestors[bookId]?.forEach { categoryCounts.merge(it, count, Int::plus) } - } - return SearchFacets( - totalHits = collectors.sumOf { it.totalHits }, - categoryCounts = categoryCounts, - bookCounts = bookCounts - ) - } - } - ) - } - } - - private fun readStoredBookId(storedFields: StoredFields, doc: Int): Long? { - var bookId: Long? = null - storedFields.document( - doc, - object : StoredFieldVisitor() { - override fun needsField(fieldInfo: FieldInfo): Status = when { - bookId != null -> Status.STOP - fieldInfo.name == FIELD_BOOK_ID -> Status.YES - else -> Status.NO - } - - override fun longField(fieldInfo: FieldInfo, value: Long) { - bookId = value - } - - override fun intField(fieldInfo: FieldInfo, value: Int) { - bookId = value.toLong() - } - } - ) - return bookId - } - - private fun loadAncestors(storedFields: StoredFields, doc: Int, bookId: Long) { - val ancestors = storedFields.document(doc, setOf(FIELD_ANCESTOR_CATEGORY_IDS)) - .getField(FIELD_ANCESTOR_CATEGORY_IDS)?.stringValue() - ?.split(",") - ?.mapNotNull { it.trim().toLongOrNull() } - .orEmpty() - // Lines added by delta updates lack the field: try again on the book's next run - if (ancestors.isNotEmpty()) bookAncestors[bookId] = ancestors.toLongArray() - } - - // --- Inner SearchSession class --- - - private inner class LuceneSearchSession( - private val query: Query, - private val context: SearchContext, - private val searcher: IndexSearcher - ) : SearchSession { - private var after: ScoreDoc? = null - private var finished = false - private var totalHitsValue: Long? = null - - override suspend fun nextPage(limit: Int, snippets: Boolean): SearchPage? = withContext(Dispatchers.IO) { - if (finished) return@withContext null - val top = searcher.searchAfter(after, query, limit) - if (totalHitsValue == null) totalHitsValue = top.totalHits?.value - if (top.scoreDocs.isEmpty()) { - finished = true - return@withContext null - } - val stored = searcher.storedFields() - val hits = mapScoreDocs(stored, top.scoreDocs.asList(), context, snippets) - after = top.scoreDocs.last() - val isLast = top.scoreDocs.size < limit - if (isLast) finished = true - SearchPage( - hits = hits, - totalHits = totalHitsValue ?: hits.size.toLong(), - isLastPage = isLast - ) - } - - override fun close() { - searcherManager.release(searcher) - } - } - - // --- Additional public search methods --- - - suspend fun searchAllText(rawQuery: String, near: Int = 5, limit: Int, offset: Int = 0): List = - doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = null) - - suspend fun searchInBook(rawQuery: String, near: Int, bookId: Long, limit: Int, offset: Int = 0): List = - doSearch(rawQuery, near, limit, offset, bookFilter = bookId, categoryFilter = null) - - suspend fun searchInCategory(rawQuery: String, near: Int, categoryId: Long, limit: Int, offset: Int = 0): List = - doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = categoryId) - - suspend fun searchInBooks(rawQuery: String, near: Int, bookIds: Collection, limit: Int, offset: Int = 0): List = - doSearchInBooks(rawQuery, near, limit, offset, bookIds) - - // --- Private implementation --- - - private class SearchContext( - val query: Query, - // The required clauses only: the same matches as [query], without the ranking-only clauses (fuzzy, n-grams, - // synonym boosts) that cost a rewrite but can't change the hit set - val matchQuery: Query, - private val plan: Lazy - ) { - val highlightPlan: HighlightPlan get() = plan.value - } - - private fun buildSearchContext( +class LuceneSearchEngine private constructor( + /** The Lucene index this engine reads, for the searches that share it (the semantic search's vectors). */ + val index: LuceneIndex, + private val engine: LexicalSearchEngine, +) : SearchEngine by engine { + private constructor( + index: LuceneIndex, + indexDir: Path, + snippetProvider: SnippetProvider?, + dictionaryPath: Path?, + ) : this( + index, + LexicalSearchEngine(index, snippetProvider, lazy { loadDictionary(indexDir, dictionaryPath) }), + ) + + constructor( + indexDir: Path, + snippetProvider: SnippetProvider? = null, + dictionaryPath: Path? = null, + ) : this(LuceneIndex(indexDir.toString()), indexDir, snippetProvider, dictionaryPath) + + suspend fun searchAllText( rawQuery: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean = false - ): SearchContext? { - // Split the raw query into exact phrases (wrapped in ASCII double quotes) and free text. - // Quoted segments are matched verbatim WITHOUT dictionary expansion (Google-style). - val parsed = SearchQueryParser.parse(rawQuery) - val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) - val exactPhrasesNorm = parsed.exactPhrases - .map { HebrewTextUtils.normalizeHebrew(it) } - .filter { it.isNotBlank() } - - if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null - - val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList() - - // Check if the original query contained ה׳ (Hashem) before normalization - val hasHashem = rawQuery.contains("ה׳") || rawQuery.contains("ה'") - - // Filter out single Hebrew letters and stop words BEFORE dictionary expansion - // BUT preserve "ה" if the original query had "ה׳" (Hashem) - val analyzedStd = analyzedRaw.filter { token -> - // Special case: if query has ה׳, keep "ה" token - if (token == "ה" && hasHashem) return@filter true - // Preserve numeric tokens (e.g., "6") so they can expand via MagicDictionary - if (token.any { it.isDigit() }) return@filter true - - token.length >= 2 && token !in setOf( - "א", "ב", "ג", "ד", "ה", "ו", "ז", "ח", "ט", "י", "כ", "ל", "מ", - "נ", "ס", "ע", "פ", "צ", "ק", "ר", "ש", "ת", - ) - } - - logger.d { "[DEBUG] Original query had Hashem (ה׳): $hasHashem" } - logger.d { "[DEBUG] Analyzed tokens: $analyzedStd" } - - // Get all possible expansions for each token (a token can belong to multiple bases) - // These expansions are used for SEARCH - we keep all of them for better recall - magicDict?.prefetch(analyzedStd) - val tokenExpansions: Map> = - analyzedStd.associateWith { token -> - // Get best expansion (prefers matching base, then largest) - val expansion = magicDict?.expansionFor(token) ?: return@associateWith emptyList() - listOf(expansion) - } - tokenExpansions.forEach { (token, exps) -> - exps.forEach { exp -> - logger.d { "[DEBUG] Token '$token' -> expansion: surface=${exp.surface.take(10)}..., variants=${exp.variants.take(10)}..., base=${exp.base}" } - } - } - - // What the snippets highlight: built on first use (facets need none) - val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhrasesNorm) } - - val builder = BooleanQuery.Builder() - // Mirrors every required clause of builder (the facets need only the hit set) - val matchBuilder = BooleanQuery.Builder() - fun require(query: Query, occur: BooleanClause.Occur) { - builder.add(query, occur) - matchBuilder.add(query, occur) - } - require(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER) - if (bookFilter != null) require(IntPoint.newExactQuery("book_id", bookFilter.toInt()), BooleanClause.Occur.FILTER) - if (categoryFilter != null) require(IntPoint.newExactQuery("category_id", categoryFilter.toInt()), BooleanClause.Occur.FILTER) - // Filter by base books only (is_base_book = 1) when baseBookOnly is true - if (baseBookOnly) require(IntPoint.newExactQuery("is_base_book", 1), BooleanClause.Occur.FILTER) - val bookIdsArray = bookIds?.map { it.toInt() }?.toIntArray() - if (bookIdsArray != null && bookIdsArray.isNotEmpty()) { - require(IntPoint.newSetQuery("book_id", *bookIdsArray), BooleanClause.Occur.FILTER) - } - val lineIdsArray = lineIds?.map { it.toInt() }?.toIntArray() - if (lineIdsArray != null && lineIdsArray.isNotEmpty()) { - require(IntPoint.newSetQuery("line_id", *lineIdsArray), BooleanClause.Occur.FILTER) - } - // Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase. - var exactClausesAdded = 0 - for (phrase in exactPhrasesNorm) { - val exactQuery = buildExactPhraseQuery(phrase) ?: continue - require(exactQuery, BooleanClause.Occur.MUST) - exactClausesAdded++ - logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" } - } - - // A fully-quoted query that produced no usable phrase clause (e.g. just punctuation) - // must not fall through to a match-all query. - if (norm.isBlank() && exactClausesAdded == 0) return null - - // Free (unquoted) text: dictionary-aware ranking + presence filtering. - if (norm.isNotBlank()) { - val rankedQuery = buildExpandedQuery(norm, near, analyzedStd, tokenExpansions) - val mustAllTokensQuery: Query? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions) - val phraseQuery: Query? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near) - if (mustAllTokensQuery != null) { - require(mustAllTokensQuery, BooleanClause.Occur.FILTER) - logger.d { "[DEBUG] Added mustAllTokensQuery as FILTER" } - } - if (phraseQuery != null && analyzedStd.size >= 2) { - if (near == 0) require(phraseQuery, BooleanClause.Occur.MUST) else builder.add(phraseQuery, BooleanClause.Occur.SHOULD) - logger.d { "[DEBUG] Added phraseQuery, near=$near" } - } - builder.add(rankedQuery, BooleanClause.Occur.SHOULD) - logger.d { "[DEBUG] Added rankedQuery as SHOULD" } - } - - val finalQuery = builder.build() - logger.d { "[DEBUG] Final query: $finalQuery" } - - return SearchContext( - query = finalQuery, - matchQuery = matchBuilder.build(), - plan = highlightPlan - ) - } - - // suspend because it calls snippetProvider.getSnippetSources(); - // callers are responsible for providing Dispatchers.IO context. - private suspend fun mapScoreDocs( - stored: StoredFields, - scoreDocs: List, - context: SearchContext, - snippets: Boolean = true - ): List { - if (scoreDocs.isEmpty()) return emptyList() - - // First pass: extract metadata from index - data class DocMeta( - val sd: ScoreDoc, - val bookId: Long, - val bookTitle: String, - val lineId: Long, - val lineIndex: Int, - val isBaseBook: Boolean, - val orderIndex: Int, - val indexedRaw: String // from text_raw field, may be empty if not stored - ) - - val docMetas = scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - DocMeta( - sd = sd, - bookId = doc.getField("book_id").numericValue().toLong(), - bookTitle = doc.getField("book_title").stringValue() ?: "", - lineId = doc.getField("line_id").numericValue().toLong(), - lineIndex = doc.getField("line_index").numericValue().toInt(), - isBaseBook = doc.getField("is_base_book")?.numericValue()?.toInt() == 1, - orderIndex = doc.getField("order_index")?.numericValue()?.toInt() ?: 999, - indexedRaw = doc.getField("text_raw")?.stringValue() ?: "" - ) - } - - val snippetSources: Map = - if (snippets) snippetSources(docMetas.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) else emptyMap() - - val hits = docMetas.map { meta -> - val source = snippetSources[meta.lineId] ?: SnippetSource(meta.indexedRaw, 0, meta.indexedRaw.length) - val raw = source.text - val baseScore = meta.sd.score - - // Calculate boost: lower orderIndex = higher boost (only for base books) - val boostedScore = if (meta.isBaseBook) { - // Formula: boost = baseScore * (1 + (120 - orderIndex) / 60) - // orderIndex 1 gets ~3x boost, orderIndex 50 gets ~2.2x boost, orderIndex 100+ gets ~1.3x boost - val boostFactor = 1.0f + (120 - meta.orderIndex).coerceAtLeast(0) / 60.0f - baseScore * boostFactor - } else { - baseScore - } - - val snippet = if (snippets) SnippetBuilder.build(raw, context.highlightPlan, source.lineStart, source.lineEnd) else "" - LineHit( - bookId = meta.bookId, - bookTitle = meta.bookTitle, - lineId = meta.lineId, - lineIndex = meta.lineIndex, - snippet = snippet, - score = boostedScore, - rawText = raw, - isBaseBook = meta.isBaseBook - ) - } - // Re-sort by boosted score (descending) - return hits.sortedByDescending { it.score } - } - - // Snippet sources: from the provider if available, otherwise none (the indexed text_raw fallback) - private suspend fun snippetSources(lines: List): Map = - snippetProvider?.getSnippetSources(lines) ?: emptyMap() + near: Int = 5, + limit: Int, + offset: Int = 0, + ): List = engine.searchAllText(rawQuery, near, limit, offset) - private suspend fun doSearch( + suspend fun searchInBook( rawQuery: String, near: Int, + bookId: Long, limit: Int, - offset: Int, - bookFilter: Long?, - categoryFilter: Long? - ): List { - val context = buildSearchContext(rawQuery, near, bookFilter, categoryFilter, null, null) ?: return emptyList() - return withContext(Dispatchers.IO) { - withSearcher { searcher -> - val top = searcher.search(context.query, offset + limit) - val stored: StoredFields = searcher.storedFields() - val sliced = top.scoreDocs.drop(offset) - mapScoreDocs(stored, sliced, context) - } - } - } + offset: Int = 0, + ): List = engine.searchInBook(rawQuery, near, bookId, limit, offset) - private suspend fun doSearchInBooks( + suspend fun searchInCategory( rawQuery: String, near: Int, + categoryId: Long, limit: Int, - offset: Int, - bookIds: Collection - ): List { - if (bookIds.isEmpty()) return emptyList() - val context = buildSearchContext(rawQuery, near, bookFilter = null, categoryFilter = null, bookIds = bookIds, lineIds = null) ?: return emptyList() - return withContext(Dispatchers.IO) { - withSearcher { searcher -> - val top = searcher.search(context.query, offset + limit) - val stored: StoredFields = searcher.storedFields() - val sliced = top.scoreDocs.drop(offset) - mapScoreDocs(stored, sliced, context) - } - } - } - - private fun analyzeToTerms(analyzer: Analyzer, text: String): List? = try { - val out = mutableListOf() - val ts: TokenStream = analyzer.tokenStream("text", text) - val termAtt = ts.addAttribute(CharTermAttribute::class.java) - ts.reset() - while (ts.incrementToken()) { - val t = termAtt.toString() - if (t.isNotBlank()) out += t - } - ts.end(); ts.close() - out - } catch (_: Exception) { null } - - private fun buildNgramPresenceForToken(token: String): Query? { - if (token.length < 4) return null - val grams = mutableListOf() - var i = 0 - val L = token.length - while (i + 4 <= L) { - grams += token.substring(i, i + 4) - i += 1 - } - if (grams.isEmpty()) return null - val b = BooleanQuery.Builder() - for (g in grams.distinct()) { - b.add(TermQuery(Term("text_ng4", g)), BooleanClause.Occur.MUST) - } - return b.build() - } + offset: Int = 0, + ): List = engine.searchInCategory(rawQuery, near, categoryId, limit, offset) - private fun buildPresenceFilterForTokens( - tokens: List, + suspend fun searchInBooks( + rawQuery: String, near: Int, - expansionsByToken: Map> - ): Query? { - if (tokens.isEmpty()) return null - val outer = BooleanQuery.Builder() - for (t in tokens) { - val expansions = expansionsByToken[t] ?: emptyList() - val synonymTerms = buildLimitedTermsForToken(t, expansions) - val ngram = if (near > 0) buildNgramPresenceForToken(t) else null - val clause = BooleanQuery.Builder().apply { - add(TermQuery(Term("text", t)), BooleanClause.Occur.SHOULD) - if (ngram != null) add(ngram, BooleanClause.Occur.SHOULD) - for (term in synonymTerms) { - if (term != t) { - add(TermQuery(Term("text", term)), BooleanClause.Occur.SHOULD) - } + bookIds: Collection, + limit: Int, + offset: Int = 0, + ): List = engine.searchInBooks(rawQuery, near, bookIds, limit, offset) + + private companion object { + fun loadDictionary( + indexDir: Path, + dictionaryPath: Path?, + ): MagicDictionaryIndex? { + val candidates = + listOfNotNull( + dictionaryPath, + System.getProperty("magicDict")?.let { Path.of(it) }, + System.getenv("SEFORIM_MAGIC_DICT")?.let { Path.of(it) }, + indexDir.resolveSibling("lexical.db"), + indexDir.resolveSibling("seforim.db").resolveSibling("lexical.db"), + Path.of("SeforimLibrary/SeforimMagicIndexer/magicindexer/build/db/lexical.db"), + ).distinct() + val firstExisting = MagicDictionaryIndex.findValidDictionary(candidates.map { it.toString() }) + if (firstExisting == null) { + logger.d { + "[MagicDictionary] Missing lexical.db; search will run without dictionary expansions. " + + "Provide -DmagicDict=/path/lexical.db or SEFORIM_MAGIC_DICT. Checked: " + + candidates.joinToString() } - }.build() - outer.add(clause, BooleanClause.Occur.MUST) - } - return outer.build() - } - - private fun buildHebrewStdQuery(norm: String, near: Int): Query { - val qb = QueryBuilder(stdAnalyzer) - val phrase = qb.createPhraseQuery("text", norm, near) - if (phrase != null) return phrase - val bool = qb.createBooleanQuery("text", norm, BooleanClause.Occur.MUST) - return bool ?: BooleanQuery.Builder().build() - } - - /** - * Builds a strict, verbatim phrase query for a quoted segment: terms must appear in the - * given order and be adjacent (slop = 0), with NO dictionary expansion, fuzzy matching or - * n-gram substring matching. A single-token segment becomes an exact [TermQuery]. - * - * Nikud and final letters are still normalised (the segment is normalised by the caller and - * tokenised through [stdAnalyzer]) so that the phrase matches the indexed form. - */ - private fun buildExactPhraseQuery(phraseNorm: String): Query? { - if (phraseNorm.isBlank()) return null - val qb = QueryBuilder(stdAnalyzer) - qb.createPhraseQuery("text", phraseNorm, 0)?.let { return it } - return qb.createBooleanQuery("text", phraseNorm, BooleanClause.Occur.MUST) - } - - private fun buildMagicBoostQuery(expansions: List): Query? { - if (expansions.isEmpty()) return null - val surfaceTerms = LinkedHashSet() - val variantTerms = LinkedHashSet() - val baseTerms = LinkedHashSet() - for (exp in expansions) { - surfaceTerms.addAll(exp.surface) - variantTerms.addAll(exp.variants) - baseTerms.addAll(exp.base) - } - - val limitedSurfaces = surfaceTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedVariants = variantTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedBases = baseTerms.take(MAX_SYNONYM_BOOST_TERMS) - if (surfaceTerms.size > limitedSurfaces.size || - variantTerms.size > limitedVariants.size || - baseTerms.size > limitedBases.size - ) { - logger.d { - "[DEBUG] Capped magic boost terms: " + - "surface=${surfaceTerms.size}->${limitedSurfaces.size}, " + - "variants=${variantTerms.size}->${limitedVariants.size}, " + - "base=${baseTerms.size}->${limitedBases.size}" - } - } - - val b = BooleanQuery.Builder() - for (s in limitedSurfaces) { - b.add(BoostQuery(TermQuery(Term("text", s)), 2.0f), BooleanClause.Occur.SHOULD) - } - for (v in limitedVariants) { - b.add(BoostQuery(TermQuery(Term("text", v)), 1.5f), BooleanClause.Occur.SHOULD) - } - for (ba in limitedBases) { - b.add(BoostQuery(TermQuery(Term("text", ba)), 1.0f), BooleanClause.Occur.SHOULD) - } - return b.build() - } - - private fun buildSynonymBoostQuery(expansions: List): Query? { - if (expansions.isEmpty()) return null - val surfaceTerms = LinkedHashSet() - val variantTerms = LinkedHashSet() - val baseTerms = LinkedHashSet() - for (exp in expansions) { - surfaceTerms.addAll(exp.surface) - variantTerms.addAll(exp.variants) - baseTerms.addAll(exp.base) - } - - val limitedSurfaces = surfaceTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedVariants = variantTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedBases = baseTerms.take(MAX_SYNONYM_BOOST_TERMS) - if (surfaceTerms.size > limitedSurfaces.size || - variantTerms.size > limitedVariants.size || - baseTerms.size > limitedBases.size - ) { - logger.d { - "[DEBUG] Capped synonym boost terms: " + - "surface=${surfaceTerms.size}->${limitedSurfaces.size}, " + - "variants=${variantTerms.size}->${limitedVariants.size}, " + - "base=${baseTerms.size}->${limitedBases.size}" + return null } - } - - val b = BooleanQuery.Builder() - for (s in limitedSurfaces) { - b.add(TermQuery(Term("text", s)), BooleanClause.Occur.SHOULD) - } - for (v in limitedVariants) { - b.add(TermQuery(Term("text", v)), BooleanClause.Occur.SHOULD) - } - for (ba in limitedBases) { - b.add(TermQuery(Term("text", ba)), BooleanClause.Occur.SHOULD) - } - return b.build() - } - - private fun buildSynonymPhrases( - tokens: List, - expansionsByToken: Map> - ): List> { - if (tokens.isEmpty()) return emptyList() - val termExpansions = buildTermAlternativesForTokens(tokens, expansionsByToken) - logger.d { "[DEBUG] buildSynonymPhrases - termExpansions sizes: ${termExpansions.map { it.size }}" } - fun buildMultiPhrase(slop: Int): Query { - val builder = org.apache.lucene.search.MultiPhraseQuery.Builder() - builder.setSlop(slop) - var pos = 0 - for (alts in termExpansions) { - builder.add(alts.map { Term("text", it) }.toTypedArray(), pos) - pos++ + logger.d { "[MagicDictionary] Loading lexical db from $firstExisting" } + return MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, firstExisting).also { + if (it == null) logger.d { "[MagicDictionary] Failed to load lexical db at $firstExisting; continuing without dictionary expansions" } } - return builder.build() } - return listOf( - buildMultiPhrase(0) to 50.0f, - buildMultiPhrase(3) to 20.0f, - buildMultiPhrase(8) to 5.0f - ) } +} - private fun buildSynonymPhraseQuery( - tokens: List, - expansionsByToken: Map>, - near: Int - ): Query? { - if (tokens.isEmpty()) return null - val termExpansions = buildTermAlternativesForTokens(tokens, expansionsByToken) - val builder = org.apache.lucene.search.MultiPhraseQuery.Builder() - builder.setSlop(near) - var position = 0 - for (alts in termExpansions) { - builder.add(alts.map { Term("text", it) }.toTypedArray(), position) - position++ - } - return builder.build() - } - - private fun buildNgram4Query(norm: String): Query? { - val tokens = norm.split("\\s+".toRegex()).map { it.trim() }.filter { it.length >= 4 } - if (tokens.isEmpty()) return null - val grams = mutableListOf() - for (t in tokens) { - val L = t.length - var i = 0 - while (i + 4 <= L) { - grams += t.substring(i, i + 4) - i += 1 - } - } - val uniq = grams.distinct() - if (uniq.isEmpty()) return null - val b = BooleanQuery.Builder() - for (g in uniq) { - b.add(TermQuery(Term("text_ng4", g)), BooleanClause.Occur.MUST) - } - return b.build() - } - - private fun buildExpandedQuery( - norm: String, - near: Int, - tokens: List, - expansionsByToken: Map> - ): Query { - val base = buildHebrewStdQuery(norm, near) - val allExpansions = expansionsByToken.values.flatten() - val synonymPhrases = buildSynonymPhrases(tokens, expansionsByToken) - val ngram = buildNgram4Query(norm) - val fuzzy = buildFuzzyQuery(norm, near) - val builder = BooleanQuery.Builder() - builder.add(base, BooleanClause.Occur.SHOULD) - for ((query, boost) in synonymPhrases) { - builder.add(BoostQuery(query, boost), BooleanClause.Occur.SHOULD) - } - if (ngram != null) builder.add(ngram, BooleanClause.Occur.SHOULD) - if (fuzzy != null) builder.add(fuzzy, BooleanClause.Occur.SHOULD) - val magic = buildMagicBoostQuery(allExpansions) - if (magic != null) builder.add(magic, BooleanClause.Occur.SHOULD) - val synonymBoost = buildSynonymBoostQuery(allExpansions) - if (synonymBoost != null) builder.add(synonymBoost, BooleanClause.Occur.SHOULD) - return builder.build() - } - - private fun buildFuzzyQuery(norm: String, near: Int): Query? { - if (near == 0) return null - if (norm.length < 4) return null - val tokens = analyzeToTerms(stdAnalyzer, norm)?.filter { it.length >= 4 } ?: emptyList() - if (tokens.isEmpty()) return null - val b = BooleanQuery.Builder() - for (t in tokens.distinct()) { - b.add(FuzzyQuery(Term("text", t), 1), BooleanClause.Occur.MUST) - } - return b.build() - } - - private fun buildLimitedTermsForToken( - token: String, - expansions: List - ): List { - if (expansions.isEmpty()) return listOf(token) - - val baseTerms = expansions.flatMap { it.base }.distinct() - val otherTerms = expansions.flatMap { it.surface + it.variants }.distinct() - - val ordered = LinkedHashSet() - if (token.isNotBlank()) { - ordered += token - } - baseTerms.forEach { ordered += it } - otherTerms.forEach { ordered += it } - - val totalSize = ordered.size - val limited = ordered.take(MAX_SYNONYM_TERMS_PER_TOKEN) - if (totalSize > limited.size) { - logger.d { - "[DEBUG] Capped synonym terms for token '$token' from $totalSize to ${limited.size}" - } - } - return limited - } - - private fun buildTermAlternativesForTokens( - tokens: List, - expansionsByToken: Map> - ): List> { - if (tokens.isEmpty()) return emptyList() - return tokens.map { token -> - val expansions = expansionsByToken[token] ?: emptyList() - buildLimitedTermsForToken(token, expansions) - } - } - - // The divine name's forms never change: read once from the dictionary, not on every search - private val hashemHighlightTerms: List by lazy { readHashemHighlightTerms() } - - private fun loadHashemHighlightTerms(): List = hashemHighlightTerms - - private fun readHashemHighlightTerms(): List { - val dict = magicDict ?: return emptyList() - val raw = dict.loadHashemSurfaces() - if (raw.isEmpty()) return emptyList() +/** [MagicDictionaryIndex.load] for a [Path]. */ +fun MagicDictionaryIndex.Companion.load( + norm: (String) -> String, + candidate: Path?, +): MagicDictionaryIndex? = load(norm, candidate?.toString()) - val terms = linkedSetOf() - raw.forEach { value -> - val trimmed = value.trim() - if (trimmed.isEmpty()) return@forEach - terms += trimmed - val stripped = HebrewTextUtils.stripDiacritics(trimmed).trim() - if (stripped.isNotEmpty()) terms += stripped - val normalized = HebrewTextUtils.normalizeHebrew(trimmed).trim() - if (normalized.isNotEmpty()) terms += normalized - } - - val out = terms.toList() - logger.d { "[DEBUG] Hashem highlight terms from lexical DB: ${out.take(20)}..." } - return out - } -} +/** [MagicDictionaryIndex.findValidDictionary] for [Path]s. */ +@JvmName("findValidDictionaryPath") +fun MagicDictionaryIndex.Companion.findValidDictionary(candidates: List): Path? = + findValidDictionary(candidates.map { it.toString() })?.let { Path.of(it) } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt new file mode 100644 index 00000000..63182d68 --- /dev/null +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt @@ -0,0 +1,65 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.util.concurrent.ConcurrentLinkedQueue + +/** Through JDBC (sqlite-jdbc), with a connection per thread: SQLite connections serialize their statements. */ +actual fun openReadOnlySqlite(path: String): SqliteReader = JdbcSqliteReader(path).also { it.query("SELECT 1") } + +private class JdbcSqliteReader( + path: String, +) : SqliteReader { + // A thread's connection, with its statements prepared once: the dictionary runs the same lookup for every word + private class ThreadConnection( + val connection: Connection, + ) { + val statements = HashMap() + } + + private val url = "jdbc:sqlite:${java.io.File(path).absolutePath}" + private val opened = ConcurrentLinkedQueue() + + @Volatile + private var closed = false + private val connection = + ThreadLocal.withInitial { + val connection = + DriverManager.getConnection(url).apply { + autoCommit = false + // Enforce read-only queries without altering connection flags post-open + createStatement().use { it.execute("PRAGMA query_only=ON") } + } + opened += connection + // Opened by a thread racing close(): closed here, as close() may already have drained the queue + if (closed) { + connection.close() + error("SQLite: the database is closed") + } + ThreadConnection(connection) + } + + override fun query( + sql: String, + args: List, + ): List> { + // A thread that never queried would otherwise open a connection no one closes + check(!closed) { "SQLite: the database is closed" } + val thread = connection.get() + val stmt = thread.statements.getOrPut(sql) { thread.connection.prepareStatement(sql) } + args.forEachIndexed { i, arg -> stmt.setString(i + 1, arg) } + return stmt.executeQuery().use { rs -> + val columns = (1..rs.metaData.columnCount).map { rs.metaData.getColumnLabel(it) } + val rows = ArrayList>() + while (rs.next()) rows += columns.associateWith { rs.getString(it) } + rows + } + } + + // Closing a connection closes its statements + override fun close() { + closed = true + generateSequence { opened.poll() }.forEach { runCatching { it.close() } } + } +} diff --git a/search/src/jvmMain/resources/hallucination_blacklist.tsv b/search/src/jvmMain/resources/hallucination_blacklist.tsv deleted file mode 100644 index dd146a14..00000000 --- a/search/src/jvmMain/resources/hallucination_blacklist.tsv +++ /dev/null @@ -1,29 +0,0 @@ -# Hallucination Blacklist -# Format: tokenbase -# Lines starting with # are comments -# These are LLM-generated mappings that are incorrect and should be ignored -לחתוכ כנ -לחתוך כנ -לבקש רצה -מתפלל בעה -לשבת נתנ -לעבוד עשה -להעמיד קומ -כנ כנ -בנ בנ -לשימ שמ -לשים שמ -לדונ יד -לדון יד -אמ אמ -להרגיע יושב -לחייכ חוה -לחייך חוה -שנ שנ -עצ עצ -דנ דנ -להיכנס עלה -לשקול לקח -לבנות בנ -לשחרר שרה -להתיר שרה diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt new file mode 100644 index 00000000..1e7d286f --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt @@ -0,0 +1,23 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.coroutines.runBlocking +import java.io.File +import java.nio.file.Path +import kotlin.test.Test + +/** + * Dumps what the engine answers on a real index (-PindexDir=) into build/parity/jvm.txt, for the + * parity with Android and iOS (see SearchDump). Skipped without an index. + */ +class RealIndexParityTest { + @Test + fun dumpRealIndex() { + val indexDir = System.getProperty("seforimIndexDir") ?: return + val out = File(System.getProperty("parityOut") ?: "build/parity/jvm.txt") + LuceneSearchEngine(Path.of(indexDir)).use { engine -> + out.parentFile.mkdirs() + out.writeText(runBlocking { SearchDump.dump(engine) }) + } + println("Parity dump: ${out.absolutePath}") + } +} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt new file mode 100644 index 00000000..4d9f37a6 --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt @@ -0,0 +1,52 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import java.nio.file.Files +import java.sql.DriverManager +import kotlin.concurrent.thread +import kotlin.io.path.deleteIfExists +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertIs + +class SqliteReaderTest { + private fun withDatabase(block: (String) -> Unit) { + val file = Files.createTempFile("reader", ".db") + try { + DriverManager.getConnection("jdbc:sqlite:$file").use { conn -> + conn.createStatement().use { stmt -> + stmt.execute("CREATE TABLE word (value TEXT, base TEXT)") + stmt.execute("INSERT INTO word VALUES ('שלום', 'שלם'), ('שלם', 'שלם'), ('תורה', 'ירה')") + } + } + block(file.toString()) + } finally { + file.deleteIfExists() + } + } + + @Test + fun `a statement run again binds its new arguments`() = + withDatabase { path -> + openReadOnlySqlite(path).use { db -> + val sql = "SELECT value FROM word WHERE base = ? ORDER BY value" + assertEquals(listOf("שלום", "שלם"), db.query(sql, listOf("שלם")).map { it["value"] }) + assertEquals(listOf("תורה"), db.query(sql, listOf("ירה")).map { it["value"] }) + assertEquals(emptyList(), db.query(sql, listOf("אין"))) + } + } + + @Test + fun `a closed reader refuses queries, from any thread`() = + withDatabase { path -> + val db = openReadOnlySqlite(path) + db.query("SELECT 1 AS one") + db.close() + assertFailsWith { db.query("SELECT 1 AS one") } + // A thread that never queried doesn't open a connection either + var failure: Throwable? = null + thread { failure = runCatching { db.query("SELECT 1 AS one") }.exceptionOrNull() }.join() + assertIs(failure) + db.close() + } +} diff --git a/search/src/nativeInterop/cinterop/sqlite3.def b/search/src/nativeInterop/cinterop/sqlite3.def new file mode 100644 index 00000000..f289e75f --- /dev/null +++ b/search/src/nativeInterop/cinterop/sqlite3.def @@ -0,0 +1,4 @@ +# The system SQLite of iOS, for the lexical dictionary +headers = sqlite3.h +package = io.github.kdroidfilter.seforimlibrary.search.sqlite3 +linkerOpts = -lsqlite3 diff --git a/settings.gradle.kts b/settings.gradle.kts index be36cb50..5af0e0fb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -67,8 +67,10 @@ dependencyResolutionManagement { } } include(":core") +include(":cache") include(":dao") include(":search") +include(":search-facade") include(":cli") include(":catalog") include(":searchindex")