From 55ee971d9300475b2906ae3f84804b27e03bd4fb Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 00:45:15 +0300 Subject: [PATCH 01/14] feat(search): Kotlin Multiplatform search for iOS and Android 15+ The search (query building, dictionary, ranking, snippets, highlights) moves to common code; Lucene is reached only through the LuceneIndex port (IndexQuery: plain data, cursors, facets, KNN). - core, dao: iOS targets; the repository and the catalog loader on kotlinx-io, cachemap and a portable PRAGMA helper (the native and Android drivers can't run them through execute). - search: JVM and Android run Lucene in process (LuceneIndexBackend, androidJvmMain); iOS runs it in an embedded Java runtime (OpenJDK's mobile port, Zero) through a JSON bridge over JNI (EmbeddedJvm, BridgedLuceneIndex, IndexBridgeServer). The runtime is built by kdroidFilter/openjdk-mobile's CI and downloaded by the build, its SHA-256 checked (see IOS_RUNTIME.md). - lucene-android: lucene-core with two classes patched for ART (Android 15+), checked against the upstream classes' SHA-256. - Parity tests on a real index (SearchDump): the same facets, snippets, highlights, titles and find-in-book candidates on the JVM, in the iOS simulator and on Android 15, but for Lucene's score noise. Signed-off-by: Elie Gambache --- IOS_RUNTIME.md | 97 ++ core/build.gradle.kts | 4 +- dao/build.gradle.kts | 10 +- .../seforimlibrary/dao/CatalogLoader.kt | 80 +- .../dao/repository/SeforimRepository.kt | 40 +- .../kdroidfilter/seforimlibrary/env/EnvIos.kt | 8 + gradle.properties | 1 + gradle/libs.versions.toml | 14 +- lucene-android/build.gradle.kts | 77 ++ .../vectorization/VectorizationProvider.java | 262 ++++ .../apache/lucene/util/HotspotVMOptions.java | 101 ++ search/build.gradle.kts | 215 ++- .../search/AndroidRealIndexParityTest.kt | 37 + .../search/Platform.androidJvm.kt | 10 + .../search/index/LuceneIndexBackend.kt | 578 ++++++++ .../seforimlibrary/search/Sqlite.android.kt | 25 + .../search/HallucinationBlacklist.kt | 56 + .../seforimlibrary/search/HebrewTextUtils.kt | 0 .../search/LexicalSearchEngine.kt | 711 ++++++++++ .../seforimlibrary/search/LruCache.kt | 48 + .../search/MagicDictionaryIndex.kt | 347 +++-- .../seforimlibrary/search/Platform.kt | 26 + .../seforimlibrary/search/QueryHighlighter.kt | 0 .../seforimlibrary/search/SearchEngine.kt | 3 +- .../search/SearchQueryParser.kt | 0 .../seforimlibrary/search/SearchSession.kt | 6 +- .../seforimlibrary/search/SnippetBuilder.kt | 0 .../seforimlibrary/search/SnippetProvider.kt | 5 +- .../seforimlibrary/search/index/IndexQuery.kt | 114 ++ .../search/index/LuceneIndex.kt | 97 ++ .../search/index/bridge/BridgeProtocol.kt | 117 ++ .../search/index/bridge/BridgedLuceneIndex.kt | 97 ++ .../seforimlibrary/search/SearchDump.kt | 72 + .../seforimlibrary/search/Platform.ios.kt | 11 + .../seforimlibrary/search/Sqlite.ios.kt | 80 ++ .../search/index/bridge/EmbeddedJvm.kt | 185 +++ .../search/index/bridge/JvmLuceneIndex.kt | 16 + .../search/IosRealIndexParityTest.kt | 41 + .../search/LuceneSearchEngine.kt | 1227 ++--------------- .../seforimlibrary/search/Sqlite.jvm.kt | 42 + .../search/index/bridge/IndexBridgeServer.kt | 82 ++ .../resources/hallucination_blacklist.tsv | 29 - .../search/RealIndexParityTest.kt | 23 + search/src/nativeInterop/cinterop/jni.def | 3 + search/src/nativeInterop/cinterop/sqlite3.def | 4 + settings.gradle.kts | 1 + 46 files changed, 3556 insertions(+), 1446 deletions(-) create mode 100644 IOS_RUNTIME.md create mode 100644 dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt create mode 100644 lucene-android/build.gradle.kts create mode 100644 lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java create mode 100644 lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java create mode 100644 search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt create mode 100644 search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt create mode 100644 search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt create mode 100644 search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt (100%) create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt (51%) create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt (100%) rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt (99%) rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt (100%) rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt (97%) rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt (100%) rename search/src/{jvmMain => commonMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt (96%) create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt create mode 100644 search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt create mode 100644 search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt create mode 100644 search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt create mode 100644 search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt create mode 100644 search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt create mode 100644 search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt create mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt create mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt delete mode 100644 search/src/jvmMain/resources/hallucination_blacklist.tsv create mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt create mode 100644 search/src/nativeInterop/cinterop/jni.def create mode 100644 search/src/nativeInterop/cinterop/sqlite3.def diff --git a/IOS_RUNTIME.md b/IOS_RUNTIME.md new file mode 100644 index 00000000..6623c542 --- /dev/null +++ b/IOS_RUNTIME.md @@ -0,0 +1,97 @@ +# The search on iOS: Lucene in an embedded Java runtime + +Lucene needs a JVM. On iOS, the app embeds one: OpenJDK's mobile port ([openjdk/mobile](https://github.com/openjdk/mobile)), +built as static libraries with the Zero VM (an interpreter, as iOS forbids generated code). Only the Lucene part of the +search runs there; everything else is Kotlin/Native. + +## Architecture + +``` +iOS app (Kotlin/Native) Java runtime in the same process (OpenJDK Zero) +─────────────────────── ───────────────────────────────────────────────── +LexicalSearchEngine, MagicDictionaryIndex (sqlite3), +snippets, highlights, hybrid fusion, semantic model +(ONNX Runtime C API), view models... + │ LuceneIndex + ▼ +JvmLuceneIndex ── JSON over JNI (EmbeddedJvm.call) ──▶ IndexBridgeServer ──▶ LuceneIndexBackend (Lucene 10) +``` + +- `search/src/commonMain/.../index/LuceneIndex.kt`: the only part of the search that runs Lucene: analysis, searches + (`IndexQuery`, plain data), cursors, facets, KNN. The JVM and Android implement it in process + (`LuceneIndexBackend`, `androidJvmMain`); iOS through the bridge (`iosMain/.../bridge`), with the same implementation + running in the embedded runtime. +- `EmbeddedJvm` starts the runtime (`JNI_CreateJavaVM`), attaches the calling threads, calls + `IndexBridgeServer.call(byte[]): byte[]` (UTF-8 JSON of `BridgeRequest` / `BridgeResponse`). +- Lucene runs unpatched there (Zero is HotSpot): no ART patch (`lucene-android`), `NIOFSDirectory` (no Panama mmap). + +Verified in the simulator on the real index (7M documents): the same facets, snippets, highlights, title suggestions +and find-in-book candidates as the desktop, and the same hits but for Lucene's score noise (see `SearchDump`); the whole +app search (Zayit's `searchkit`, hybrid with the semantic search) gives the same dump as the desktop. + +## The runtime + +The runtimes are built by the CI of [kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) +(branch `mobile-runtime`: openjdk/mobile at the tested commit, its fixes, `mobile-runtime/build.sh`) and published as +the assets of its `mobile-runtime-*` releases; the build downloads them and checks their SHA-256 (`search`'s +`javaRuntime` tasks: `mobileRuntimeRelease`, `mobileRuntimeSha256`, in `search/build/javaRuntime/`). +A new runtime is a new release there and its tag and checksums here. + +The fixes, for Zero on iOS: + +1. `os_bsd_zero.cpp`: the W^X state (`DefaultWXWriteMode`, `os::_jit_exec_enabled`, `Thread::wx_enable_write`...) the + shared code needs on Apple aarch64 is only defined for the C1/C2 port (`bsd_aarch64`): Zero gets its own, without page + protection switches (it generates no code). +2. `os_bsd.cpp`: `pd_commit_memory` asks for executable pages (the code cache), which iOS refuses without the JIT + entitlement; Zero never executes them, so they are committed read-write. + +`jdk-ios-simulator.tar.gz` and `jdk-ios-device.tar.gz` hold `static/` (`libjvm.a`, the JDK's native libraries, +`libffi.a`) and `lib/` (the runtime's `java.home`: `conf/`, `lib/`, `modules//...`). `make images` and +`make jmods` don't work for iOS (they link a dynamic libjvm): the runtime boots from an exploded image instead of a +`lib/modules` jimage. + +## Packaging it in the app + +**Linking** (the app target, or a test binary): + +``` +-force_load static/libjvm.a +-force_load static/{libjava,libnio,libnet,libzip,libjimage,libverify,libextnet,libfallbackLinker,libsyslookup}.a +static/libffi.a -export_dynamic -lz -lc++ -lsqlite3 +-framework Foundation -framework CoreFoundation -framework Security -framework SystemConfiguration +``` + +`-force_load` and `-export_dynamic` are required: the runtime finds its natives by name (`dlsym`: `JIMAGE_Open`, +`Java_java_lang_...`, `JNI_OnLoad_*`), which the linker would otherwise drop or hide. The semantic search adds ONNX +Runtime's `onnxruntime.xcframework` (`onnxruntime-c` 1.20.0, the version of the desktop and Android builds). + +**Files** — the VM sets its `java.home` to `/lib` itself (`-Djava.home` is ignored), so the app bundle +holds: + +``` +App.app/lib/{conf,lib,modules}/... (the runtime's lib/: java.base, java.logging, java.management, jdk.unsupported) +App.app/lib/app/*.jar (./gradlew :search:iosJvmClasspath: search-jvm, lucene-core, kotlin-stdlib, kotlinx-serialization) +``` + +About 50 MB uncompressed. Zayit's `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/` and `lib/` +(with the jars); `searchkit`'s `IosJavaRuntime` finds the jars in `lib/app` by default. + +## Tests + +```bash +./gradlew :search:iosSimulatorArm64Test -PindexDir= -PluceneQueryCache=off +./gradlew :search:jvmTest --tests '*RealIndexParityTest*' -PindexDir= -PluceneQueryCache=off +``` + +They write `search/build/parity/ios.txt` and `jvm.txt` (`SearchDump`). `-PluceneQueryCache=off` turns off Lucene's +query cache: it changes the last digits of the scores from one search to the next. Two runs on the same platform differ +anyway by Lucene's process-dependent clause order (`BooleanQuery` rewrites through a `HashMap` keyed by class identity +hashes), so the comparison allows score differences: facets, snippets and highlights must be identical. + +## Android + +Android 15+ (API 35) only: ART runs Lucene in process, with two classes patched (`lucene-android`: a guard in +`VectorizationProvider.lookup` and in `HotspotVMOptions`, checked against the upstream classes' SHA-256 so a Lucene bump +can't ship a stale patch). Before 15, ART lacks the Java 21 APIs Lucene 10 needs, and a Java runtime beside the app +would cost those older devices too much: they aren't supported. `:search:androidDeviceTest` runs the dump on a device +or an emulator. diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 56376722..a3edcbef 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -12,11 +12,13 @@ kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) android { - namespace = "io.github.kdroidfilter.seforimlibrary" + namespace = "io.github.kdroidfilter.seforimlibrary.core" compileSdk = 35 minSdk = 21 } jvm() + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { diff --git a/dao/build.gradle.kts b/dao/build.gradle.kts index c8e42a0a..8dda76f1 100644 --- a/dao/build.gradle.kts +++ b/dao/build.gradle.kts @@ -14,11 +14,13 @@ kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) android { - namespace = "io.github.kdroidfilter.seforimlibrary" + namespace = "io.github.kdroidfilter.seforimlibrary.dao" compileSdk = 35 minSdk = 21 } jvm() + iosArm64() + iosSimulatorArm64() sourceSets { commonMain.dependencies { @@ -28,6 +30,8 @@ kotlin { implementation(libs.kotlinx.serialization.protobuf) implementation(libs.kotlinx.datetime) implementation(libs.kermit) + implementation(libs.kotlinx.io.core) + implementation(libs.cachemap) } commonTest.dependencies { @@ -48,6 +52,10 @@ kotlin { implementation(libs.sqlDelight.driver.sqlite) } + iosMain.dependencies { + implementation(libs.sqlDelight.driver.native) + } + } } diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt index cd9990a3..94eb1e11 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/CatalogLoader.kt @@ -1,18 +1,19 @@ package io.github.kdroidfilter.seforimlibrary.dao import io.github.kdroidfilter.seforimlibrary.core.models.PrecomputedCatalog +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.readByteArray import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.protobuf.ProtoBuf -import java.io.File -import java.nio.file.Path -import kotlin.io.path.exists -import kotlin.io.path.readBytes /** * Loader for the precomputed catalog tree. * Provides fast loading of the entire book/category tree from a binary file. */ object CatalogLoader { + private const val CATALOG_FILE = "catalog.pb" /** * Loads the precomputed catalog from a file next to the database. @@ -20,53 +21,15 @@ object CatalogLoader { * @param dbPath Path to the database file. The catalog file should be in the same directory. * @return The precomputed catalog, or null if the file doesn't exist or can't be loaded */ - @OptIn(ExperimentalSerializationApi::class) - fun loadCatalog(dbPath: String): PrecomputedCatalog? { - return try { - val dbFile = File(dbPath) - val catalogFile = File(dbFile.parentFile, "catalog.pb") - - if (!catalogFile.exists()) { - println("Catalog file not found: ${catalogFile.absolutePath}") - return null - } - - val bytes = catalogFile.readBytes() - val catalog = ProtoBuf.decodeFromByteArray(PrecomputedCatalog.serializer(), bytes) - - println("Loaded precomputed catalog: ${catalog.totalCategories} categories, ${catalog.totalBooks} books") - catalog - } catch (e: Exception) { - println("Failed to load precomputed catalog: ${e.message}") - e.printStackTrace() - null - } - } + fun loadCatalog(dbPath: String): PrecomputedCatalog? = load(catalogBeside(dbPath)) /** - * Loads the precomputed catalog from a specific Path. + * Loads the precomputed catalog from a specific path. * * @param catalogPath Direct path to the catalog.pb file * @return The precomputed catalog, or null if the file doesn't exist or can't be loaded */ - fun loadCatalogFromPath(catalogPath: Path): PrecomputedCatalog? { - return try { - if (!catalogPath.exists()) { - println("Catalog file not found: $catalogPath") - return null - } - - val bytes = catalogPath.readBytes() - val catalog = ProtoBuf.decodeFromByteArray(PrecomputedCatalog.serializer(), bytes) - - println("Loaded precomputed catalog: ${catalog.totalCategories} categories, ${catalog.totalBooks} books") - catalog - } catch (e: Exception) { - println("Failed to load precomputed catalog: ${e.message}") - e.printStackTrace() - null - } - } + fun loadCatalogFromPath(catalogPath: String): PrecomputedCatalog? = load(Path(catalogPath)) /** * Checks if a catalog file exists next to the database. @@ -74,9 +37,28 @@ object CatalogLoader { * @param dbPath Path to the database file * @return true if catalog.pb exists, false otherwise */ - fun catalogExists(dbPath: String): Boolean { - val dbFile = File(dbPath) - val catalogFile = File(dbFile.parentFile, "catalog.pb") - return catalogFile.exists() + fun catalogExists(dbPath: String): Boolean = SystemFileSystem.exists(catalogBeside(dbPath)) + + private fun catalogBeside(dbPath: String): Path { + val parent = Path(dbPath).parent + return if (parent != null) Path(parent, CATALOG_FILE) else Path(CATALOG_FILE) } + + @OptIn(ExperimentalSerializationApi::class) + private fun load(catalogFile: Path): PrecomputedCatalog? = + try { + if (!SystemFileSystem.exists(catalogFile)) { + println("Catalog file not found: $catalogFile") + null + } else { + val bytes = SystemFileSystem.source(catalogFile).buffered().use { it.readByteArray() } + ProtoBuf.decodeFromByteArray(PrecomputedCatalog.serializer(), bytes).also { + println("Loaded precomputed catalog: ${it.totalCategories} categories, ${it.totalBooks} books") + } + } + } catch (e: Exception) { + println("Failed to load precomputed catalog: ${e.message}") + e.printStackTrace() + null + } } diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt index 75136b5e..b23ccf45 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt @@ -25,7 +25,9 @@ import io.github.kdroidfilter.seforimlibrary.core.models.Topic import io.github.kdroidfilter.seforimlibrary.dao.extensions.toModel import io.github.kdroidfilter.seforimlibrary.db.SeforimDb import io.github.kdroidfilter.seforimlibrary.env.getEnvironmentVariable +import io.github.charlietap.cachemap.cacheMapOf import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json @@ -44,19 +46,18 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L // Category rows are immutable at runtime, so a plain read-through cache avoids // paying a SQLite prepare + connection round-trip for every breadcrumb/tree walk. // Profiling (see JFR 2026-04-23) showed 70 `sqlite3_prepare` calls in 20 s all - // coming from `getCategory`. Both KMP targets (JVM + Android) are JVM-based so - // `ConcurrentHashMap` is safe to use directly from commonMain. - private val categoryCache = java.util.concurrent.ConcurrentHashMap() + // coming from `getCategory`. + private val categoryCache = cacheMapOf() // book.orderIndex is denormalized onto each link row at insert time (targetBookOrderIndex) // so the commentaries path can sort without JOINing book. Books are immutable once inserted // and exist before any link references them, so a read-through cache is safe during import. - private val bookOrderIndexCache = java.util.concurrent.ConcurrentHashMap() + private val bookOrderIndexCache = cacheMapOf() private fun resolveBookOrderIndex(bookId: Long): Long = - bookOrderIndexCache.getOrPut(bookId) { - database.bookQueriesQueries.selectOrderIndexById(bookId).executeAsOneOrNull() ?: 0L - } + bookOrderIndexCache[bookId] + ?: (database.bookQueriesQueries.selectOrderIndexById(bookId).executeAsOneOrNull() ?: 0L) + .also { bookOrderIndexCache[bookId] = it } init { val repositoryLoggingEnv = getEnvironmentVariable("SEFORIMAPP_REPOSITORY_LOGGING")?.lowercase() @@ -69,11 +70,11 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L // Create the database schema (fresh builds only; no runtime migrations needed) SeforimDb.Schema.create(driver) // SQLite optimizations (balanced mode for desktop: 256MB cache + 512MB mmap) - driver.execute(null, "PRAGMA journal_mode=WAL", 0) - driver.execute(null, "PRAGMA synchronous=NORMAL", 0) - driver.execute(null, "PRAGMA cache_size=-256000", 0) // 256MB cache (negative = KiB) - driver.execute(null, "PRAGMA mmap_size=536870912", 0) // 512MB memory-mapped I/O - driver.execute(null, "PRAGMA temp_store=MEMORY", 0) + pragma("journal_mode=WAL") + pragma("synchronous=NORMAL") + pragma("cache_size=-256000") // 256MB cache (negative = KiB) + pragma("mmap_size=536870912") // 512MB memory-mapped I/O + pragma("temp_store=MEMORY") // Check if the database is empty try { @@ -96,6 +97,17 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L + /** + * Sets a PRAGMA on every driver: some answer with a row (journal_mode, mmap_size), which the Android and native + * drivers only accept as a query, and the JDBC one only as a statement. + */ + private fun pragma(assignment: String) { + val sql = "PRAGMA $assignment" + runCatching { driver.execute(null, sql, 0) } + .recoverCatching { driver.executeQuery(null, sql, { cursor -> QueryResult.Value(cursor.next().value) }, 0) } + .onFailure { logger.w(it) { "Couldn't set $sql" } } + } + // --- Line ⇄ TOC mapping --- /** @@ -275,7 +287,9 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L val loaded = withContext(Dispatchers.IO) { database.categoryQueriesQueries.selectById(id).executeAsOneOrNull()?.toModel() } ?: return null - return categoryCache.putIfAbsent(id, loaded) ?: loaded + // Rows are immutable: a concurrent load writes the same value + categoryCache[id] = loaded + return loaded } /** diff --git a/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt b/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt new file mode 100644 index 00000000..134d1b20 --- /dev/null +++ b/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/env/EnvIos.kt @@ -0,0 +1,8 @@ +package io.github.kdroidfilter.seforimlibrary.env + +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.toKString +import platform.posix.getenv + +@OptIn(ExperimentalForeignApi::class) +actual fun getEnvironmentVariable(name: String): String? = getenv(name)?.toKString() diff --git a/gradle.properties b/gradle.properties index a6460d48..e0b5e18d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,3 +14,4 @@ kotlin.incremental.wasm=true #Android android.useAndroidX=true android.nonTransitiveRClass=true +kotlin.mpp.enableCInteropCommonization=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index cb3b201b..f45cb89d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,7 +1,7 @@ [versions] filekitCore = "0.13.0" -kotlin = "2.3.20" +kotlin = "2.4.20" agp = "9.1.1" jvmToolchain = "25" lucene = "10.5.2" @@ -12,6 +12,12 @@ kotlinx-coroutines = "1.10.2" kotlinx-serialization = "1.10.0" kotlinx-datetime = "0.7.1" kermit = "2.1.0" +kotlinx-io = "0.9.1" +cachemap = "0.3.0" +atomicfu = "0.33.0" +androidx-test-runner = "1.7.0" +androidx-test-junit = "1.3.0" +ksoup = "0.2.6" sqlDelight = "2.3.2" compose = "1.10.2" androidx-activityCompose = "1.13.0" @@ -32,6 +38,12 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa kotlinx-serialization-protobuf = { module = "org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref = "kotlinx-serialization" } kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime" } kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } +kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } +cachemap = { module = "io.github.charlietap:cachemap", version.ref = "cachemap" } +atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "atomicfu" } +androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } +androidx-test-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-junit" } +ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.ref = "lucene" } lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "lucene" } lucene-analysis-common = { module = "org.apache.lucene:lucene-analysis-common", version.ref = "lucene" } diff --git a/lucene-android/build.gradle.kts b/lucene-android/build.gradle.kts new file mode 100644 index 00000000..a58bab66 --- /dev/null +++ b/lucene-android/build.gradle.kts @@ -0,0 +1,77 @@ +import java.security.MessageDigest +import java.util.zip.ZipFile +import javax.inject.Inject + +// lucene-core for Android (ART, API 35+): the upstream jar with two classes patched, as Lucene 10 assumes a HotSpot JVM +// in them (Runtime.version(), modules). The patched sources are the upstream ones plus a guard, marked "SeforimLibrary". +plugins { + `java-library` +} + +group = "io.github.kdroidfilter.seforimlibrary" + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(libs.versions.jvmToolchain.get().toInt())) +} + +tasks.withType().configureEach { + // Lucene's own level + options.release.set(21) +} + +val upstream: Configuration by configurations.creating { + isTransitive = false +} + +dependencies { + compileOnly(libs.lucene.core) + upstream(libs.lucene.core) +} + +// The upstream classes the patch replaces, by SHA-256: a Lucene bump that changes them fails the build, so the patch +// is diffed against the new sources and redone instead of shipping stale code +val replaced = + mapOf( + "org/apache/lucene/internal/vectorization/VectorizationProvider.class" to "317cbaed36506e22e6cfaec16eb2e1612465b55bb36d06125fd5b04e72d9e21c", + "org/apache/lucene/internal/vectorization/VectorizationProvider\$Holder.class" to "f416745bad90bfde50eaee0e21ec6ec3763e762114914700a8ad0915f54ff023", + "org/apache/lucene/util/HotspotVMOptions.class" to "beb595fb64a8cc73b602976d77ac2f84b2935b113305117074440ac706a301a3", + ) + +val checkUpstream by tasks.registering { + val jars: FileCollection = files(upstream) + val expected = replaced + inputs.files(jars) + doLast { + ZipFile(jars.singleFile).use { zip -> + for ((entry, sha) in expected) { + val bytes = zip.getInputStream(zip.getEntry(entry) ?: error("$entry is gone from ${jars.singleFile.name}")).readBytes() + val actual = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + check(actual == sha) { "$entry changed in ${jars.singleFile.name}: redo the patch of lucene-android on its new sources" } + } + } + } +} + +// ArchiveOperations, not the script's zipTree: the configuration cache can't keep references to the script +interface ArchiveAccess { + @get:Inject + val archives: ArchiveOperations +} +val archiveAccess = objects.newInstance() + +tasks.jar { + dependsOn(checkUpstream) + // The patched classes (compiled here) replace the upstream ones; everything else, services included, is upstream + val access = archiveAccess + from(upstream.elements.map { jars -> jars.map { access.archives.zipTree(it.asFile) } }) { + exclude(replaced.keys) + exclude("META-INF/MANIFEST.MF") + } + duplicatesStrategy = DuplicatesStrategy.FAIL +} + +// Consumers get the merged jar only: the secondary variants (classes, resources directories) would hand them the two +// patched classes without the rest of Lucene +listOf("apiElements", "runtimeElements").forEach { name -> + configurations.named(name) { outgoing.variants.clear() } +} diff --git a/lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java new file mode 100644 index 00000000..8d5f7dbb --- /dev/null +++ b/lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.internal.vectorization; + +import java.io.IOException; +import java.lang.StackWalker.StackFrame; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Stream; +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Constants; +import org.apache.lucene.util.VectorUtil; + +/** + * A provider of vectorization implementations. Depending on the Java version and availability of + * vectorization modules in the Java runtime this class provides optimized implementations (using + * SIMD) of several algorithms used throughout Apache Lucene. + * + *

Expert: set the {@value #UPPER_JAVA_FEATURE_VERSION_SYSPROP} system property to increase the + * set of Java versions this class will provide optimized implementations for. + * + * @lucene.internal + */ +public abstract class VectorizationProvider { + + static final OptionalInt TESTS_VECTOR_SIZE; + static final boolean TESTS_FORCE_INTEGER_VECTORS; + static final int UPPER_JAVA_FEATURE_VERSION = getUpperJavaFeatureVersion(); + + static { + var vs = OptionalInt.empty(); + try { + vs = + Stream.ofNullable(System.getProperty("tests.vectorsize")) + .filter(Predicate.not(Set.of("", "default")::contains)) + .mapToInt(Integer::parseInt) + .findAny(); + } catch ( + @SuppressWarnings("unused") + SecurityException se) { + // ignored + } + TESTS_VECTOR_SIZE = vs; + + boolean enforce = false; + try { + enforce = Boolean.getBoolean("tests.forceintegervectors"); + } catch ( + @SuppressWarnings("unused") + SecurityException se) { + // ignored + } + TESTS_FORCE_INTEGER_VECTORS = enforce; + } + + private static final String UPPER_JAVA_FEATURE_VERSION_SYSPROP = + "org.apache.lucene.vectorization.upperJavaFeatureVersion"; + private static final int DEFAULT_UPPER_JAVA_FEATURE_VERSION = 26; + + private static int getUpperJavaFeatureVersion() { + int runtimeVersion = DEFAULT_UPPER_JAVA_FEATURE_VERSION; + try { + String str = System.getProperty(UPPER_JAVA_FEATURE_VERSION_SYSPROP); + if (str != null) { + runtimeVersion = Math.max(Integer.parseInt(str), runtimeVersion); + } + } catch (@SuppressWarnings("unused") NumberFormatException | SecurityException ignored) { + Logger.getLogger(VectorizationProvider.class.getName()) + .warning( + "Cannot read sysprop " + + UPPER_JAVA_FEATURE_VERSION_SYSPROP + + ", so the default value will be used."); + } + return runtimeVersion; + } + + /** + * Returns the default instance of the provider matching vectorization possibilities of actual + * runtime. + * + * @throws IllegalCallerException if the singleton getter is not called from known Lucene classes. + */ + public static VectorizationProvider getInstance() { + ensureCaller(); + return Objects.requireNonNull( + Holder.INSTANCE, "call to getInstance() from subclass of VectorizationProvider"); + } + + VectorizationProvider() { + // no instance/subclass except from this package + } + + /** + * Returns a singleton (stateless) {@link VectorUtilSupport} to support SIMD usage in {@link + * VectorUtil}. + */ + public abstract VectorUtilSupport getVectorUtilSupport(); + + /** Returns a FlatVectorsScorer that supports the Lucene99 format. */ + public abstract FlatVectorsScorer getLucene99FlatVectorsScorer(); + + /** Returns a FlatVectorsScorer that supports the Lucene99 format. */ + public abstract FlatVectorsScorer getLucene99ScalarQuantizedVectorsScorer(); + + /** Create a new {@link PostingDecodingUtil} for the given {@link IndexInput}. */ + public abstract PostingDecodingUtil newPostingDecodingUtil(IndexInput input) throws IOException; + + /** Returns a {@link DocValuesRangeSupport} for SIMD-accelerated range evaluation. */ + public DocValuesRangeSupport getDocValuesRangeSupport() { + return DefaultDocValuesRangeSupport.INSTANCE; + } + + /** + * Returns a {@link DocValuesBulkDecodeSupport} instance for bulk numeric value decode. The + * returned instance uses SIMD when available (Panama Vector API), falling back to a scalar loop + * otherwise. + */ + public abstract DocValuesBulkDecodeSupport getDocValuesBulkDecodeSupport(); + + // *** Lookup mechanism: *** + + private static final Logger LOG = Logger.getLogger(VectorizationProvider.class.getName()); + + // visible for tests + static VectorizationProvider lookup(boolean testMode) { + // SeforimLibrary (Android): ART has no Runtime.version() nor Panama; the scalar provider is the only one there + if (!Constants.IS_HOTSPOT_VM) { + return new DefaultVectorizationProvider(); + } + final int runtimeVersion = Runtime.version().feature(); + assert runtimeVersion >= 21; + if (runtimeVersion <= UPPER_JAVA_FEATURE_VERSION) { + // only use vector module with Hotspot VM + if (!Constants.IS_HOTSPOT_VM) { + LOG.warning( + "Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled."); + return new DefaultVectorizationProvider(); + } + // don't use vector module with JVMCI (it does not work) + if (Constants.IS_JVMCI_VM) { + LOG.warning( + "Java runtime is using JVMCI Compiler; Java vector incubator API can't be enabled."); + return new DefaultVectorizationProvider(); + } + // is the incubator module present and readable (JVM providers may to exclude them or it is + // build with jlink) + final var vectorMod = lookupVectorModule(); + if (vectorMod.isEmpty()) { + LOG.warning( + "Java vector incubator module is not readable. For optimal vector performance, pass '--add-modules jdk.incubator.vector' to enable Vector API."); + return new DefaultVectorizationProvider(); + } + vectorMod.ifPresent(VectorizationProvider.class.getModule()::addReads); + // check for testMode and otherwise fallback to default if slowness could happen + if (!testMode) { + if (TESTS_VECTOR_SIZE.isPresent() || TESTS_FORCE_INTEGER_VECTORS) { + LOG.warning( + "Vector bitsize and/or integer vectors enforcement; using default vectorization provider outside of testMode"); + return new DefaultVectorizationProvider(); + } + if (Constants.IS_CLIENT_VM) { + LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled"); + return new DefaultVectorizationProvider(); + } + } + try { + // we use method handles with lookup, so we do not need to deal with setAccessible as we + // have private access through the lookup: + final var lookup = MethodHandles.lookup(); + final var cls = + lookup.findClass( + "org.apache.lucene.internal.vectorization.PanamaVectorizationProvider"); + final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class)); + try { + return (VectorizationProvider) constr.invoke(); + } catch (UnsupportedOperationException uoe) { + // not supported because preferred vector size too small or similar + LOG.warning("Java vector incubator API was not enabled. " + uoe.getMessage()); + return new DefaultVectorizationProvider(); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable th) { + throw new AssertionError(th); + } + } catch (NoSuchMethodException | IllegalAccessException e) { + throw new LinkageError( + "PanamaVectorizationProvider is missing correctly typed constructor", e); + } catch (ClassNotFoundException cnfe) { + throw new LinkageError("PanamaVectorizationProvider is missing in Lucene JAR file", cnfe); + } + } else { + LOG.warning( + "You are running with unsupported Java " + + runtimeVersion + + ". To make full use of the Vector API, please update Apache Lucene."); + } + return new DefaultVectorizationProvider(); + } + + /** + * Looks up the vector module from Lucene's {@link ModuleLayer} or the root layer (if unnamed). + */ + private static Optional lookupVectorModule() { + return Optional.ofNullable(VectorizationProvider.class.getModule().getLayer()) + .orElse(ModuleLayer.boot()) + .findModule("jdk.incubator.vector"); + } + + // add all possible callers here as FQCN: + private static final Set VALID_CALLERS = + Set.of( + "org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil", + "org.apache.lucene.util.VectorUtil", + "org.apache.lucene.codecs.lucene104.Lucene104PostingsReader", + "org.apache.lucene.codecs.lucene104.PostingIndexInput", + "org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer", + "org.apache.lucene.tests.util.TestSysoutsLimits"); + + private static void ensureCaller() { + final boolean validCaller = + StackWalker.getInstance() + .walk( + s -> + s.skip(2) + .limit(1) + .map(StackFrame::getClassName) + .anyMatch(VALID_CALLERS::contains)); + if (!validCaller) { + throw new IllegalCallerException( + "VectorizationProvider is internal and can only be used by known Lucene classes."); + } + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final VectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java b/lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java new file mode 100644 index 00000000..7eddee21 --- /dev/null +++ b/lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.lang.reflect.Method; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.logging.Logger; + +/** Accessor to get Hotspot VM Options (if available). */ +final class HotspotVMOptions { + private HotspotVMOptions() {} // can't construct + + /** True iff the Java VM is based on Hotspot and has the Hotspot MX bean readable by Lucene */ + public static final boolean IS_HOTSPOT_VM; + + /** + * Returns an optional with the value of a Hotspot VM option. If the VM option does not exist or + * is not readable, returns an empty optional. + */ + public static Optional get(String name) { + return ACCESSOR.apply(Objects.requireNonNull(name, "name")); + } + + private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory"; + private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean"; + private static final Function> ACCESSOR; + + static { + boolean isHotspot = false; + Function> accessor = name -> Optional.empty(); + try { + final Class beanClazz = Class.forName(HOTSPOT_BEAN_CLASS); + // we use reflection for this, because the management factory is not part + // of java.base module: + final Object hotSpotBean = + Class.forName(MANAGEMENT_FACTORY_CLASS) + .getMethod("getPlatformMXBean", Class.class) + .invoke(null, beanClazz); + if (hotSpotBean != null) { + final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class); + final Method getValueMethod = getVMOptionMethod.getReturnType().getMethod("getValue"); + isHotspot = true; + accessor = + name -> { + try { + final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, name); + return Optional.of(getValueMethod.invoke(vmOption).toString()); + } catch (@SuppressWarnings("unused") + ReflectiveOperationException + | RuntimeException e) { + return Optional.empty(); + } + }; + } + } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) { + isHotspot = false; + final Logger log = Logger.getLogger(HotspotVMOptions.class.getName()); + // SeforimLibrary (Android): ART has no modules (Class.getModule, ModuleLayer) + if (jdkManagementUnreadable()) { + log.warning( + "Lucene cannot access JVM internals to optimize algorithms or calculate object sizes, unless the 'jdk.management' Java module " + + "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor]."); + } else { + log.warning( + "Lucene cannot optimize algorithms or calculate object sizes for JVMs that are not based on Hotspot or a compatible implementation."); + } + } + IS_HOTSPOT_VM = isHotspot; + ACCESSOR = accessor; + } + + // Whether a module layer holds this class without the jdk.management module readable; false where there are no + // modules (classpath, ART) + private static boolean jdkManagementUnreadable() { + try { + final Module module = HotspotVMOptions.class.getModule(); + final ModuleLayer layer = module.getLayer(); + // classpath / unnamed module has no layer, so we need to check: + return layer != null + && layer.findModule("jdk.management").map(module::canRead).orElse(false) == false; + } catch (@SuppressWarnings("unused") LinkageError e) { + return false; + } + } +} diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 52c283fb..89b8274c 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -1,25 +1,177 @@ +import org.jetbrains.kotlin.gradle.plugin.mpp.TestExecutable +import org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeSimulatorTest +import java.net.URI +import java.security.DigestInputStream +import java.security.MessageDigest + plugins { alias(libs.plugins.multiplatform) + alias(libs.plugins.android.library) alias(libs.plugins.kotlinx.serialization) } group = "io.github.kdroidfilter.seforimlibrary" +// The Java runtime Lucene runs in on iOS (OpenJDK's mobile port, see IOS_RUNTIME.md), built by the CI of +// kdroidFilter/openjdk-mobile (branch mobile-runtime) and downloaded from its release, checked +val mobileRuntimeRelease = "mobile-runtime-1" +val mobileRuntimeSha256 = + mapOf( + "ios-simulator" to "5f392f51cece76b0bc318bced666738e3ce16fa559a7a385d345ab3955458384", + "ios-device" to "62ca2a74da57913613ae076905cb812d117afcec6472ffae3574488682fbf758", + ) + +// build/javaRuntime/: static/ (the libraries an app links) and lib/ (its java.home) +val javaRuntimes = + mobileRuntimeSha256.mapValues { (target, sha256) -> + val suffix = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } + val archive = layout.buildDirectory.file("javaRuntime/jdk-$target.tar.gz") + val download = + tasks.register("downloadJavaRuntime$suffix") { + description = "Downloads the Java runtime for $target ($mobileRuntimeRelease)." + val url = "https://github.com/kdroidFilter/openjdk-mobile/releases/download/$mobileRuntimeRelease/jdk-$target.tar.gz" + inputs.property("url", url) + inputs.property("sha256", sha256) + outputs.file(archive) + doLast { + val file = archive.get().asFile.apply { parentFile.mkdirs() } + val digest = MessageDigest.getInstance("SHA-256") + DigestInputStream(URI(url).toURL().openStream(), digest).use { input -> + file.outputStream().use { input.copyTo(it) } + } + val actual = digest.digest().joinToString("") { "%02x".format(it) } + if (actual != sha256) { + file.delete() + error("Unexpected jdk-$target.tar.gz from $url (SHA-256 $actual)") + } + } + } + tasks.register("javaRuntime$suffix") { + description = "Extracts the Java runtime for $target." + dependsOn(download) + from(tarTree(archive)) + into(layout.buildDirectory.dir("javaRuntime/$target")) + // Some of the JDK's files are read-only: the next sync couldn't replace them + filePermissions { user { write = true } } + } + } + +// The JNI headers of that runtime, for the iOS bindings (cinterop "jni"): from the sources of the same release +val jniHeaders = layout.buildDirectory.dir("jni/$mobileRuntimeRelease/include") +val downloadJniHeaders by tasks.registering { + description = "Downloads the JNI headers of the Java runtime ($mobileRuntimeRelease) for the iOS bindings." + val sources = "https://raw.githubusercontent.com/kdroidFilter/openjdk-mobile/$mobileRuntimeRelease/src/java.base" + val headers = + mapOf( + "jni.h" to ("$sources/share/native/include/jni.h" to "1e03bd83d1662ac00647f7a06c11d32333767ee537803ec1d153e89888414520"), + "jni_md.h" to ("$sources/unix/native/include/jni_md.h" to "8232a5a68d36060aaa426fbfdb4537983ecfdfc5fe07a43230005ddd1a4b986a"), + ) + val dir = jniHeaders + inputs.property("headers", headers.toString()) + outputs.dir(dir) + doLast { + headers.forEach { (name, source) -> + val (url, sha256) = source + val bytes = URI(url).toURL().readBytes() + val actual = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + check(actual == sha256) { "Unexpected $name from $url (SHA-256 $actual)" } + dir.get().file(name).asFile.apply { parentFile.mkdirs() }.writeBytes(bytes) + } + } +} + +// The runtime's static libraries an iOS binary links (force-loaded: the runtime finds its natives by name) +val jdkStaticLibs = + listOf( + "libjvm.a", "libjava.a", "libnio.a", "libnet.a", "libzip.a", "libjimage.a", "libverify.a", "libextnet.a", + "libfallbackLinker.a", "libsyslookup.a", + ) +val iosSimulatorRuntime = layout.buildDirectory.dir("javaRuntime/ios-simulator").get().asFile + kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) jvm() + android { + namespace = "io.github.kdroidfilter.seforimlibrary.search" + compileSdk = 35 + // Lucene 10 needs the Java 21 APIs ART only has from Android 15 + minSdk = 35 + // The search on ART, on a device or an emulator (the parity dump on a real index, see SearchDump) + withDeviceTestBuilder { sourceSetTreeName = "test" }.configure { + instrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + } + // The iOS tests link the runtime and run the search end to end in the simulator + iosSimulatorArm64 { + binaries.withType().configureEach { + val static = iosSimulatorRuntime.resolve("static") + linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve(it).path) }) + linkerOpts(static.resolve("libffi.a").path, "-export_dynamic", "-lz", "-lc++") + listOf("Foundation", "CoreFoundation", "Security", "SystemConfiguration").forEach { linkerOpts("-framework", it) } + linkTaskProvider.configure { dependsOn(javaRuntimes.getValue("ios-simulator")) } + } + } + listOf(iosArm64(), iosSimulatorArm64()).forEach { target -> + target.compilations.getByName("main").cinterops { + create("jni") { + definitionFile.set(project.file("src/nativeInterop/cinterop/jni.def")) + includeDirs(jniHeaders) + } + create("sqlite3") { + definitionFile.set(project.file("src/nativeInterop/cinterop/sqlite3.def")) + } + } + } sourceSets { - jvmMain.dependencies { + commonMain.dependencies { api(project(":core")) implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.serialization.json) - implementation(libs.lucene.core) - implementation(libs.lucene.analysis.common) - implementation(libs.sqlDelight.driver.sqlite) + implementation(libs.kotlinx.io.core) + implementation(libs.atomicfu) implementation(libs.kermit) - implementation(libs.jsoup) + } + + // Lucene runs in process on the JVM and Android (ART, API 35+): the same code serves both + val androidJvmMain by creating { + dependsOn(commonMain.get()) + dependencies { + compileOnly(libs.lucene.core) + implementation(libs.jsoup) + } + } + jvmMain { + dependsOn(androidJvmMain) + dependencies { + implementation(libs.lucene.core) + implementation(libs.lucene.analysis.common) + implementation(libs.sqlite.jdbc) + } + } + androidMain { + dependsOn(androidJvmMain) + dependencies { + // lucene-core patched for ART + implementation(project(":lucene-android")) + } + } + + // The manual source set above turns off the default hierarchy: iOS shares its own one by hand + val iosMain by creating { + dependsOn(commonMain.get()) + dependencies { + implementation(libs.ksoup) + } + } + iosArm64Main { dependsOn(iosMain) } + iosSimulatorArm64Main { dependsOn(iosMain) } + + getByName("androidDeviceTest").dependencies { + implementation(kotlin("test")) + implementation(libs.androidx.test.runner) + implementation(libs.androidx.test.junit) } jvmTest.dependencies { @@ -28,3 +180,56 @@ kotlin { } } } + +tasks.matching { it.name.startsWith("cinteropJni") }.configureEach { dependsOn(downloadJniHeaders) } + +// The jars the embedded Java runtime runs: this module (its Lucene side and IndexBridgeServer), Lucene, Kotlin +val iosJvmClasspath by tasks.registering(Sync::class) { + description = "Collects the jars the Java runtime of an iOS app runs for the search." + from(tasks.named("jvmJar")) + from( + configurations.named("jvmRuntimeClasspath").map { classpath -> + classpath.filter { jar -> listOf("lucene-core", "kotlin-stdlib", "kotlinx-serialization-core", "kotlinx-serialization-json").any { jar.name.startsWith(it) } } + }, + ) + into(layout.buildDirectory.dir("iosJvmClasspath")) +} + +run { + val testBinaryDir = layout.buildDirectory.dir("bin/iosSimulatorArm64/debugTest") + + // java.home of the runtime is /lib: its classes (an exploded image), conf and data, the jars beside + val iosTestRuntime by tasks.registering(Sync::class) { + dependsOn(javaRuntimes.getValue("ios-simulator")) + from(iosSimulatorRuntime.resolve("lib")) + from(iosJvmClasspath) { into("app") } + into(testBinaryDir.map { it.dir("lib") }) + filePermissions { user { write = true } } + } + + tasks.withType().configureEach { + dependsOn(iosTestRuntime) + device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) + val jars = iosJvmClasspath.map { sync -> sync.destinationDir.listFiles().orEmpty().map { testBinaryDir.get().dir("lib/app/${it.name}").asFile.path } } + val indexDir = providers.gradleProperty("indexDir") + val parityOut = layout.buildDirectory.file("parity/ios.txt") + val luceneQueryCache = providers.gradleProperty("luceneQueryCache") + doFirst { + // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix + environment("SIMCTL_CHILD_SEFORIM_JVM_CLASSPATH", jars.get().joinToString(":")) + indexDir.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_INDEX_DIR", it) } + environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", parityOut.get().asFile.path) + luceneQueryCache.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_JVM_OPTIONS", "-Dseforim.lucene.queryCache=$it") } + } + } +} + +tasks.named("jvmTest") { + // The parity dump on a real index: -PindexDir= + providers.gradleProperty("indexDir").orNull?.let { systemProperty("seforimIndexDir", it) } + systemProperty("parityOut", layout.buildDirectory.file("parity/jvm.txt").get().asFile.path) + providers.gradleProperty("parityCores").orNull?.let { jvmArgs("-XX:ActiveProcessorCount=$it") } + providers.gradleProperty("luceneQueryCache").orNull?.let { systemProperty("seforim.lucene.queryCache", it) } + providers.gradleProperty("probeOut").orNull?.let { systemProperty("probeOut", it) } + testLogging.showStandardStreams = true +} diff --git a/search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt b/search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt new file mode 100644 index 00000000..38adfa24 --- /dev/null +++ b/search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt @@ -0,0 +1,37 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import android.os.SystemClock +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File + +/** + * The search on ART (Android 15+), with Lucene in process: dumps what it answers on a real index, like + * RealIndexParityTest on the JVM. The index and lexical.db go in the test app's files dir, under seforim/ (adb root, push, + * chown to the app); the dump comes back as parity-android.txt there. Skipped without an index. + */ +@RunWith(AndroidJUnit4::class) +class AndroidRealIndexParityTest { + @Test + fun dumpRealIndex() { + val files = InstrumentationRegistry.getInstrumentation().targetContext.filesDir + val indexDir = File(files, "seforim/seforim.db.lucene") + println("Parity index: $indexDir exists=${indexDir.exists()} files=${indexDir.list()?.size}") + if (!indexDir.isDirectory) return + val arguments = InstrumentationRegistry.getArguments() + arguments.getString("luceneQueryCache")?.let { System.setProperty("seforim.lucene.queryCache", it) } + val engine = + LexicalSearchEngine( + index = LuceneIndexBackend(indexDir.toPath()), + dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, File(indexDir.parentFile, "lexical.db").path) }, + ) + val start = SystemClock.elapsedRealtime() + val dump = engine.use { runBlocking { SearchDump.dump(it) } } + println("Parity dump in ${SystemClock.elapsedRealtime() - start} ms") + File(files, "parity-android.txt").writeText(dump) + } +} diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt new file mode 100644 index 00000000..5d6c3649 --- /dev/null +++ b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.androidJvm.kt @@ -0,0 +1,10 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.runBlocking +import org.jsoup.Jsoup +import org.jsoup.safety.Safelist + +internal actual fun escapedText(html: String): String = Jsoup.clean(html, Safelist.none()) + +internal actual fun runBlockingIo(block: suspend CoroutineScope.() -> T): T = runBlocking(block = block) diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt new file mode 100644 index 00000000..710b5b4e --- /dev/null +++ b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt @@ -0,0 +1,578 @@ +package io.github.kdroidfilter.seforimlibrary.search.index + +import io.github.kdroidfilter.seforimlibrary.search.SearchFacets +import org.apache.lucene.analysis.Analyzer +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute +import org.apache.lucene.document.IntPoint +import org.apache.lucene.index.DocValuesType +import org.apache.lucene.index.FieldInfo +import org.apache.lucene.index.IndexReader +import org.apache.lucene.index.LeafReaderContext +import org.apache.lucene.index.NumericDocValues +import org.apache.lucene.index.ReaderUtil +import org.apache.lucene.index.StoredFieldVisitor +import org.apache.lucene.index.StoredFields +import org.apache.lucene.index.Term +import org.apache.lucene.index.VectorSimilarityFunction +import org.apache.lucene.search.BooleanClause +import org.apache.lucene.search.BooleanQuery +import org.apache.lucene.search.BoostQuery +import org.apache.lucene.search.Collector +import org.apache.lucene.search.CollectorManager +import org.apache.lucene.search.FuzzyQuery +import org.apache.lucene.search.IndexSearcher +import org.apache.lucene.search.KnnByteVectorQuery +import org.apache.lucene.search.KnnFloatVectorQuery +import org.apache.lucene.search.LRUQueryCache +import org.apache.lucene.search.LeafCollector +import org.apache.lucene.search.MatchAllDocsQuery +import org.apache.lucene.search.MultiPhraseQuery +import org.apache.lucene.search.PointRangeQuery +import org.apache.lucene.search.PrefixQuery +import org.apache.lucene.search.Query +import org.apache.lucene.search.QueryCachingPolicy +import org.apache.lucene.search.ReferenceManager +import org.apache.lucene.search.Scorable +import org.apache.lucene.search.ScoreDoc +import org.apache.lucene.search.ScoreMode +import org.apache.lucene.search.SearcherFactory +import org.apache.lucene.search.SearcherManager +import org.apache.lucene.search.Sort +import org.apache.lucene.search.TermQuery +import org.apache.lucene.search.TopDocs +import org.apache.lucene.search.UsageTrackingQueryCachingPolicy +import org.apache.lucene.store.Directory +import org.apache.lucene.store.FSDirectory +import org.apache.lucene.store.NIOFSDirectory +import org.apache.lucene.util.QueryBuilder +import java.nio.file.Path +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import org.apache.lucene.index.VectorEncoding as LuceneVectorEncoding +import org.apache.lucene.search.SortField as LuceneSortField + +private const val FIELD_BOOK_ID = "book_id" +private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" + +// Smallest partition worth a task of its own when a segment is split across cores +private const val MIN_DOCS_PER_SLICE = 100_000 + +/** + * The [LuceneIndex] of a Lucene directory, read with Lucene in this process: on the JVM, on Android (ART, API 35+), + * and inside the Java runtime the iOS app embeds (see `IndexBridgeServer`). + * + * @param indexDir the index directory + * @param cacheQueries whether to cache any query seen often (small indexes: the lookup index); else only the filters + * every search repeats + * @param analyzer the analyzer the index was written with + */ +class LuceneIndexBackend( + private val indexDir: Path, + private val cacheQueries: Boolean = false, + private val analyzer: Analyzer = StandardAnalyzer(), +) : LuceneIndex { + // Open lazily to avoid any I/O at app startup + private val dir: Directory by lazy { openDirectory(indexDir) } + + // One reader shared by every search: opening one per search re-reads the term index of each segment and maps, + // then unmaps, all the files. Refreshed when the index changes on disk (delta updates write into it). + private val searcherManagerLazy = + lazy { + SearcherManager(dir, searcherFactory()).apply { + addListener( + object : ReferenceManager.RefreshListener { + override fun beforeRefresh() = Unit + + override fun afterRefresh(didRefresh: Boolean) { + if (didRefresh) bookAncestors.clear() + } + }, + ) + } + } + private val searcherManager: SearcherManager by searcherManagerLazy + + // Ancestor category ids per book, for the facets: every line of a book shares them + private val bookAncestors = ConcurrentHashMap() + + private fun searcherFactory(): SearcherFactory = + if (!queryCacheEnabled) { + // Lucene's query cache changes how the filters run, and with it the last digits of the scores: off, the + // same search ranks the same every time (the parity tests across platforms) + object : SearcherFactory() { + override fun newSearcher( + reader: IndexReader, + previousReader: IndexReader?, + ): IndexSearcher = PartitionedSearcherFactory.newSearcher(reader, previousReader).apply { queryCache = null } + } + } else if (!cacheQueries) { + PartitionedSearcherFactory + } else { + object : SearcherFactory() { + override fun newSearcher( + reader: IndexReader, + previousReader: IndexReader?, + ): IndexSearcher = + IndexSearcher(reader).apply { + queryCache = LRUQueryCache(1024, 64L * 1024 * 1024) // up to 64MiB cache + queryCachingPolicy = UsageTrackingQueryCachingPolicy() + } + } + } + + private fun acquire(): IndexSearcher { + val manager = searcherManager + manager.maybeRefresh() + return manager.acquire() + } + + private inline fun withSearcher(block: (IndexSearcher) -> T): T { + val searcher = acquire() + try { + return block(searcher) + } finally { + searcherManager.release(searcher) + } + } + + override fun analyze(text: String): List = + try { + val out = mutableListOf() + analyzer.tokenStream("text", text).use { ts -> + val termAtt = ts.addAttribute(CharTermAttribute::class.java) + ts.reset() + while (ts.incrementToken()) { + val t = termAtt.toString() + if (t.isNotBlank()) out += t + } + ts.end() + } + out + } catch (_: Exception) { + emptyList() + } + + override fun search( + query: IndexQuery, + limit: Int, + fields: Set, + sort: List?, + ): IndexPage = + withSearcher { searcher -> + val luceneQuery = toLucene(query) + val top = + if (sort.isNullOrEmpty()) { + searcher.search(luceneQuery, limit) + } else { + searcher.search(luceneQuery, limit, Sort(*sort.map { it.toLucene() }.toTypedArray())) + } + top.toPage(searcher.storedFields(), fields) + } + + override fun openCursor( + query: IndexQuery, + fields: Set, + ): IndexCursor = Cursor(toLucene(query), fields, acquire()) + + private inner class Cursor( + private val query: Query, + private val fields: Set, + private val searcher: IndexSearcher, + ) : IndexCursor { + private var after: ScoreDoc? = null + private var totalHits: Long? = null + private var closed = false + + override fun next(limit: Int): IndexPage { + val top = searcher.searchAfter(after, query, limit) + if (totalHits == null) totalHits = top.totalHits.value() + top.scoreDocs.lastOrNull()?.let { after = it } + return IndexPage(readDocs(searcher.storedFields(), top.scoreDocs.asList(), fields), totalHits ?: 0) + } + + override fun close() { + if (closed) return + closed = true + searcherManager.release(searcher) + } + } + + override fun facets(query: IndexQuery): SearchFacets = + withSearcher { searcher -> + // Only book ids are read per hit (doc values when the index has them, else the first stored field); + // category counts follow from the book counts, as every line of a book shares its ancestors. One + // collector per slice, as the slices may run concurrently + class FacetCollector : Collector { + val bookCounts = HashMap() + var totalHits = 0L + + override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { + val reader = leafContext.reader() + val storedFields = reader.storedFields() + val bookIdValues: NumericDocValues? = + reader.fieldInfos + .fieldInfo(FIELD_BOOK_ID) + ?.takeIf { it.docValuesType == DocValuesType.NUMERIC } + ?.let { reader.getNumericDocValues(FIELD_BOOK_ID) } + // Lines of a book are mostly contiguous: count runs, then fold them into the map + var runBook = -1L + var runCount = 0 + + fun flushRun() { + if (runCount > 0) bookCounts.merge(runBook, runCount, Int::plus) + runCount = 0 + } + + return object : LeafCollector { + override fun setScorer(scorer: Scorable) = Unit + + override fun collect(doc: Int) { + totalHits++ + val bookId = + if (bookIdValues != null && bookIdValues.advanceExact(doc)) { + bookIdValues.longValue() + } else { + readStoredBookId(storedFields, doc) ?: return + } + if (bookId != runBook) { + flushRun() + runBook = bookId + if (!bookAncestors.containsKey(bookId)) loadAncestors(storedFields, doc, bookId) + } + runCount++ + } + + override fun finish() = flushRun() + } + } + + override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES + } + + searcher.search( + toLucene(query), + object : CollectorManager { + override fun newCollector() = FacetCollector() + + override fun reduce(collectors: Collection): SearchFacets { + val bookCounts = HashMap() + collectors.forEach { c -> c.bookCounts.forEach { (book, n) -> bookCounts.merge(book, n, Int::plus) } } + val categoryCounts = HashMap() + for ((bookId, count) in bookCounts) { + bookAncestors[bookId]?.forEach { categoryCounts.merge(it, count, Int::plus) } + } + return SearchFacets( + totalHits = collectors.sumOf { it.totalHits }, + categoryCounts = categoryCounts, + bookCounts = bookCounts, + ) + } + }, + ) + } + + override fun collectLongs( + query: IndexQuery, + field: String, + ): LongArray = + withSearcher { searcher -> + // One collector per slice, as the slices may run concurrently + class LongCollector : Collector { + var values = LongArray(256) + var count = 0 + + override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { + val storedFields = leafContext.reader().storedFields() + val wanted = setOf(field) + return object : LeafCollector { + override fun setScorer(scorer: Scorable) = Unit + + override fun collect(doc: Int) { + val value = + storedFields + .document(doc, wanted) + .getField(field) + ?.numericValue() + ?.toLong() ?: return + if (count == values.size) values = values.copyOf(count * 2) + values[count++] = value + } + } + } + + override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES + } + searcher.search( + toLucene(query), + object : CollectorManager { + override fun newCollector() = LongCollector() + + override fun reduce(collectors: Collection): LongArray { + val out = LongArray(collectors.sumOf { it.count }) + var at = 0 + for (c in collectors) { + c.values.copyInto(out, at, 0, c.count) + at += c.count + } + return out + } + }, + ) + } + + override fun count(query: IndexQuery): Int = withSearcher { it.count(toLucene(query)) } + + override fun vectorEncoding(field: String): VectorEncoding? = + withSearcher { searcher -> + searcher.indexReader + .leaves() + .firstNotNullOfOrNull { it.reader().fieldInfos.fieldInfo(field) } + ?.let { if (it.vectorEncoding == LuceneVectorEncoding.BYTE) VectorEncoding.BYTE else VectorEncoding.FLOAT32 } + } + + override fun knn(request: KnnRequest): List = + withSearcher { searcher -> + val filter = request.filter?.let { toLucene(it) } + val byteVector = request.byteVector + val floatVector = request.floatVector + val top: List = + when { + byteVector != null -> + searcher.search(KnnByteVectorQuery(request.field, byteVector, request.k, filter), request.k).scoreDocs.asList() + floatVector != null && request.oversample > 1 -> + rescoredKnn(searcher, request.field, floatVector, request.k, request.oversample, filter) + floatVector != null -> + searcher.search(KnnFloatVectorQuery(request.field, floatVector, request.k, filter), request.k).scoreDocs.asList() + else -> error("KnnRequest without a vector") + } + readDocs(searcher.storedFields(), top, request.fields) + } + + /** + * KNN over a field quantized by Lucene (float vectors stored next to their int8 copy): the graph search ranks on the + * approximate int8 similarity, so it gathers [oversample] times more candidates and keeps the [k] best by their + * exact float similarity. Empty when the field holds no eligible document. + */ + private fun rescoredKnn( + searcher: IndexSearcher, + field: String, + query: FloatArray, + k: Int, + oversample: Int, + filter: Query?, + ): List { + val candidates = searcher.search(KnnFloatVectorQuery(field, query, k * oversample, filter), k * oversample).scoreDocs + if (candidates.isEmpty()) return emptyList() + val leaves = searcher.indexReader.leaves() + val rescored = ArrayList(candidates.size) + // By segment, in doc order: the vector iterators only move forward + for ((leafIndex, docs) in candidates.groupBy { ReaderUtil.subIndex(it.doc, leaves) }) { + val leaf = leaves[leafIndex] + val values = leaf.reader().getFloatVectorValues(field) ?: continue + val iterator = values.iterator() + for (doc in docs.map { it.doc }.sorted()) { + if (iterator.advance(doc - leaf.docBase) != doc - leaf.docBase) continue + rescored += ScoreDoc(doc, VectorSimilarityFunction.COSINE.compare(query, values.vectorValue(iterator.index()))) + } + } + return rescored.sortedByDescending { it.score }.take(k) + } + + override fun close() { + if (searcherManagerLazy.isInitialized()) runCatching { searcherManager.close() } + } + + // --- Lucene mapping --- + + private fun toLucene(query: IndexQuery): Query = + when (query) { + is IndexQuery.Term -> TermQuery(Term(query.field, query.text)) + is IndexQuery.Prefix -> PrefixQuery(Term(query.field, query.text)) + is IndexQuery.Fuzzy -> FuzzyQuery(Term(query.field, query.text), query.maxEdits) + is IndexQuery.Bool -> + BooleanQuery + .Builder() + .apply { query.clauses.forEach { add(toLucene(it.query), it.occur.toLucene()) } } + .build() + is IndexQuery.Boost -> BoostQuery(toLucene(query.query), query.boost) + is IndexQuery.MultiPhrase -> + MultiPhraseQuery + .Builder() + .apply { + setSlop(query.slop) + query.positions.forEachIndexed { position, alternatives -> + add(alternatives.map { Term(query.field, it) }.toTypedArray(), position) + } + }.build() + is IndexQuery.AnalyzedPhrase -> { + val builder = QueryBuilder(analyzer) + builder.createPhraseQuery(query.field, query.text, query.slop) + ?: builder.createBooleanQuery(query.field, query.text, BooleanClause.Occur.MUST) + ?: BooleanQuery.Builder().build() + } + is IndexQuery.IntExact -> IntPoint.newExactQuery(query.field, query.value) + is IndexQuery.IntSet -> IntPoint.newSetQuery(query.field, *query.values) + IndexQuery.MatchAll -> MatchAllDocsQuery() + } + + private fun Occur.toLucene(): BooleanClause.Occur = + when (this) { + Occur.MUST -> BooleanClause.Occur.MUST + Occur.SHOULD -> BooleanClause.Occur.SHOULD + Occur.FILTER -> BooleanClause.Occur.FILTER + Occur.MUST_NOT -> BooleanClause.Occur.MUST_NOT + } + + private fun SortField.toLucene(): LuceneSortField = + LuceneSortField( + field, + when (type) { + SortField.Type.INT -> LuceneSortField.Type.INT + SortField.Type.LONG -> LuceneSortField.Type.LONG + }, + reverse, + ) + + private fun TopDocs.toPage( + stored: StoredFields, + fields: Set, + ): IndexPage = IndexPage(readDocs(stored, scoreDocs.asList(), fields), totalHits.value()) + + private fun readDocs( + stored: StoredFields, + scoreDocs: List, + fields: Set, + ): List = + scoreDocs.map { sd -> + val longs = HashMap() + val strings = HashMap() + if (fields.isNotEmpty()) { + for (field in stored.document(sd.doc, fields)) { + val number = field.numericValue() + if (number != null) { + longs[field.name()] = number.toLong() + } else { + field.stringValue()?.let { strings[field.name()] = it } + } + } + } + // A sorted search has no score: NaN can't cross the iOS bridge's JSON + IndexDoc(if (sd.score.isNaN()) 0f else sd.score, longs, strings) + } + + private fun readStoredBookId( + storedFields: StoredFields, + doc: Int, + ): Long? { + var bookId: Long? = null + storedFields.document( + doc, + object : StoredFieldVisitor() { + override fun needsField(fieldInfo: FieldInfo): Status = + when { + bookId != null -> Status.STOP + fieldInfo.name == FIELD_BOOK_ID -> Status.YES + else -> Status.NO + } + + override fun longField( + fieldInfo: FieldInfo, + value: Long, + ) { + bookId = value + } + + override fun intField( + fieldInfo: FieldInfo, + value: Int, + ) { + bookId = value.toLong() + } + }, + ) + return bookId + } + + private fun loadAncestors( + storedFields: StoredFields, + doc: Int, + bookId: Long, + ) { + val ancestors = + storedFields + .document(doc, setOf(FIELD_ANCESTOR_CATEGORY_IDS)) + .getField(FIELD_ANCESTOR_CATEGORY_IDS) + ?.stringValue() + ?.split(",") + ?.mapNotNull { it.trim().toLongOrNull() } + .orEmpty() + // Lines added by delta updates lack the field: try again on the book's next run + if (ancestors.isNotEmpty()) bookAncestors[bookId] = ancestors.toLongArray() + } + + companion object { + /** + * The directory of [path]: memory-mapped where Lucene can (HotSpot), read through NIO where its Panama-based + * mapping isn't available (GraalVM native image, Android, the iOS Java runtime). + */ + fun openDirectory(path: Path): Directory = if (mmapAvailable) FSDirectory.open(path) else NIOFSDirectory(path) + + private val queryCacheEnabled: Boolean = System.getProperty("seforim.lucene.queryCache") != "off" + + private val mmapAvailable: Boolean by lazy { + System.getProperty("org.graalvm.nativeimage.imagecode") == null && + System.getProperty("java.vm.vendor")?.contains("Android") != true && + System.getProperty("os.name") != "Darwin" && + System.getProperty("java.vm.name")?.contains("Zero") != true + } + } +} + +/** + * Searchers that split the index, even a single segment, into one partition per core and search them concurrently: + * a merged index is one big segment, which a plain searcher scores on one thread. + */ +private object PartitionedSearcherFactory : SearcherFactory() { + private val cores = Runtime.getRuntime().availableProcessors() + private val executor: ExecutorService by lazy { + val ids = AtomicInteger() + Executors.newFixedThreadPool(cores) { task -> + Thread(task, "lucene-search-${ids.incrementAndGet()}").apply { isDaemon = true } + } + } + + override fun newSearcher( + reader: IndexReader, + previousReader: IndexReader?, + ): IndexSearcher { + val searcher = + if (cores == 1) { + IndexSearcher(reader) + } else { + val docsPerSlice = maxOf(MIN_DOCS_PER_SLICE, reader.maxDoc() / cores + 1) + object : IndexSearcher(reader, executor) { + override fun slices(leaves: List): Array = slices(leaves, docsPerSlice, Int.MAX_VALUE, true) + } + } + return searcher.apply { queryCachingPolicy = StableFiltersCachingPolicy } + } +} + +/** + * Caches only the filters every search repeats (line documents, base books). Lucene's default policy also caches any + * clause seen twice, and each search builds its word filters twice (facets, then results): it would then turn them + * into bitsets over the whole index for a query that won't come back. + */ +private object StableFiltersCachingPolicy : QueryCachingPolicy { + override fun onUse(query: Query) = Unit + + override fun shouldCache(query: Query): Boolean = + when (query) { + is TermQuery -> query.term.field() == "type" + is PointRangeQuery -> query.field == "is_base_book" + else -> false + } +} diff --git a/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt b/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt new file mode 100644 index 00000000..a696b1fa --- /dev/null +++ b/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt @@ -0,0 +1,25 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import android.database.sqlite.SQLiteDatabase + +/** Through Android's SQLite: one connection, which serializes the queries of every thread. */ +actual fun openReadOnlySqlite(path: String): SqliteReader = AndroidSqliteReader(path) + +private class AndroidSqliteReader( + path: String, +) : SqliteReader { + private val db = SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY or SQLiteDatabase.NO_LOCALIZED_COLLATORS) + + override fun query( + sql: String, + args: List, + ): List> = + db.rawQuery(sql, args.toTypedArray()).use { cursor -> + val columns = cursor.columnNames + val rows = ArrayList>(cursor.count) + while (cursor.moveToNext()) rows += columns.indices.associate { columns[it] to cursor.getString(it) } + rows + } + + override fun close() = db.close() +} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt new file mode 100644 index 00000000..1a9240fc --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HallucinationBlacklist.kt @@ -0,0 +1,56 @@ +package io.github.kdroidfilter.seforimlibrary.search + +/** + * Blacklist of hallucinated dictionary mappings: LLM-generated mappings that are incorrect and should be ignored. + * Key: normalized token, Value: set of incorrect base forms to reject. + * + * TODO: This is a temporary workaround. The proper fix is to correct the + * hallucinated mappings directly in the lexical dictionary (lexical.db) + * so this blacklist becomes unnecessary. + */ +internal object HallucinationBlacklist { + // token to base + private val ENTRIES = + listOf( + "לחתוכ" to "כנ", + "לחתוך" to "כנ", + "לבקש" to "רצה", + "מתפלל" to "בעה", + "לשבת" to "נתנ", + "לעבוד" to "עשה", + "להעמיד" to "קומ", + "כנ" to "כנ", + "בנ" to "בנ", + "לשימ" to "שמ", + "לשים" to "שמ", + "לדונ" to "יד", + "לדון" to "יד", + "אמ" to "אמ", + "להרגיע" to "יושב", + "לחייכ" to "חוה", + "לחייך" to "חוה", + "שנ" to "שנ", + "עצ" to "עצ", + "דנ" to "דנ", + "להיכנס" to "עלה", + "לשקול" to "לקח", + "לבנות" to "בנ", + "לשחרר" to "שרה", + "להתיר" to "שרה", + ) + + private val bases: Map> by lazy { + ENTRIES + .groupBy({ HebrewTextUtils.normalizeHebrew(it.first) }, { HebrewTextUtils.normalizeHebrew(it.second) }) + .mapValues { it.value.toSet() } + } + + /** Whether [expansion] reaches [token] only through a blacklisted base. */ + fun rejects( + token: String, + expansion: MagicDictionaryIndex.Expansion, + ): Boolean { + val blacklisted = bases[HebrewTextUtils.normalizeHebrew(token)] ?: return false + return expansion.base.any { HebrewTextUtils.normalizeHebrew(it) in blacklisted } + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewTextUtils.kt diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt new file mode 100644 index 00000000..5529c24f --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt @@ -0,0 +1,711 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import co.touchlab.kermit.Logger +import io.github.kdroidfilter.seforimlibrary.search.index.BoolBuilder +import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor +import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.Occur +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.withContext + +private val logger = Logger.withTag("LexicalSearchEngine") + +// Hard cap on how many synonym/expansion terms we allow per token +private const val MAX_SYNONYM_TERMS_PER_TOKEN: Int = 32 + +// Global cap for boost queries built from dictionary expansions +private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 +private const val DEFAULT_NEAR = 5 +private const val PLAN_CACHE_SIZE = 16 + +// Searches run by warmUp(): one and several words, common and rarer ones, a quoted phrase, the divine name (whose +// dictionary entry is the largest) +val WARMUP_QUERIES = listOf("ברוך ה׳ לעולם", "אמר רבי יוחנן", "תפילין", "\"קריאת שמע\"") +const val WARMUP_ROUNDS = 2 +const val WARMUP_PAGE_SIZE = 25 + +// Single Hebrew letters, dropped from a query before its dictionary expansion +private val SINGLE_LETTERS = + setOf("א", "ב", "ג", "ד", "ה", "ו", "ז", "ח", "ט", "י", "כ", "ל", "מ", "נ", "ס", "ע", "פ", "צ", "ק", "ר", "ש", "ת") + +private val WHITESPACE = Regex("\\s+") + +// The stored fields a hit is read from +private val HIT_FIELDS = setOf("book_id", "book_title", "line_id", "line_index", "is_base_book", "order_index", "text_raw") + +/** + * The full-text search of Hebrew religious texts, with diacritics handling, dictionary expansion and fuzzy matching, + * over a Lucene [index] (see [LuceneIndex]: in process on the JVM and Android, through the embedded Java runtime on + * iOS). Hits are ranked by Lucene, then boosted for base books; their snippets come from [snippetProvider] (the + * indexed text when there is none). Inflections come from the lexical [dictionary] when it is available. + */ +class LexicalSearchEngine( + private val index: LuceneIndex, + private val snippetProvider: SnippetProvider? = null, + dictionary: Lazy = lazyOf(null), +) : SearchEngine { + private val magicDict: MagicDictionaryIndex? by dictionary + + // --- SearchEngine interface implementation --- + + override fun openSession( + query: String, + near: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection?, + lineIds: Collection?, + baseBookOnly: Boolean, + ): SearchSession? { + val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) ?: return null + return LexicalSearchSession(context, index.openCursor(context.query, HIT_FIELDS)) + } + + override suspend fun attachSnippets( + hits: List, + query: String, + near: Int, + ): List { + if (hits.isEmpty()) return hits + val context = buildSearchContext(query, near, null, null, null, null) ?: return hits + val sources = withContext(Dispatchers.IO) { snippetSources(hits.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) } + return hits.map { hit -> + val source = sources[hit.lineId] ?: SnippetSource(hit.rawText, 0, hit.rawText.length) + hit.copy( + snippet = SnippetBuilder.build(source.text, context.highlightPlan, source.lineStart, source.lineEnd), + rawText = source.text, + lineStart = source.lineStart, + lineEnd = source.lineEnd, + ) + } + } + + override suspend fun warmUp() { + withContext(Dispatchers.IO) { hashemHighlightTerms } + // A few searches down the whole path (facets, ranking, snippets) load the index structures and, on the JVM, + // get the hot code compiled before the user's first search + repeat(WARMUP_ROUNDS) { + for (query in WARMUP_QUERIES) { + computeFacets(query, baseBookOnly = true) + openSession(query, baseBookOnly = true)?.use { it.nextPage(WARMUP_PAGE_SIZE) } + } + } + } + + override fun searchBooksByTitlePrefix( + query: String, + limit: Int, + ): List { + val q = HebrewTextUtils.normalizeHebrew(query) + if (q.isBlank()) return emptyList() + val tokens = q.split(WHITESPACE).map { it.trim() }.filter { it.isNotEmpty() } + if (tokens.isEmpty()) return emptyList() + + val titleQuery = + BoolBuilder() + // Restrict to book_title docs + .add(IndexQuery.Term("type", "book_title"), Occur.FILTER) + .apply { + // prefix on analyzed 'title' + tokens.forEach { tok -> add(IndexQuery.Prefix("title", tok), Occur.MUST) } + }.build() + val ids = LinkedHashSet() + for (doc in index.search(titleQuery, limit, setOf("book_id")).docs) { + doc.longs["book_id"]?.let { ids.add(it) } + } + return ids.toList() + } + + override fun buildSnippet( + rawText: String, + query: String, + near: Int, + ): String = SnippetBuilder.build(SnippetSources.clean(rawText), planFor(query, near)) + + override fun highlightRanges( + text: String, + query: String, + ): List = mergedRanges(text, planFor(query, DEFAULT_NEAR).highlights(text)) + + override fun rangeSnippet( + text: String, + ranges: List, + lineStart: Int, + lineEnd: Int, + ): String = SnippetBuilder.build(text, ranges.map { Occurrence(it.first, it.last + 1, 0, EvidenceLevel.LITERAL) }, lineStart, lineEnd) + + // The highlight plans of the latest queries: a results page and its preview ask for the same one many times + private val planCache = LruCache(PLAN_CACHE_SIZE) + + private fun planFor( + query: String, + near: Int, + ): HighlightPlan { + planCache[query]?.let { return it } + val plan = buildSearchContext(query, near, null, null, null, null)?.highlightPlan ?: HighlightPlan.EMPTY + planCache[query] = plan + return plan + } + + /** + * What a query highlights: each word with its inflections, from the dictionary entries it truly is a form of and + * that aren't blacklisted for it. With ה׳ in the query, the lone ה is the divine name. + */ + private fun buildHighlightPlan( + tokens: List, + hasHashem: Boolean, + exactPhrases: List, + ): HighlightPlan { + val forms = + tokens.distinct().map { token -> + if (hasHashem && highlightKey(token) == "ה") { + return@map TokenForms(token, extraLiterals = hashemHighlightTerms) + } + val key = highlightKey(token) + val inflections = + magicDict + ?.expansionsFor(listOf(token)) + .orEmpty() + .filter { exp -> exp.inflections.any { highlightKey(it) == key } && !HallucinationBlacklist.rejects(token, exp) } + .flatMap { it.inflections } + TokenForms(token, inflections) + } + val phrases = + exactPhrases + .map { index.analyze(it).joinToString(" ") } + .filter { it.isNotBlank() } + return HighlightPlan.build(forms, phrases) + } + + override fun findInBookCandidates( + query: String, + bookId: Long, + ): LongArray? { + val norm = HebrewTextUtils.normalizeHebrew(query) + if (norm.isBlank()) return LongArray(0) + val bookLines = + BoolBuilder() + .add(IndexQuery.Term("type", "line"), Occur.FILTER) + .add(IndexQuery.IntExact("book_id", bookId.toInt()), Occur.FILTER) + .build() + // Nothing to filter on: every line would be a candidate, cheaper scanned from the DB + val clauses = literalPresenceClauses(index.analyze(norm)) + if (clauses.isEmpty()) return null + val builder = BoolBuilder().add(bookLines, Occur.FILTER) + clauses.forEach { builder.add(it, Occur.FILTER) } + + // Only line_id is read per hit: no scoring, no other stored field + val ids = index.collectLongs(builder.build(), "line_id") + // No candidate may mean a book the index lacks (older index, book added later): unknown, not "no match" + return if (ids.isEmpty() && index.count(bookLines) == 0) null else ids + } + + /** + * Filters every line holding the analyzed [tokens] as one literal substring satisfies. The + * substring may start and end mid-word: the first token is only a word suffix, the last a + * word prefix, the inner ones whole words. A first or lone token is matched anywhere in a + * word through its 4-grams, so one under 4 letters can't filter (a leading wildcard would + * scan the whole term dictionary); a last one under 3 neither, its prefix being too common. + */ + private fun literalPresenceClauses(tokens: List): List = + tokens.mapIndexedNotNull { i, token -> + when { + i in 1 until tokens.lastIndex -> IndexQuery.Term("text", token) + i > 0 && token.length >= 3 -> IndexQuery.Prefix("text", token) + else -> buildNgramPresenceForToken(token) + } + } + + override fun close() { + index.close() + } + + override fun computeFacets( + query: String, + near: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection?, + lineIds: Collection?, + baseBookOnly: Boolean, + ): SearchFacets? { + val context = + buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) + ?: return null + return index.facets(context.matchQuery) + } + + // --- Inner SearchSession class --- + + private inner class LexicalSearchSession( + private val context: SearchContext, + private val cursor: IndexCursor, + ) : SearchSession { + private var finished = false + private var totalHitsValue: Long? = null + + override suspend fun nextPage( + limit: Int, + snippets: Boolean, + ): SearchPage? = + withContext(Dispatchers.IO) { + if (finished) return@withContext null + val page = cursor.next(limit) + if (totalHitsValue == null) totalHitsValue = page.totalHits + if (page.docs.isEmpty()) { + finished = true + return@withContext null + } + val hits = mapDocs(page.docs, context, snippets) + val isLast = page.docs.size < limit + if (isLast) finished = true + SearchPage( + hits = hits, + totalHits = totalHitsValue ?: hits.size.toLong(), + isLastPage = isLast, + ) + } + + override fun close() { + cursor.close() + } + } + + // --- Additional public search methods --- + + suspend fun searchAllText( + rawQuery: String, + near: Int = 5, + limit: Int, + offset: Int = 0, + ): List = doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = null) + + suspend fun searchInBook( + rawQuery: String, + near: Int, + bookId: Long, + limit: Int, + offset: Int = 0, + ): List = doSearch(rawQuery, near, limit, offset, bookFilter = bookId, categoryFilter = null) + + suspend fun searchInCategory( + rawQuery: String, + near: Int, + categoryId: Long, + limit: Int, + offset: Int = 0, + ): List = doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = categoryId) + + suspend fun searchInBooks( + rawQuery: String, + near: Int, + bookIds: Collection, + limit: Int, + offset: Int = 0, + ): List { + if (bookIds.isEmpty()) return emptyList() + return doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = null, bookIds = bookIds) + } + + // --- Private implementation --- + + private class SearchContext( + val query: IndexQuery, + // The required clauses only: the same matches as [query], without the ranking-only clauses (fuzzy, n-grams, + // synonym boosts) that cost a rewrite but can't change the hit set + val matchQuery: IndexQuery, + private val plan: Lazy, + ) { + val highlightPlan: HighlightPlan get() = plan.value + } + + private fun buildSearchContext( + rawQuery: String, + near: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection?, + lineIds: Collection?, + baseBookOnly: Boolean = false, + ): SearchContext? { + // Split the raw query into exact phrases (wrapped in ASCII double quotes) and free text. + // Quoted segments are matched verbatim WITHOUT dictionary expansion (Google-style). + val parsed = SearchQueryParser.parse(rawQuery) + val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) + val exactPhrasesNorm = + parsed.exactPhrases + .map { HebrewTextUtils.normalizeHebrew(it) } + .filter { it.isNotBlank() } + + if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null + + val analyzedRaw = if (norm.isBlank()) emptyList() else index.analyze(norm) + + // Check if the original query contained ה׳ (Hashem) before normalization + val hasHashem = rawQuery.contains("ה׳") || rawQuery.contains("ה'") + + // Filter out single Hebrew letters and stop words BEFORE dictionary expansion + // BUT preserve "ה" if the original query had "ה׳" (Hashem) + val analyzedStd = + analyzedRaw.filter { token -> + // Special case: if query has ה׳, keep "ה" token + if (token == "ה" && hasHashem) return@filter true + // Preserve numeric tokens (e.g., "6") so they can expand via MagicDictionary + if (token.any { it.isDigit() }) return@filter true + + token.length >= 2 && token !in SINGLE_LETTERS + } + + logger.d { "[DEBUG] Original query had Hashem (ה׳): $hasHashem" } + logger.d { "[DEBUG] Analyzed tokens: $analyzedStd" } + + // Get all possible expansions for each token (a token can belong to multiple bases) + // These expansions are used for SEARCH - we keep all of them for better recall + magicDict?.prefetch(analyzedStd) + val tokenExpansions: Map> = + analyzedStd.associateWith { token -> + // Get best expansion (prefers matching base, then largest) + val expansion = magicDict?.expansionFor(token) ?: return@associateWith emptyList() + listOf(expansion) + } + + // What the snippets highlight: built on first use (facets need none) + val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhrasesNorm) } + + val builder = BoolBuilder() + // Mirrors every required clause of builder (the facets need only the hit set) + val matchBuilder = BoolBuilder() + + fun require( + query: IndexQuery, + occur: Occur, + ) { + builder.add(query, occur) + matchBuilder.add(query, occur) + } + require(IndexQuery.Term("type", "line"), Occur.FILTER) + if (bookFilter != null) require(IndexQuery.IntExact("book_id", bookFilter.toInt()), Occur.FILTER) + if (categoryFilter != null) require(IndexQuery.IntExact("category_id", categoryFilter.toInt()), Occur.FILTER) + // Filter by base books only (is_base_book = 1) when baseBookOnly is true + if (baseBookOnly) require(IndexQuery.IntExact("is_base_book", 1), Occur.FILTER) + val bookIdsArray = bookIds?.map { it.toInt() }?.toIntArray() + if (bookIdsArray != null && bookIdsArray.isNotEmpty()) { + require(IndexQuery.IntSet("book_id", bookIdsArray), Occur.FILTER) + } + val lineIdsArray = lineIds?.map { it.toInt() }?.toIntArray() + if (lineIdsArray != null && lineIdsArray.isNotEmpty()) { + require(IndexQuery.IntSet("line_id", lineIdsArray), Occur.FILTER) + } + // Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase. + var exactClausesAdded = 0 + for (phrase in exactPhrasesNorm) { + val exactQuery = buildExactPhraseQuery(phrase) ?: continue + require(exactQuery, Occur.MUST) + exactClausesAdded++ + logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" } + } + + // A fully-quoted query that produced no usable phrase clause (e.g. just punctuation) + // must not fall through to a match-all query. + if (norm.isBlank() && exactClausesAdded == 0) return null + + // Free (unquoted) text: dictionary-aware ranking + presence filtering. + if (norm.isNotBlank()) { + val rankedQuery = buildExpandedQuery(norm, near, analyzedRaw, analyzedStd, tokenExpansions) + val mustAllTokensQuery: IndexQuery? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions) + val phraseQuery: IndexQuery? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near) + if (mustAllTokensQuery != null) { + require(mustAllTokensQuery, Occur.FILTER) + } + if (phraseQuery != null && analyzedStd.size >= 2) { + if (near == 0) require(phraseQuery, Occur.MUST) else builder.add(phraseQuery, Occur.SHOULD) + } + builder.add(rankedQuery, Occur.SHOULD) + } + + return SearchContext( + query = builder.build(), + matchQuery = matchBuilder.build(), + plan = highlightPlan, + ) + } + + // suspend because it calls snippetProvider.getSnippetSources(); + // callers are responsible for providing Dispatchers.IO context. + private suspend fun mapDocs( + docs: List, + context: SearchContext, + snippets: Boolean = true, + ): List { + if (docs.isEmpty()) return emptyList() + + // First pass: extract metadata from index + class DocMeta( + val score: Float, + val bookId: Long, + val bookTitle: String, + val lineId: Long, + val lineIndex: Int, + val isBaseBook: Boolean, + val orderIndex: Int, + // from text_raw field, may be empty if not stored + val indexedRaw: String, + ) + + val docMetas = + docs.map { doc -> + DocMeta( + score = doc.score, + bookId = doc.longs.getValue("book_id"), + bookTitle = doc.strings["book_title"] ?: "", + lineId = doc.longs.getValue("line_id"), + lineIndex = doc.longs.getValue("line_index").toInt(), + isBaseBook = doc.longs["is_base_book"]?.toInt() == 1, + orderIndex = doc.longs["order_index"]?.toInt() ?: 999, + indexedRaw = doc.strings["text_raw"] ?: "", + ) + } + + val snippetSources: Map = + if (snippets) snippetSources(docMetas.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) else emptyMap() + + val hits = + docMetas.map { meta -> + val source = snippetSources[meta.lineId] ?: SnippetSource(meta.indexedRaw, 0, meta.indexedRaw.length) + val raw = source.text + val baseScore = meta.score + + // Calculate boost: lower orderIndex = higher boost (only for base books) + val boostedScore = + if (meta.isBaseBook) { + // Formula: boost = baseScore * (1 + (120 - orderIndex) / 60) + // orderIndex 1 gets ~3x boost, orderIndex 50 gets ~2.2x boost, orderIndex 100+ gets ~1.3x boost + val boostFactor = 1.0f + (120 - meta.orderIndex).coerceAtLeast(0) / 60.0f + baseScore * boostFactor + } else { + baseScore + } + + val snippet = if (snippets) SnippetBuilder.build(raw, context.highlightPlan, source.lineStart, source.lineEnd) else "" + LineHit( + bookId = meta.bookId, + bookTitle = meta.bookTitle, + lineId = meta.lineId, + lineIndex = meta.lineIndex, + snippet = snippet, + score = boostedScore, + rawText = raw, + isBaseBook = meta.isBaseBook, + ) + } + // Re-sort by boosted score (descending) + return hits.sortedByDescending { it.score } + } + + // Snippet sources: from the provider if available, otherwise none (the indexed text_raw fallback) + private suspend fun snippetSources(lines: List): Map = + snippetProvider?.getSnippetSources(lines) ?: emptyMap() + + private suspend fun doSearch( + rawQuery: String, + near: Int, + limit: Int, + offset: Int, + bookFilter: Long?, + categoryFilter: Long?, + bookIds: Collection? = null, + ): List { + val context = buildSearchContext(rawQuery, near, bookFilter, categoryFilter, bookIds, null) ?: return emptyList() + return withContext(Dispatchers.IO) { + val top = index.search(context.query, offset + limit, HIT_FIELDS) + mapDocs(top.docs.drop(offset), context) + } + } + + private fun buildNgramPresenceForToken(token: String): IndexQuery? { + if (token.length < 4) return null + val grams = (0..token.length - 4).map { token.substring(it, it + 4) }.distinct() + if (grams.isEmpty()) return null + return BoolBuilder().apply { grams.forEach { add(IndexQuery.Term("text_ng4", it), Occur.MUST) } }.build() + } + + private fun buildPresenceFilterForTokens( + tokens: List, + near: Int, + expansionsByToken: Map>, + ): IndexQuery? { + if (tokens.isEmpty()) return null + val outer = BoolBuilder() + for (t in tokens) { + val expansions = expansionsByToken[t] ?: emptyList() + val synonymTerms = buildLimitedTermsForToken(t, expansions) + val ngram = if (near > 0) buildNgramPresenceForToken(t) else null + val clause = + BoolBuilder() + .apply { + add(IndexQuery.Term("text", t), Occur.SHOULD) + if (ngram != null) add(ngram, Occur.SHOULD) + for (term in synonymTerms) { + if (term != t) add(IndexQuery.Term("text", term), Occur.SHOULD) + } + }.build() + outer.add(clause, Occur.MUST) + } + return outer.build() + } + + /** + * Builds a strict, verbatim phrase query for a quoted segment: terms must appear in the + * given order and be adjacent (slop = 0), with NO dictionary expansion, fuzzy matching or + * n-gram substring matching. A single-token segment becomes an exact term query. + * + * Nikud and final letters are still normalised (the segment is normalised by the caller and + * tokenised like the index) so that the phrase matches the indexed form. + */ + private fun buildExactPhraseQuery(phraseNorm: String): IndexQuery? { + if (phraseNorm.isBlank() || index.analyze(phraseNorm).isEmpty()) return null + return IndexQuery.AnalyzedPhrase("text", phraseNorm, 0) + } + + private fun expansionTerms(expansions: List): Triple, List, List> { + val surfaceTerms = LinkedHashSet() + val variantTerms = LinkedHashSet() + val baseTerms = LinkedHashSet() + for (exp in expansions) { + surfaceTerms.addAll(exp.surface) + variantTerms.addAll(exp.variants) + baseTerms.addAll(exp.base) + } + return Triple( + surfaceTerms.take(MAX_SYNONYM_BOOST_TERMS), + variantTerms.take(MAX_SYNONYM_BOOST_TERMS), + baseTerms.take(MAX_SYNONYM_BOOST_TERMS), + ) + } + + private fun buildMagicBoostQuery(expansions: List): IndexQuery? { + if (expansions.isEmpty()) return null + val (surfaces, variants, bases) = expansionTerms(expansions) + val b = BoolBuilder() + for (s in surfaces) b.add(IndexQuery.Boost(IndexQuery.Term("text", s), 2.0f), Occur.SHOULD) + for (v in variants) b.add(IndexQuery.Boost(IndexQuery.Term("text", v), 1.5f), Occur.SHOULD) + for (ba in bases) b.add(IndexQuery.Boost(IndexQuery.Term("text", ba), 1.0f), Occur.SHOULD) + return b.build() + } + + private fun buildSynonymBoostQuery(expansions: List): IndexQuery? { + if (expansions.isEmpty()) return null + val (surfaces, variants, bases) = expansionTerms(expansions) + val b = BoolBuilder() + for (term in surfaces + variants + bases) b.add(IndexQuery.Term("text", term), Occur.SHOULD) + return b.build() + } + + private fun buildSynonymPhrases( + tokens: List, + expansionsByToken: Map>, + ): List> { + if (tokens.isEmpty()) return emptyList() + val termExpansions = buildTermAlternativesForTokens(tokens, expansionsByToken) + return listOf( + IndexQuery.MultiPhrase("text", termExpansions, 0) to 50.0f, + IndexQuery.MultiPhrase("text", termExpansions, 3) to 20.0f, + IndexQuery.MultiPhrase("text", termExpansions, 8) to 5.0f, + ) + } + + private fun buildSynonymPhraseQuery( + tokens: List, + expansionsByToken: Map>, + near: Int, + ): IndexQuery? { + if (tokens.isEmpty()) return null + return IndexQuery.MultiPhrase("text", buildTermAlternativesForTokens(tokens, expansionsByToken), near) + } + + private fun buildNgram4Query(norm: String): IndexQuery? { + val tokens = norm.split(WHITESPACE).map { it.trim() }.filter { it.length >= 4 } + if (tokens.isEmpty()) return null + val uniq = tokens.flatMap { t -> (0..t.length - 4).map { t.substring(it, it + 4) } }.distinct() + if (uniq.isEmpty()) return null + return BoolBuilder().apply { uniq.forEach { add(IndexQuery.Term("text_ng4", it), Occur.MUST) } }.build() + } + + private fun buildExpandedQuery( + norm: String, + near: Int, + analyzed: List, + tokens: List, + expansionsByToken: Map>, + ): IndexQuery { + val base = IndexQuery.AnalyzedPhrase("text", norm, near) + val allExpansions = expansionsByToken.values.flatten() + val builder = BoolBuilder() + builder.add(base, Occur.SHOULD) + for ((query, boost) in buildSynonymPhrases(tokens, expansionsByToken)) { + builder.add(IndexQuery.Boost(query, boost), Occur.SHOULD) + } + buildNgram4Query(norm)?.let { builder.add(it, Occur.SHOULD) } + buildFuzzyQuery(norm, near, analyzed)?.let { builder.add(it, Occur.SHOULD) } + buildMagicBoostQuery(allExpansions)?.let { builder.add(it, Occur.SHOULD) } + buildSynonymBoostQuery(allExpansions)?.let { builder.add(it, Occur.SHOULD) } + return builder.build() + } + + private fun buildFuzzyQuery( + norm: String, + near: Int, + analyzed: List, + ): IndexQuery? { + if (near == 0) return null + if (norm.length < 4) return null + val tokens = analyzed.filter { it.length >= 4 } + if (tokens.isEmpty()) return null + return BoolBuilder().apply { tokens.distinct().forEach { add(IndexQuery.Fuzzy("text", it, 1), Occur.MUST) } }.build() + } + + private fun buildLimitedTermsForToken( + token: String, + expansions: List, + ): List { + if (expansions.isEmpty()) return listOf(token) + + val baseTerms = expansions.flatMap { it.base }.distinct() + val otherTerms = expansions.flatMap { it.surface + it.variants }.distinct() + + val ordered = LinkedHashSet() + if (token.isNotBlank()) ordered += token + baseTerms.forEach { ordered += it } + otherTerms.forEach { ordered += it } + return ordered.take(MAX_SYNONYM_TERMS_PER_TOKEN) + } + + private fun buildTermAlternativesForTokens( + tokens: List, + expansionsByToken: Map>, + ): List> = tokens.map { token -> buildLimitedTermsForToken(token, expansionsByToken[token] ?: emptyList()) } + + // The divine name's forms never change: read once from the dictionary, not on every search + private val hashemHighlightTerms: List by lazy { readHashemHighlightTerms() } + + private fun readHashemHighlightTerms(): List { + val dict = magicDict ?: return emptyList() + val raw = dict.loadHashemSurfaces() + if (raw.isEmpty()) return emptyList() + + val terms = linkedSetOf() + raw.forEach { value -> + val trimmed = value.trim() + if (trimmed.isEmpty()) return@forEach + terms += trimmed + val stripped = HebrewTextUtils.stripDiacritics(trimmed).trim() + if (stripped.isNotEmpty()) terms += stripped + val normalized = HebrewTextUtils.normalizeHebrew(trimmed).trim() + if (normalized.isNotEmpty()) terms += normalized + } + return terms.toList() + } +} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt new file mode 100644 index 00000000..a2a3da18 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LruCache.kt @@ -0,0 +1,48 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized + +/** A thread-safe cache of the [capacity] most recently used entries. */ +internal class LruCache( + private val capacity: Int, +) { + private val lock = SynchronizedObject() + + // Insertion order is kept: a hit moves its entry to the end, so the first entry is the least recently used + private val map = LinkedHashMap() + + operator fun get(key: K): V? = + synchronized(lock) { + map.remove(key)?.also { map[key] = it } + } + + operator fun set( + key: K, + value: V, + ) { + synchronized(lock) { putLocked(key, value) } + } + + /** The value of [key], else [create]'s, stored; [create] runs under the lock, so it must be cheap. */ + fun getOrPut( + key: K, + create: () -> V, + ): V = + synchronized(lock) { + map.remove(key)?.also { map[key] = it } ?: create().also { putLocked(key, it) } + } + + fun clear() { + synchronized(lock) { map.clear() } + } + + private fun putLocked( + key: K, + value: V, + ) { + map.remove(key) + map[key] = value + if (map.size > capacity) map.remove(map.keys.first()) + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt similarity index 51% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt index c39f5a99..96d3bee0 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/MagicDictionaryIndex.kt @@ -1,15 +1,14 @@ package io.github.kdroidfilter.seforimlibrary.search import co.touchlab.kermit.Logger -import java.nio.file.Files -import java.nio.file.Path -import java.sql.Connection -import java.sql.DriverManager -import java.sql.PreparedStatement -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem /** * Streaming dictionary index backed by SQLite (tables: surface, variant, base). @@ -19,8 +18,8 @@ import java.util.concurrent.atomic.AtomicInteger */ class MagicDictionaryIndex private constructor( private val norm: (String) -> String, - private val dbFile: Path -) { + private val db: SqliteReader, +) : AutoCloseable { /** * The forms a token expands to. [surface] holds every form (inflections, then the dictionary's variants, then * the base) as the ranking uses them; [inflections] only the base's inflected forms and the base itself: the @@ -30,54 +29,30 @@ class MagicDictionaryIndex private constructor( val surface: List, val variants: List, val base: List, - val inflections: List = emptyList() + val inflections: List = emptyList(), ) - private val url = "jdbc:sqlite:${dbFile.toAbsolutePath()}" - - /** - * Prepared statement per thread to avoid re-opening connections on every token. - */ - private val stmtProvider: ThreadLocal = ThreadLocal.withInitial { - val conn = DriverManager.getConnection(url).apply { - autoCommit = false - // Enforce read-only queries without altering connection flags post-open - createStatement().use { stmt -> stmt.execute("PRAGMA query_only=ON") } - } - // Dictionaries since v0.4.2 list the extra lemmas of ambiguous surfaces (פנינו: פנה and פנים) - val sql = if (conn.hasTable(EXTRA_BASE_TABLE)) LOOKUP_WITH_EXTRA_BASES_SQL else LOOKUP_SQL - LookupContext( - conn = conn, - stmt = conn.prepareStatement(sql), - paramCount = sql.count { it == '?' } - ) - } + // Dictionaries since v0.4.2 list the extra lemmas of ambiguous surfaces (פנינו: פנה and פנים) + private val lookupSql: String by lazy { if (db.hasTable(EXTRA_BASE_TABLE)) LOOKUP_WITH_EXTRA_BASES_SQL else LOOKUP_SQL } /** * Cache expansions per normalized token to avoid repeated DB hits. Lookups in progress are cached too, so * concurrent searches (facets and results start together) share them. */ - private val tokenCache = object : LinkedHashMap>>(TOKEN_CACHE_SIZE, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry>>?): Boolean = - size > TOKEN_CACHE_SIZE - } + private val tokenCache = LruCache>>(TOKEN_CACHE_SIZE) /** * Cache fully-normalized expansions per base id so repeated hits to the same base * avoid re-normalizing rows. */ - private val baseCache = object : LinkedHashMap(BASE_CACHE_SIZE, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = - size > BASE_CACHE_SIZE - } + private val baseCache = LruCache(BASE_CACHE_SIZE) /** Starts looking up [tokens] in parallel, so the following [expansionFor] calls find them ready. */ fun prefetch(tokens: Collection) { tokens.forEach { lookup(it, async = true) } } - fun expansionsFor(tokens: List): List = - tokens.flatMap { expansionsForToken(it) }.distinct() + fun expansionsFor(tokens: List): List = tokens.flatMap { expansionsForToken(it) }.distinct() fun expansionFor(token: String): Expansion? { val expansions = expansionsForToken(token) @@ -85,9 +60,10 @@ class MagicDictionaryIndex private constructor( val normalized = norm(token) // Strategy: prefer the expansion whose base matches the token - val matchingBase = expansions.firstOrNull { exp -> - exp.base.any { it == normalized } - } + val matchingBase = + expansions.firstOrNull { exp -> + exp.base.any { it == normalized } + } if (matchingBase != null) return matchingBase // Then the entries the token truly is a form of: one reached only through a dictionary variant (loose @@ -98,25 +74,35 @@ class MagicDictionaryIndex private constructor( return ownEntries.ifEmpty { expansions }.maxByOrNull { it.surface.size } } - private fun expansionsForToken(token: String): List = lookup(token, async = false).join() + private fun expansionsForToken(token: String): List { + val result = lookup(token, async = false) + @Suppress("OPT_IN_USAGE") + return if (result.isCompleted) result.getCompleted() else runBlockingIo { result.await() } + } - private fun lookup(token: String, async: Boolean): CompletableFuture> { + private fun lookup( + token: String, + async: Boolean, + ): CompletableDeferred> { val normalized = norm(token) - if (normalized.isEmpty()) return CompletableFuture.completedFuture(emptyList()) - val future = CompletableFuture>() - synchronized(tokenCache) { - tokenCache[normalized]?.let { return it } - tokenCache[normalized] = future - } - val fetch = Runnable { - // fetchExpansions logs and skips its failures: an empty result is cached like any other - future.complete(runCatching { fetchForToken(token, normalized) }.getOrDefault(emptyList())) - } - if (async) lookupExecutor.execute(fetch) else fetch.run() - return future + if (normalized.isEmpty()) return CompletableDeferred(emptyList()) + var created = false + val result = + tokenCache.getOrPut(normalized) { + created = true + CompletableDeferred() + } + if (!created) return result + // fetchExpansions logs and skips its failures: an empty result is cached like any other + val fetch = { result.complete(runCatching { fetchForToken(token, normalized) }.getOrDefault(emptyList())) } + if (async) lookupScope.launch { fetch() } else fetch() + return result } - private fun fetchForToken(token: String, normalized: String): List { + private fun fetchForToken( + token: String, + normalized: String, + ): List { // Try raw, normalized, and final-form variants to match DB values. val candidates = buildLookupCandidates(token, normalized) val mergedByBase = LinkedHashMap() @@ -142,57 +128,58 @@ class MagicDictionaryIndex private constructor( /** * Fetch expansions for a token. Returns a list of (baseId, Expansion) pairs so callers can merge by base id. */ - private fun fetchExpansions(rawToken: String, normalizedToken: String): List> { + private fun fetchExpansions( + rawToken: String, + normalizedToken: String, + ): List> { val expansions = mutableListOf>() - val ctx = stmtProvider.get() runCatching { - synchronized(ctx) { - repeat(ctx.paramCount) { idx -> ctx.stmt.setString(idx + 1, rawToken) } - val rs = ctx.stmt.executeQuery() - val accum = mutableMapOf() - while (rs.next()) { - val baseId = rs.getLong("base_id") - val bucket = accum.getOrPut(baseId) { + val sql = lookupSql + val rows = db.query(sql, List(sql.count { it == '?' }) { rawToken }) + val accum = LinkedHashMap() + for (row in rows) { + val baseId = row["base_id"]?.toLongOrNull() ?: continue + val bucket = + accum.getOrPut(baseId) { BaseBucket( - baseRaw = rs.getString("base") ?: "", + baseRaw = row["base"] ?: "", surfaces = linkedSetOf(), - variants = linkedSetOf() + variants = linkedSetOf(), ) } - rs.getString("surface")?.let { bucket.surfaces += it } - rs.getString("variant")?.let { bucket.variants += it } - } + row["surface"]?.let { bucket.surfaces += it } + row["variant"]?.let { bucket.variants += it } + } - for ((baseId, bucket) in accum) { - val cached = synchronized(baseCache) { baseCache[baseId] } - if (cached != null) { - expansions += baseId to cached - continue - } + for ((baseId, bucket) in accum) { + val cached = baseCache[baseId] + if (cached != null) { + expansions += baseId to cached + continue + } - val surfaceN = bucket.surfaces.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } - val variantsN = bucket.variants.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } - val baseN = norm(bucket.baseRaw).takeIf { it.isNotEmpty() } + val surfaceN = bucket.surfaces.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } + val variantsN = bucket.variants.mapNotNull { v -> norm(v).takeIf { it.isNotEmpty() } } + val baseN = + norm(bucket.baseRaw).takeIf { it.isNotEmpty() } ?: surfaceN.firstOrNull() ?: normalizedToken - val baseTerms = listOfNotNull(baseN.takeIf { it.isNotEmpty() }) - val allTerms = (surfaceN + variantsN + baseTerms).distinct() - if (allTerms.isEmpty()) continue + val baseTerms = listOfNotNull(baseN.takeIf { it.isNotEmpty() }) + val allTerms = (surfaceN + variantsN + baseTerms).distinct() + if (allTerms.isEmpty()) continue - val exp = Expansion( + val exp = + Expansion( surface = allTerms, variants = emptyList(), base = baseTerms, - inflections = (surfaceN + baseTerms).distinct() + inflections = (surfaceN + baseTerms).distinct(), ) - synchronized(baseCache) { - baseCache[baseId] = exp - } - expansions += baseId to exp - } + baseCache[baseId] = exp + expansions += baseId to exp } }.onFailure { logger.d { "[MagicDictionary] Failed to fetch expansions for '$rawToken' : ${it.message}" } @@ -201,76 +188,100 @@ class MagicDictionaryIndex private constructor( return expansions } + /** + * Load all surface forms whose base lemma directly from the underlying SQLite DB. + * This is used for snippet highlighting of Hashem names, independent of token-level expansions. + */ + fun loadHashemSurfaces(): List { + val terms = linkedSetOf() + runCatching { + val sql = + """ + SELECT s.value AS surface + FROM surface s + JOIN base b ON s.base_id = b.id + WHERE b.value = 'יהוה' + """.trimIndent() + for (row in db.query(sql)) { + val trimmed = row["surface"]?.trim() ?: continue + if (trimmed.isNotEmpty()) terms += trimmed + } + }.onFailure { + logger.d { "[MagicDictionary] Failed to load Hashem surfaces: ${it.message}" } + } + return terms.toList() + } + + override fun close() { + db.close() + } + companion object { private val logger = Logger.withTag("MagicDictionary") private const val TOKEN_CACHE_SIZE = 1024 - - // Parallel lookups of a query's words (each thread keeps its own connection) - private val lookupExecutor: ExecutorService by lazy { - val ids = AtomicInteger() - Executors.newFixedThreadPool(minOf(Runtime.getRuntime().availableProcessors(), 4)) { task -> - Thread(task, "magic-dictionary-${ids.incrementAndGet()}").apply { isDaemon = true } - } - } private const val BASE_CACHE_SIZE = 512 + private const val LOOKUP_PARALLELISM = 4 + + // Parallel lookups of a query's words + private val lookupScope = CoroutineScope(SupervisorJob() + Dispatchers.IO.limitedParallelism(LOOKUP_PARALLELISM)) + + private val FINALS = mapOf('כ' to 'ך', 'מ' to 'ם', 'נ' to 'ן', 'פ' to 'ף', 'צ' to 'ץ') + + private val REQUIRED_TABLES = listOf("surface", "variant", "base", "surface_variant") /** * Load from SQLite DB (expected tables: surface(value, base_id), variant(value, surface_id), base(value)). * Uses streaming lookup to avoid holding the entire dictionary in memory. */ - fun load(norm: (String) -> String, candidate: Path?): MagicDictionaryIndex? { - val file = candidate?.takeIf { Files.isRegularFile(it) && hasRequiredTables(it) } ?: run { - if (candidate != null) { - logger.d { "[MagicDictionary] Ignoring candidate $candidate because required tables are missing" } - } - return null - } + fun load( + norm: (String) -> String, + candidate: String?, + ): MagicDictionaryIndex? { + if (candidate == null || !isRegularFile(candidate)) return null return runCatching { - // Validate DB is reachable - DriverManager.getConnection("jdbc:sqlite:${file.toAbsolutePath()}").use { conn -> - conn.createStatement().use { stmt -> - stmt.execute("SELECT 1") - } + val db = openReadOnlySqlite(candidate) + if (!db.hasRequiredTables()) { + db.close() + logger.d { "[MagicDictionary] Ignoring candidate $candidate because required tables are missing" } + return null } - logger.d { "[MagicDictionary] Streaming lexical db from $file (lazy on-demand)" } - MagicDictionaryIndex(norm, file) + logger.d { "[MagicDictionary] Streaming lexical db from $candidate (lazy on-demand)" } + MagicDictionaryIndex(norm, db) }.onFailure { - logger.d { "[MagicDictionary] Failed to load from $file : ${it.message}" } + logger.d { "[MagicDictionary] Failed to load from $candidate : ${it.message}" } }.getOrNull() } /** * Find the first candidate path that exists and contains the required tables. */ - fun findValidDictionary(candidates: List): Path? { + fun findValidDictionary(candidates: List): String? { for (candidate in candidates) { - if (!Files.isRegularFile(candidate)) continue - if (hasRequiredTables(candidate)) { + if (!isRegularFile(candidate)) continue + val valid = runCatching { openReadOnlySqlite(candidate).use { it.hasRequiredTables() } }.getOrDefault(false) + if (valid) { logger.d { "[MagicDictionary] Using validated lexical db at $candidate" } return candidate - } else { - logger.d { - "[MagicDictionary] Candidate $candidate is present but missing required tables; skipping" - } } + logger.d { "[MagicDictionary] Candidate $candidate is present but missing required tables; skipping" } } return null } - private fun hasRequiredTables(file: Path): Boolean = runCatching { - DriverManager.getConnection("jdbc:sqlite:${file.toAbsolutePath()}").use { conn -> - val sql = """ - SELECT name FROM sqlite_master - WHERE type = 'table' AND name IN ('surface', 'variant', 'base', 'surface_variant') - """.trimIndent() - conn.createStatement().use { stmt -> - val rs = stmt.executeQuery(sql) - val names = mutableSetOf() - while (rs.next()) names += rs.getString("name") ?: "" - names.containsAll(listOf("surface", "variant", "base", "surface_variant")) - } - } - }.getOrElse { false } + private fun isRegularFile(path: String): Boolean = SystemFileSystem.metadataOrNull(Path(path))?.isRegularFile == true + + private fun SqliteReader.hasRequiredTables(): Boolean = + runCatching { + val names = + query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN " + + REQUIRED_TABLES.joinToString(prefix = "(", postfix = ")") { "'$it'" }, + ).mapNotNull { it["name"] } + names.containsAll(REQUIRED_TABLES) + }.getOrDefault(false) + + private fun SqliteReader.hasTable(name: String): Boolean = + query("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = ?", listOf(name)).isNotEmpty() private const val LOOKUP_SQL = """ WITH matches AS ( @@ -332,39 +343,22 @@ class MagicDictionaryIndex private constructor( LEFT JOIN surface_variant sv ON sv.surface_id = s.id AND mb.own = 1 LEFT JOIN variant v ON sv.variant_id = v.id """ - - private fun Connection.hasTable(name: String): Boolean = - prepareStatement("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").use { stmt -> - stmt.setString(1, name) - stmt.executeQuery().use { it.next() } - } } - private data class LookupContext( - val conn: Connection, - val stmt: PreparedStatement, - val paramCount: Int - ) - - private data class BaseBucket( + private class BaseBucket( val baseRaw: String, val surfaces: MutableSet, - val variants: MutableSet + val variants: MutableSet, ) - private fun buildLookupCandidates(rawToken: String, normalized: String): List { - val finalsMap = mapOf( - 'כ' to 'ך', - 'מ' to 'ם', - 'נ' to 'ן', - 'פ' to 'ף', - 'צ' to 'ץ' - ) - + private fun buildLookupCandidates( + rawToken: String, + normalized: String, + ): List { fun applyFinalForm(t: String): String { if (t.isEmpty()) return t val last = t.last() - val final = finalsMap[last] ?: last + val final = FINALS[last] ?: last return if (final == last) t else t.dropLast(1) + final } @@ -372,38 +366,7 @@ class MagicDictionaryIndex private constructor( rawToken, normalized, applyFinalForm(rawToken), - applyFinalForm(normalized) + applyFinalForm(normalized), ).filter { it.isNotBlank() }.distinct() } - - /** - * Load all surface forms whose base lemma directly from the underlying SQLite DB. - * This is used for snippet highlighting of Hashem names, independent of token-level expansions. - */ - fun loadHashemSurfaces(): List { - val terms = linkedSetOf() - runCatching { - DriverManager.getConnection(url).use { conn -> - val sql = """ - SELECT s.value AS surface - FROM surface s - JOIN base b ON s.base_id = b.id - WHERE b.value = 'יהוה' - """.trimIndent() - conn.createStatement().use { stmt -> - val rs = stmt.executeQuery(sql) - while (rs.next()) { - val v = rs.getString("surface") ?: continue - val trimmed = v.trim() - if (trimmed.isNotEmpty()) { - terms += trimmed - } - } - } - } - }.onFailure { - logger.d { "[MagicDictionary] Failed to load Hashem surfaces: ${it.message}" } - } - return terms.toList() - } } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt new file mode 100644 index 00000000..baae6173 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.kt @@ -0,0 +1,26 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.coroutines.CoroutineScope + +/** + * The text of [html] as snippet text is built from: tags dropped, special characters escaped as HTML entities (what + * `Jsoup.clean(html, Safelist.none())` returns, which the JVM and Android keep using). + */ +internal expect fun escapedText(html: String): String + +/** Runs [block] to completion on the calling thread, for the synchronous parts of the API that wait on I/O. */ +internal expect fun runBlockingIo(block: suspend CoroutineScope.() -> T): T + +/** + * A read-only SQLite database (the lexical dictionary). Safe to use from several threads at once. + */ +interface SqliteReader : AutoCloseable { + /** The rows [sql] returns with [args] bound in order, each as its column names to their text values. */ + fun query( + sql: String, + args: List = emptyList(), + ): List> +} + +/** Opens the SQLite database at [path] read-only; throws when it can't be opened. */ +expect fun openReadOnlySqlite(path: String): SqliteReader diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/QueryHighlighter.kt diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt similarity index 99% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt index 64cc5d0d..4445951f 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchEngine.kt @@ -1,6 +1,5 @@ package io.github.kdroidfilter.seforimlibrary.search -import java.io.Closeable /** * Main interface for full-text search operations on Hebrew religious texts. @@ -31,7 +30,7 @@ import java.io.Closeable * @see SearchSession for paginated result access * @see LineHit for individual search result structure */ -interface SearchEngine : Closeable { +interface SearchEngine : AutoCloseable { /** * Opens a search session for the given query with optional filters. diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt similarity index 97% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt index 2226885b..7d52443e 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchSession.kt @@ -1,6 +1,7 @@ package io.github.kdroidfilter.seforimlibrary.search -import java.io.Closeable +import kotlinx.serialization.Serializable + /** * A stateful search session providing paginated access to search results. @@ -26,7 +27,7 @@ import java.io.Closeable * @see SearchEngine.openSession to create a session * @see SearchPage for the structure of returned pages */ -interface SearchSession : Closeable { +interface SearchSession : AutoCloseable { /** * Retrieves the next page of search results. * @@ -100,6 +101,7 @@ data class LineHit( * @property categoryCounts Map of categoryId to count (includes ancestor categories) * @property bookCounts Map of bookId to count */ +@Serializable data class SearchFacets( val totalHits: Long, val categoryCounts: Map, diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt similarity index 100% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetBuilder.kt diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt similarity index 96% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt rename to search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt index 858dc31b..290184b8 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SnippetProvider.kt @@ -1,8 +1,5 @@ package io.github.kdroidfilter.seforimlibrary.search -import org.jsoup.Jsoup -import org.jsoup.safety.Safelist - /** * Metadata about a line needed to fetch its snippet source text. * @@ -98,7 +95,7 @@ object SnippetSources { } /** [html] as snippet text: tags dropped, entities escaped, whitespace collapsed. */ - fun clean(html: String): String = Jsoup.clean(html, Safelist.none()).replace(WHITESPACE, " ").trim() + fun clean(html: String): String = escapedText(html).replace(WHITESPACE, " ").trim() private fun windows(lineIndexes: List): List { val out = ArrayList() diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt new file mode 100644 index 00000000..c695249d --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt @@ -0,0 +1,114 @@ +package io.github.kdroidfilter.seforimlibrary.search.index + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * A Lucene query as plain data: the engines build it in common code, a [LuceneIndex] turns it into Lucene's own + * query, in this process (JVM, Android) or in the Java runtime the iOS app embeds. + */ +@Serializable +sealed interface IndexQuery { + /** Documents whose [field] holds the term [text]. */ + @Serializable + @SerialName("term") + data class Term( + val field: String, + val text: String, + ) : IndexQuery + + /** Documents whose [field] holds a term starting with [text]. */ + @Serializable + @SerialName("prefix") + data class Prefix( + val field: String, + val text: String, + ) : IndexQuery + + /** Documents whose [field] holds a term within [maxEdits] edits of [text]. */ + @Serializable + @SerialName("fuzzy") + data class Fuzzy( + val field: String, + val text: String, + val maxEdits: Int, + ) : IndexQuery + + @Serializable + @SerialName("bool") + data class Bool( + val clauses: List, + ) : IndexQuery + + @Serializable + @SerialName("boost") + data class Boost( + val query: IndexQuery, + val boost: Float, + ) : IndexQuery + + /** Each position matches any of its terms, within [slop] moves. */ + @Serializable + @SerialName("multiPhrase") + data class MultiPhrase( + val field: String, + val positions: List>, + val slop: Int, + ) : IndexQuery + + /** + * [text] analyzed like the index (StandardAnalyzer) into a phrase within [slop] moves (a term query for one term), + * every term required when the analysis can't build a phrase, and nothing when it yields no term. + */ + @Serializable + @SerialName("analyzedPhrase") + data class AnalyzedPhrase( + val field: String, + val text: String, + val slop: Int, + ) : IndexQuery + + /** Documents whose int point [field] is [value]. */ + @Serializable + @SerialName("intExact") + data class IntExact( + val field: String, + val value: Int, + ) : IndexQuery + + /** Documents whose int point [field] is one of [values]. */ + @Serializable + @SerialName("intSet") + class IntSet( + val field: String, + val values: IntArray, + ) : IndexQuery + + /** Every document. */ + @Serializable + @SerialName("all") + data object MatchAll : IndexQuery +} + +@Serializable +data class Clause( + val query: IndexQuery, + val occur: Occur, +) + +@Serializable +enum class Occur { MUST, SHOULD, FILTER, MUST_NOT } + +/** A [IndexQuery.Bool] built clause by clause. */ +class BoolBuilder { + private val clauses = ArrayList() + + fun add( + query: IndexQuery, + occur: Occur, + ): BoolBuilder = apply { clauses += Clause(query, occur) } + + fun build(): IndexQuery.Bool = IndexQuery.Bool(clauses.toList()) +} + +inline fun bool(block: BoolBuilder.() -> Unit): IndexQuery.Bool = BoolBuilder().apply(block).build() diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt new file mode 100644 index 00000000..735cc39b --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt @@ -0,0 +1,97 @@ +package io.github.kdroidfilter.seforimlibrary.search.index + +import io.github.kdroidfilter.seforimlibrary.search.SearchFacets +import kotlinx.serialization.Serializable + +/** + * The Lucene index of a search, read only: the only part of the search that runs Lucene. The JVM and Android read it in + * process; iOS through the Java runtime it embeds (Lucene needs a JVM), with the same implementation. Everything else + * (query building, dictionary, ranking, snippets, highlighting, fusion) is common code. + * + * Implementations are thread-safe; a [IndexCursor] is used by one caller at a time. + */ +interface LuceneIndex : AutoCloseable { + /** The terms of [text] as the index's analyzer (StandardAnalyzer, the `text` field) splits it. */ + fun analyze(text: String): List + + /** The best [limit] documents for [query] (by score, or by [sort]), with their stored [fields]. */ + fun search( + query: IndexQuery, + limit: Int, + fields: Set, + sort: List? = null, + ): IndexPage + + /** Pages through the documents of [query] by score, on the index as it is now, with their stored [fields]. */ + fun openCursor( + query: IndexQuery, + fields: Set, + ): IndexCursor + + /** How the documents of [query] spread over the books and categories (`book_id`, `ancestor_category_ids`). */ + fun facets(query: IndexQuery): SearchFacets + + /** The stored numeric [field] of every document of [query], unscored and in no particular order. */ + fun collectLongs( + query: IndexQuery, + field: String, + ): LongArray + + fun count(query: IndexQuery): Int + + /** How the vectors of [field] are stored, null when no segment has the field. */ + fun vectorEncoding(field: String): VectorEncoding? + + /** The nearest documents to a vector (see [KnnRequest]), best first, with their stored [KnnRequest.fields]. */ + fun knn(request: KnnRequest): List +} + +/** A page of a [LuceneIndex.openCursor]. */ +interface IndexCursor : AutoCloseable { + /** The next [limit] documents, none once all were read. */ + fun next(limit: Int): IndexPage +} + +/** A document found: its score and the stored fields asked for, numeric ones in [longs], the others in [strings]. */ +@Serializable +class IndexDoc( + val score: Float, + val longs: Map = emptyMap(), + val strings: Map = emptyMap(), +) + +/** Documents found, with the number of matches ([totalHits] may be a lower bound, as Lucene counts them). */ +@Serializable +class IndexPage( + val docs: List, + val totalHits: Long, +) + +@Serializable +data class SortField( + val field: String, + val type: Type, + val reverse: Boolean = false, +) { + @Serializable + enum class Type { INT, LONG } +} + +@Serializable +enum class VectorEncoding { FLOAT32, BYTE } + +/** + * A nearest neighbors search of [field]: with [byteVector] on a byte field, else with [floatVector]; on a float field + * quantized by Lucene, [oversample] times more candidates are gathered, then the best [k] kept by their exact + * similarity. [filter] restricts the documents considered. + */ +@Serializable +class KnnRequest( + val field: String, + val k: Int, + val fields: Set, + val floatVector: FloatArray? = null, + val byteVector: ByteArray? = null, + val filter: IndexQuery? = null, + val oversample: Int = 1, +) diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt new file mode 100644 index 00000000..070e8f5b --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt @@ -0,0 +1,117 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest +import io.github.kdroidfilter.seforimlibrary.search.index.SortField +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement + +/** + * The calls of a [io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex] across the bridge between the iOS + * app and the Java runtime it embeds: JSON in UTF-8 both ways, each index and cursor named by a handle. + */ +@Serializable +internal sealed interface BridgeRequest { + @Serializable + @SerialName("open") + data class Open( + val indexDir: String, + val cacheQueries: Boolean, + ) : BridgeRequest + + @Serializable + @SerialName("close") + data class Close( + val index: Int, + ) : BridgeRequest + + @Serializable + @SerialName("analyze") + data class Analyze( + val index: Int, + val text: String, + ) : BridgeRequest + + @Serializable + @SerialName("search") + data class Search( + val index: Int, + val query: IndexQuery, + val limit: Int, + val fields: Set, + val sort: List? = null, + ) : BridgeRequest + + @Serializable + @SerialName("openCursor") + data class OpenCursor( + val index: Int, + val query: IndexQuery, + val fields: Set, + ) : BridgeRequest + + @Serializable + @SerialName("next") + data class CursorNext( + val cursor: Int, + val limit: Int, + ) : BridgeRequest + + @Serializable + @SerialName("closeCursor") + data class CloseCursor( + val cursor: Int, + ) : BridgeRequest + + @Serializable + @SerialName("facets") + data class Facets( + val index: Int, + val query: IndexQuery, + ) : BridgeRequest + + @Serializable + @SerialName("collectLongs") + data class CollectLongs( + val index: Int, + val query: IndexQuery, + val field: String, + ) : BridgeRequest + + @Serializable + @SerialName("count") + data class Count( + val index: Int, + val query: IndexQuery, + ) : BridgeRequest + + @Serializable + @SerialName("vectorEncoding") + data class VectorEncodingOf( + val index: Int, + val field: String, + ) : BridgeRequest + + @Serializable + @SerialName("knn") + class Knn( + val index: Int, + val request: KnnRequest, + ) : BridgeRequest +} + +/** The result of a call ([result], its type depending on the call), or the failure it raised ([error]). */ +@Serializable +internal class BridgeResponse( + val result: JsonElement? = null, + val error: String? = null, +) + +internal val BridgeJson = + Json { + ignoreUnknownKeys = true + // Scores and vectors are plain floats; a NaN would mean a bug on the Lucene side, not data to carry + allowSpecialFloatingPointValues = false + } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt new file mode 100644 index 00000000..8783a1d5 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt @@ -0,0 +1,97 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.SearchFacets +import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor +import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc +import io.github.kdroidfilter.seforimlibrary.search.index.IndexPage +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.SortField +import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.builtins.LongArraySerializer +import kotlinx.serialization.builtins.nullable +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull + +/** + * How the calls reach the Java runtime Lucene runs in: one [BridgeRequest] in, its [BridgeResponse] out, both UTF-8 JSON. + * Thread-safe: calls may overlap. + */ +fun interface BridgeTransport { + fun call(request: ByteArray): ByteArray +} + +/** + * The [LuceneIndex] at [indexDir] where Lucene can't run in process: in the Java runtime an iOS app embeds, in the Java + * process an Android app before 15 starts. Lucene runs there with the same implementation as on the JVM and Android 15+ + * (`LuceneIndexBackend`, behind `IndexBridgeServer`); this side only carries the calls, through [transport]. + * + * @param cacheQueries see `LuceneIndexBackend` + */ +class BridgedLuceneIndex( + private val transport: BridgeTransport, + indexDir: String, + cacheQueries: Boolean = false, +) : LuceneIndex { + private val handle: Int = call(BridgeRequest.Open(indexDir, cacheQueries), Int.serializer()) + + override fun analyze(text: String): List = call(BridgeRequest.Analyze(handle, text), ListSerializer(String.serializer())) + + override fun search( + query: IndexQuery, + limit: Int, + fields: Set, + sort: List?, + ): IndexPage = call(BridgeRequest.Search(handle, query, limit, fields, sort), IndexPage.serializer()) + + override fun openCursor( + query: IndexQuery, + fields: Set, + ): IndexCursor { + val cursor = call(BridgeRequest.OpenCursor(handle, query, fields), Int.serializer()) + return object : IndexCursor { + private var closed = false + + override fun next(limit: Int): IndexPage = call(BridgeRequest.CursorNext(cursor, limit), IndexPage.serializer()) + + override fun close() { + if (closed) return + closed = true + call(BridgeRequest.CloseCursor(cursor), null) + } + } + } + + override fun facets(query: IndexQuery): SearchFacets = call(BridgeRequest.Facets(handle, query), SearchFacets.serializer()) + + override fun collectLongs( + query: IndexQuery, + field: String, + ): LongArray = call(BridgeRequest.CollectLongs(handle, query, field), LongArraySerializer()) + + override fun count(query: IndexQuery): Int = call(BridgeRequest.Count(handle, query), Int.serializer()) + + override fun vectorEncoding(field: String): VectorEncoding? = + call(BridgeRequest.VectorEncodingOf(handle, field), VectorEncoding.serializer().nullable) + + override fun knn(request: KnnRequest): List = call(BridgeRequest.Knn(handle, request), ListSerializer(IndexDoc.serializer())) + + override fun close() { + call(BridgeRequest.Close(handle), null) + } + + private fun call( + request: BridgeRequest, + result: DeserializationStrategy?, + ): T { + val bytes = transport.call(BridgeJson.encodeToString(BridgeRequest.serializer(), request).encodeToByteArray()) + val response = BridgeJson.decodeFromString(BridgeResponse.serializer(), bytes.decodeToString()) + response.error?.let { throw IllegalStateException("Lucene failed in the Java runtime: $it") } + @Suppress("UNCHECKED_CAST") + return if (result == null) Unit as T else BridgeJson.decodeFromJsonElement(result, response.result ?: JsonNull as JsonElement) + } +} diff --git a/search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt b/search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt new file mode 100644 index 00000000..9575b929 --- /dev/null +++ b/search/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchDump.kt @@ -0,0 +1,72 @@ +package io.github.kdroidfilter.seforimlibrary.search + +/** + * A text dump of what a search engine answers for [PARITY_QUERIES]: facets, ranked hits with their snippets, + * highlights, title suggestions, find-in-book candidates. The same index gives the same dump on every platform (the + * JVM, Android, iOS through its Java runtime): the parity tests compare them. + */ +object SearchDump { + val PARITY_QUERIES = + WARMUP_QUERIES + + listOf( + "שבת", + "בראשית ברא", + "רבי עקיבא", + "\"ואהבת לרעך כמוך\"", + "תפילין של ראש", + "משה רבינו", + "רש״י", + "ה׳ אחד", + "קידוש השם", + "גמילות חסדים", + "וַיֹּאמֶר אֱלֹהִים", + ) + + private const val PAGE_SIZE = 20 + private const val TOP_FACETS = 5 + + suspend fun dump(engine: SearchEngine): String = + buildString { + for (query in PARITY_QUERIES) { + appendLine("## $query") + for (baseBookOnly in listOf(false, true)) { + val facets = engine.computeFacets(query, baseBookOnly = baseBookOnly) + appendLine("facets[base=$baseBookOnly] total=${facets?.totalHits}") + facets?.categoryCounts?.entries?.sortedWith(compareByDescending> { it.value }.thenBy { it.key }) + ?.take(TOP_FACETS) + ?.forEach { appendLine(" category ${it.key}=${it.value}") } + facets?.bookCounts?.entries?.sortedWith(compareByDescending> { it.value }.thenBy { it.key }) + ?.take(TOP_FACETS) + ?.forEach { appendLine(" book ${it.key}=${it.value}") } + engine.openSession(query, baseBookOnly = baseBookOnly)?.use { session -> + repeat(2) { pageIndex -> + val page = session.nextPage(PAGE_SIZE) ?: return@repeat + appendLine("page $pageIndex total=${page.totalHits} last=${page.isLastPage}") + for (hit in page.hits) { + appendLine(" ${hit.lineId} book=${hit.bookId} line=${hit.lineIndex} base=${hit.isBaseBook} score=${hit.score.format()}") + appendLine(" ${hit.snippet}") + } + } + } ?: appendLine("no session") + } + val sample = "בראשית ברא אלהים את השמים ואת הארץ, ויאמר ה׳ אחד ושבת קודש" + appendLine("highlight ${engine.highlightRanges(sample, query)}") + appendLine("snippet ${engine.buildSnippet(sample, query, 5)}") + } + appendLine("## titles") + for (prefix in listOf("בראש", "משנה ב", "שולחן")) appendLine("$prefix -> ${engine.searchBooksByTitlePrefix(prefix, 10)}") + appendLine("## find in book") + val firstBook = engine.openSession("שבת", baseBookOnly = true)?.use { it.nextPage(1)?.hits?.firstOrNull()?.bookId } + if (firstBook != null) { + for (query in listOf("שבת", "ויכלו השמים", "ים")) { + appendLine("$query in $firstBook -> ${engine.findInBookCandidates(query, firstBook)?.sorted()?.take(30)}") + } + } + } + + // Rounded: the scores are computed by the same Lucene everywhere, the rounding only keeps the dump readable + private fun Float.format(): String { + val scaled = kotlin.math.round(this * 1000).toLong() + return "${scaled / 1000}.${(scaled % 1000).toString().padStart(3, '0')}" + } +} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt new file mode 100644 index 00000000..4d127938 --- /dev/null +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Platform.ios.kt @@ -0,0 +1,11 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.safety.Safelist +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.runBlocking + +// Ksoup is jsoup's Kotlin port: the same cleaning as the JVM and Android +internal actual fun escapedText(html: String): String = Ksoup.clean(html, Safelist.none()) + +internal actual fun runBlockingIo(block: suspend CoroutineScope.() -> T): T = runBlocking(block = block) diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt new file mode 100644 index 00000000..d44b8ac2 --- /dev/null +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt @@ -0,0 +1,80 @@ +@file:OptIn(ExperimentalForeignApi::class) + +package io.github.kdroidfilter.seforimlibrary.search + +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OK +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OPEN_FULLMUTEX +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OPEN_READONLY +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_ROW +import cnames.structs.sqlite3 +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_bind_text +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_close_v2 +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_column_count +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_column_name +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_column_text +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_errmsg +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_finalize +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_open_v2 +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_prepare_v2 +import cnames.structs.sqlite3_stmt +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_step +import kotlinx.cinterop.CFunction +import kotlinx.cinterop.COpaquePointer +import kotlinx.cinterop.CPointer +import kotlinx.cinterop.CPointerVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.alloc +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.toCPointer +import kotlinx.cinterop.toKString +import kotlinx.cinterop.value + +/** Through the system SQLite, one connection in serialized mode: its queries are safe from any thread. */ +actual fun openReadOnlySqlite(path: String): SqliteReader = NativeSqliteReader(path) + +// SQLITE_TRANSIENT: SQLite copies the bound text (a C macro cinterop can't map) +private val TRANSIENT = (-1L).toCPointer Unit>>() + +private class NativeSqliteReader( + path: String, +) : SqliteReader { + private val db: CPointer = + memScoped { + val out = alloc>() + val status = sqlite3_open_v2(path, out.ptr, SQLITE_OPEN_READONLY or SQLITE_OPEN_FULLMUTEX, null) + val opened = out.value + if (status != SQLITE_OK || opened == null) { + val message = opened?.let { sqlite3_errmsg(it)?.toKString() } + sqlite3_close_v2(opened) + error("Can't open $path: ${message ?: status}") + } + opened + } + + override fun query( + sql: String, + args: List, + ): List> = + memScoped { + val out = alloc>() + check(sqlite3_prepare_v2(db, sql, -1, out.ptr, null) == SQLITE_OK) { "SQLite: ${sqlite3_errmsg(db)?.toKString()}" } + val stmt = out.value!! + try { + args.forEachIndexed { i, arg -> sqlite3_bind_text(stmt, i + 1, arg, -1, TRANSIENT) } + val columns = (0 until sqlite3_column_count(stmt)).map { sqlite3_column_name(stmt, it)?.toKString() ?: "" } + val rows = ArrayList>() + while (sqlite3_step(stmt) == SQLITE_ROW) { + rows += columns.indices.associate { columns[it] to sqlite3_column_text(stmt, it)?.reinterpret()?.toKString() } + } + rows + } finally { + sqlite3_finalize(stmt) + } + } + + override fun close() { + sqlite3_close_v2(db) + } +} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt new file mode 100644 index 00000000..d13f9495 --- /dev/null +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt @@ -0,0 +1,185 @@ +@file:OptIn(ExperimentalForeignApi::class) + +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.jni.JNIEnvVar +import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_CreateJavaVM +import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_EDETACHED +import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_GetCreatedJavaVMs +import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_OK +import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_TRUE +import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_VERSION_21 +import io.github.kdroidfilter.seforimlibrary.search.jni.JavaVMInitArgs +import io.github.kdroidfilter.seforimlibrary.search.jni.JavaVMOption +import io.github.kdroidfilter.seforimlibrary.search.jni.JavaVMVar +import io.github.kdroidfilter.seforimlibrary.search.jni.jbyteVar +import io.github.kdroidfilter.seforimlibrary.search.jni.jclass +import io.github.kdroidfilter.seforimlibrary.search.jni.jintVar +import io.github.kdroidfilter.seforimlibrary.search.jni.jmethodID +import io.github.kdroidfilter.seforimlibrary.search.jni.jvalue +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized +import kotlinx.cinterop.CPointer +import kotlinx.cinterop.CPointerVar +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.alloc +import kotlinx.cinterop.allocArray +import kotlinx.cinterop.convert +import kotlinx.cinterop.cstr +import kotlinx.cinterop.get +import kotlinx.cinterop.invoke +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.pointed +import kotlinx.cinterop.ptr +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.usePinned +import kotlinx.cinterop.value + +/** + * The Java runtime the iOS app embeds: OpenJDK's mobile port (Zero, the interpreter, as iOS forbids generated code), + * linked statically into the app. Lucene runs there; the app calls it through [call]. + * + * The runtime reads its classes from `/lib` (its `java.home`, set by the VM itself); [classPath] + * names the jars of the search (see the `iosJvmClasspath` task of SeforimLibrary's `search` module). + */ +class EmbeddedJvm private constructor( + private val vm: CPointer, +) { + private val lock = SynchronizedObject() + + // One bridge method per process (IndexBridgeServer.call) + private var bridge: Pair? = null + + /** + * Calls the static method `byte[] method(byte[])` of [className] with [request], from the calling thread + * (attached to the runtime on its first call), and returns its result; throws when it threw. + */ + fun call( + className: String, + method: String, + request: ByteArray, + ): ByteArray { + val env = attachedEnv() + val functions = env.pointed.pointed!! + val (clazz, methodId) = bridgeMethod(env, className, method) + val array = functions.NewByteArray!!(env, request.size) ?: throwPending(env, "NewByteArray") + try { + if (request.isNotEmpty()) { + request.usePinned { pinned -> + functions.SetByteArrayRegion!!(env, array, 0, request.size, pinned.addressOf(0).reinterpret()) + } + } + val result = + memScoped { + val args = allocArray(1) + args[0].l = array + functions.CallStaticObjectMethodA!!(env, clazz, methodId, args) + } + if (functions.ExceptionCheck!!(env).toInt() != 0) throwPending(env, "$className.$method") + result ?: error("$className.$method returned null") + try { + val size = functions.GetArrayLength!!(env, result) + val bytes = ByteArray(size) + if (size > 0) { + bytes.usePinned { pinned -> + functions.GetByteArrayRegion!!(env, result, 0, size, pinned.addressOf(0).reinterpret()) + } + } + return bytes + } finally { + functions.DeleteLocalRef!!(env, result) + } + } finally { + functions.DeleteLocalRef!!(env, array) + } + } + + // The class and method are resolved once: a global reference keeps the class + private fun bridgeMethod( + env: CPointer, + className: String, + method: String, + ): Pair = + synchronized(lock) { + bridge ?: memScoped { + val functions = env.pointed.pointed!! + val local = functions.FindClass!!(env, className.replace('.', '/').cstr.getPointer(this)) ?: throwPending(env, className) + val global = functions.NewGlobalRef!!(env, local)!! + functions.DeleteLocalRef!!(env, local) + val methodId = functions.GetStaticMethodID!!(env, global, method.cstr.getPointer(this), "([B)[B".cstr.getPointer(this)) + ?: throwPending(env, "$className.$method") + (global to methodId).also { bridge = it } + } + } + + private fun attachedEnv(): CPointer = + memScoped { + val env = alloc>() + val invoke = vm.pointed.pointed!! + val status = invoke.GetEnv!!(vm, env.ptr.reinterpret(), JNI_VERSION_21) + if (status == JNI_EDETACHED) { + // Daemon: an attached Kotlin worker never holds the runtime up at exit + check(invoke.AttachCurrentThreadAsDaemon!!(vm, env.ptr.reinterpret(), null) == JNI_OK) { "Can't attach the thread to the Java runtime" } + } else { + check(status == JNI_OK) { "GetEnv failed: $status" } + } + env.value!! + } + + private fun throwPending( + env: CPointer, + what: String, + ): Nothing { + val functions = env.pointed.pointed!! + if (functions.ExceptionCheck!!(env).toInt() != 0) { + functions.ExceptionDescribe!!(env) + functions.ExceptionClear!!(env) + } + error("Java call failed: $what") + } + + companion object { + private val lock = SynchronizedObject() + private var instance: EmbeddedJvm? = null + + /** + * The Java runtime of this process, started on the first call with [classPath] and [options] (`-Xmx...`, + * `-D...`); a process has one, so later calls get it as it is. + */ + fun start( + classPath: List, + options: List = emptyList(), + ): EmbeddedJvm = + synchronized(lock) { + instance ?: (existing() ?: create(classPath, options)).also { instance = it } + } + + private fun existing(): EmbeddedJvm? = + memScoped { + val vms = allocArray>(1) + val count = alloc() + if (JNI_GetCreatedJavaVMs(vms, 1, count.ptr) == JNI_OK && count.value > 0) vms[0]?.let { EmbeddedJvm(it) } else null + } + + private fun create( + classPath: List, + options: List, + ): EmbeddedJvm = + memScoped { + val all = listOf("-Djava.class.path=" + classPath.joinToString(":")) + options + val vmOptions = allocArray(all.size) + all.forEachIndexed { i, option -> vmOptions[i].optionString = option.cstr.getPointer(this) } + val args = alloc() + args.version = JNI_VERSION_21 + args.nOptions = all.size + args.options = vmOptions + args.ignoreUnrecognized = JNI_TRUE.convert() + val vm = alloc>() + val env = alloc>() + val status = JNI_CreateJavaVM(vm.ptr, env.ptr.reinterpret(), args.ptr) + check(status == JNI_OK) { "JNI_CreateJavaVM failed: $status" } + EmbeddedJvm(vm.value!!) + } + } +} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt new file mode 100644 index 00000000..7e5b535e --- /dev/null +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt @@ -0,0 +1,16 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex + +private const val SERVER_CLASS = "io.github.kdroidfilter.seforimlibrary.search.index.bridge.IndexBridgeServer" +private const val SERVER_METHOD = "call" + +/** The calls to the Java runtime the iOS app embeds: `IndexBridgeServer.call`, through JNI, from the calling thread. */ +fun EmbeddedJvm.bridgeTransport(): BridgeTransport = BridgeTransport { call(SERVER_CLASS, SERVER_METHOD, it) } + +/** The [LuceneIndex] at [indexDir] on iOS: Lucene runs in the Java runtime the app embeds ([jvm]). */ +fun JvmLuceneIndex( + jvm: EmbeddedJvm, + indexDir: String, + cacheQueries: Boolean = false, +): LuceneIndex = BridgedLuceneIndex(jvm.bridgeTransport(), indexDir, cacheQueries) diff --git a/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt b/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt new file mode 100644 index 00000000..e99e60d9 --- /dev/null +++ b/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt @@ -0,0 +1,41 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import io.github.kdroidfilter.seforimlibrary.search.index.bridge.EmbeddedJvm +import io.github.kdroidfilter.seforimlibrary.search.index.bridge.JvmLuceneIndex +import kotlinx.coroutines.runBlocking +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem +import kotlinx.io.writeString +import platform.Foundation.NSProcessInfo +import kotlin.time.measureTimedValue +import kotlin.test.Test + +/** + * The search on iOS, end to end in the simulator: Lucene in the embedded Java runtime, everything else here. Dumps what + * it answers on a real index like RealIndexParityTest does on the JVM (SEFORIM_INDEX_DIR, SEFORIM_JVM_CLASSPATH and + * SEFORIM_PARITY_OUT, passed by the iosSimulatorArm64Test task). Skipped without an index. + */ +class IosRealIndexParityTest { + @Test + fun dumpRealIndex() { + val env = NSProcessInfo.processInfo.environment + val indexDir = env["SEFORIM_INDEX_DIR"] as String? ?: return + val classPath = (env["SEFORIM_JVM_CLASSPATH"] as String).split(':') + val (jvm, startup) = measureTimedValue { EmbeddedJvm.start(classPath, listOf("-Xmx768m") + (env["SEFORIM_JVM_OPTIONS"] as String?)?.split(' ').orEmpty()) } + println("Java runtime started in $startup") + val dictionary = indexDir.substringBeforeLast('/') + "/lexical.db" + val engine = + LexicalSearchEngine( + index = JvmLuceneIndex(jvm, indexDir), + dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, dictionary) }, + ) + engine.use { + val (dump, took) = measureTimedValue { runBlocking { SearchDump.dump(it) } } + println("Parity dump in $took") + val out = Path(env["SEFORIM_PARITY_OUT"] as String) + SystemFileSystem.sink(out).buffered().use { sink -> sink.writeString(dump) } + println("Parity dump: $out") + } + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index bb21b9eb..2e94b877 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -1,1176 +1,109 @@ package io.github.kdroidfilter.seforimlibrary.search import co.touchlab.kermit.Logger -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend import org.apache.lucene.analysis.Analyzer -import org.apache.lucene.analysis.TokenStream import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute -import org.apache.lucene.index.DocValuesType -import org.apache.lucene.index.FieldInfo -import org.apache.lucene.index.IndexReader -import org.apache.lucene.index.NumericDocValues -import org.apache.lucene.index.StoredFieldVisitor -import org.apache.lucene.index.StoredFields -import org.apache.lucene.index.Term -import org.apache.lucene.index.LeafReaderContext -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.BoostQuery -import org.apache.lucene.search.Collector -import org.apache.lucene.search.CollectorManager -import org.apache.lucene.search.FuzzyQuery -import org.apache.lucene.search.IndexSearcher -import org.apache.lucene.search.LeafCollector -import org.apache.lucene.search.PrefixQuery -import org.apache.lucene.search.PointRangeQuery -import org.apache.lucene.search.Query -import org.apache.lucene.search.QueryCachingPolicy -import org.apache.lucene.search.Scorable -import org.apache.lucene.search.ScoreDoc -import org.apache.lucene.search.ReferenceManager -import org.apache.lucene.search.ScoreMode -import org.apache.lucene.search.SearcherFactory -import org.apache.lucene.search.SearcherManager -import org.apache.lucene.search.TermQuery -import org.apache.lucene.util.QueryBuilder -import org.apache.lucene.store.FSDirectory -import org.apache.lucene.store.NIOFSDirectory -import org.apache.lucene.document.IntPoint import java.nio.file.Path -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.atomic.AtomicInteger -/** - * Lucene-based implementation of SearchEngine for full-text search. - * Supports Hebrew text with diacritics handling, dictionary expansion, and fuzzy matching. - */ private val logger = Logger.withTag("LuceneSearchEngine") -// Hard cap on how many synonym/expansion terms we allow per token -private const val MAX_SYNONYM_TERMS_PER_TOKEN: Int = 32 -// Global cap for boost queries built from dictionary expansions -private const val MAX_SYNONYM_BOOST_TERMS: Int = 256 -// Constants for snippet source building (must match indexer) -private const val SNIPPET_NEIGHBOR_WINDOW = 4 -private const val SNIPPET_MIN_LENGTH = 280 -private const val FIELD_BOOK_ID = "book_id" -private const val DEFAULT_NEAR = 5 -private const val PLAN_CACHE_SIZE = 16 -private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" -// Searches run by warmUp(): one and several words, common and rarer ones, a quoted phrase, the divine name (whose -// dictionary entry is the largest) -val WARMUP_QUERIES = listOf("ברוך ה׳ לעולם", "אמר רבי יוחנן", "תפילין", "\"קריאת שמע\"") -const val WARMUP_ROUNDS = 2 -const val WARMUP_PAGE_SIZE = 25 - -// Smallest partition worth a task of its own when a segment is split across cores -private const val MIN_DOCS_PER_SLICE = 100_000 - -/** - * Searchers that split the index, even a single segment, into one partition per core and search them concurrently: - * a merged index is one big segment, which a plain searcher scores on one thread. - */ -private object PartitionedSearcherFactory : SearcherFactory() { - private val cores = Runtime.getRuntime().availableProcessors() - private val executor: ExecutorService by lazy { - val ids = AtomicInteger() - Executors.newFixedThreadPool(cores) { task -> - Thread(task, "lucene-search-${ids.incrementAndGet()}").apply { isDaemon = true } - } - } - - override fun newSearcher(reader: IndexReader, previousReader: IndexReader?): IndexSearcher { - val searcher = if (cores == 1) { - IndexSearcher(reader) - } else { - val docsPerSlice = maxOf(MIN_DOCS_PER_SLICE, reader.maxDoc() / cores + 1) - object : IndexSearcher(reader, executor) { - override fun slices(leaves: List): Array = - slices(leaves, docsPerSlice, Int.MAX_VALUE, true) - } - } - return searcher.apply { queryCachingPolicy = StableFiltersCachingPolicy } - } -} - -/** - * Caches only the filters every search repeats (line documents, base books). Lucene's default policy also caches any - * clause seen twice, and each search builds its word filters twice (facets, then results): it would then turn them - * into bitsets over the whole index for a query that won't come back. - */ -private object StableFiltersCachingPolicy : QueryCachingPolicy { - override fun onUse(query: Query) = Unit - - override fun shouldCache(query: Query): Boolean = when (query) { - is TermQuery -> query.term.field() == "type" - is PointRangeQuery -> query.field == "is_base_book" - else -> false - } -} - -/** - * Blacklist of hallucinated dictionary mappings. - * Loaded from resources/hallucination_blacklist.tsv - * Key: normalized token, Value: set of incorrect base forms to reject - * - * TODO: This is a temporary workaround. The proper fix is to correct the - * hallucinated mappings directly in the lexical dictionary (lexical.db) - * so this blacklist becomes unnecessary. - */ -private val HALLUCINATION_BLACKLIST: Map> by lazy { - loadHallucinationBlacklist() -} - -private fun loadHallucinationBlacklist(): Map> { - val result = mutableMapOf>() - try { - val inputStream = LuceneSearchEngine::class.java.getResourceAsStream("/hallucination_blacklist.tsv") - if (inputStream == null) { - logger.w { "hallucination_blacklist.tsv not found in resources" } - return emptyMap() - } - inputStream.bufferedReader().useLines { lines -> - lines.forEach { line -> - val trimmed = line.trim() - if (trimmed.isEmpty() || trimmed.startsWith("#")) return@forEach - val parts = trimmed.split("\t") - if (parts.size >= 2) { - val token = HebrewTextUtils.normalizeHebrew(parts[0]) - val base = HebrewTextUtils.normalizeHebrew(parts[1]) - result.getOrPut(token) { mutableSetOf() }.add(base) - } - } - } - logger.d { "Loaded ${result.size} hallucination blacklist entries" } - } catch (e: Exception) { - logger.e(e) { "Failed to load hallucination blacklist" } - } - return result -} /** - * Check if an expansion should be rejected based on the hallucination blacklist. + * The desktop's [SearchEngine]: the common [LexicalSearchEngine] over the Lucene index at [indexDir], read in process, + * with the lexical dictionary found beside it (or at [dictionaryPath], `-DmagicDict`, `SEFORIM_MAGIC_DICT`). */ -private fun isHallucinatedExpansion(token: String, expansion: MagicDictionaryIndex.Expansion): Boolean { - val normalizedToken = HebrewTextUtils.normalizeHebrew(token) - val blacklistedBases = HALLUCINATION_BLACKLIST[normalizedToken] ?: return false - return expansion.base.any { base -> - val normalizedBase = HebrewTextUtils.normalizeHebrew(base) - blacklistedBases.contains(normalizedBase) - } -} - -class LuceneSearchEngine( - private val indexDir: Path, - private val snippetProvider: SnippetProvider? = null, - private val analyzer: Analyzer = StandardAnalyzer(), - private val dictionaryPath: Path? = null -) : SearchEngine { - - // Open Lucene directory lazily to avoid any I/O at app startup. - // GraalVM native image does not support MMapDirectory (Panama foreign downcalls), use NIOFSDirectory instead. - private val dir by lazy { - if (System.getProperty("org.graalvm.nativeimage.imagecode") != null) { - NIOFSDirectory(indexDir) - } else { - FSDirectory.open(indexDir) - } - } - - private val stdAnalyzer: Analyzer by lazy { analyzer } - private val magicDict: MagicDictionaryIndex? by lazy { - val candidates = listOfNotNull( - dictionaryPath, - System.getProperty("magicDict")?.let { Path.of(it) }, - System.getenv("SEFORIM_MAGIC_DICT")?.let { Path.of(it) }, - indexDir.resolveSibling("lexical.db"), - indexDir.resolveSibling("seforim.db").resolveSibling("lexical.db"), - Path.of("SeforimLibrary/SeforimMagicIndexer/magicindexer/build/db/lexical.db") - ).distinct() - val firstExisting = MagicDictionaryIndex.findValidDictionary(candidates) - if (firstExisting == null) { - logger.d { - "[MagicDictionary] Missing lexical.db; search will run without dictionary expansions. " + - "Provide -DmagicDict=/path/lexical.db or SEFORIM_MAGIC_DICT. Checked: " + - candidates.joinToString() - } - return@lazy null - } - logger.d { "[MagicDictionary] Loading lexical db from $firstExisting" } - val loaded = MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, firstExisting) - if (loaded == null) { - logger.d { - "[MagicDictionary] Failed to load lexical db at $firstExisting; " + - "continuing without dictionary expansions" - } - } - loaded - } - - // One reader shared by every search: opening one per search re-reads the term index of each segment and maps, - // then unmaps, all the files. Refreshed when the index changes on disk (delta updates write into it). - private val searcherManagerLazy = lazy { - SearcherManager(dir, PartitionedSearcherFactory).apply { - addListener( - object : ReferenceManager.RefreshListener { - override fun beforeRefresh() = Unit - - override fun afterRefresh(didRefresh: Boolean) { - if (didRefresh) bookAncestors.clear() - } - } - ) - } - } - - private val searcherManager: SearcherManager by searcherManagerLazy - - // Ancestor category ids per book, for the facets: every line of a book shares them - private val bookAncestors = ConcurrentHashMap() - - private fun acquireSearcher(): IndexSearcher { - val manager = searcherManager - manager.maybeRefresh() - return manager.acquire() - } - - private inline fun withSearcher(block: (IndexSearcher) -> T): T { - val searcher = acquireSearcher() - try { - return block(searcher) - } finally { - searcherManager.release(searcher) - } - } - - // --- SearchEngine interface implementation --- - - override fun openSession( - query: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean - ): SearchSession? { - val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) ?: return null - return LuceneSearchSession(context.query, context, acquireSearcher()) - } - - override suspend fun attachSnippets(hits: List, query: String, near: Int): List { - if (hits.isEmpty()) return hits - val context = buildSearchContext(query, near, null, null, null, null) ?: return hits - val sources = withContext(Dispatchers.IO) { snippetSources(hits.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) } - return hits.map { hit -> - val source = sources[hit.lineId] ?: SnippetSource(hit.rawText, 0, hit.rawText.length) - hit.copy( - snippet = SnippetBuilder.build(source.text, context.highlightPlan, source.lineStart, source.lineEnd), - rawText = source.text, - lineStart = source.lineStart, - lineEnd = source.lineEnd - ) - } - } - - override suspend fun warmUp() { - withContext(Dispatchers.IO) { hashemHighlightTerms } - // A few searches down the whole path (facets, ranking, snippets) load the index structures and, on the JVM, - // get the hot code compiled before the user's first search - repeat(WARMUP_ROUNDS) { - for (query in WARMUP_QUERIES) { - computeFacets(query, baseBookOnly = true) - openSession(query, baseBookOnly = true)?.use { it.nextPage(WARMUP_PAGE_SIZE) } - } - } - } - - override fun searchBooksByTitlePrefix(query: String, limit: Int): List { - val q = HebrewTextUtils.normalizeHebrew(query) - if (q.isBlank()) return emptyList() - val tokens = q.split("\\s+".toRegex()).map { it.trim() }.filter { it.isNotEmpty() } - if (tokens.isEmpty()) return emptyList() - - return withSearcher { searcher -> - val must = BooleanQuery.Builder() - // Restrict to book_title docs - must.add(TermQuery(Term("type", "book_title")), BooleanClause.Occur.FILTER) - tokens.forEach { tok -> - // prefix on analyzed 'title' - must.add(PrefixQuery(Term("title", tok)), BooleanClause.Occur.MUST) - } - val luceneQuery = must.build() - val top = searcher.search(luceneQuery, limit) - val stored: StoredFields = searcher.storedFields() - val ids = LinkedHashSet() - for (sd in top.scoreDocs) { - val doc = stored.document(sd.doc) - val id = doc.getField("book_id")?.numericValue()?.toLong() - if (id != null) ids.add(id) - } - ids.toList() - } - } - - override fun buildSnippet(rawText: String, query: String, near: Int): String = - SnippetBuilder.build(SnippetSources.clean(rawText), planFor(query, near)) - - override fun highlightRanges(text: String, query: String): List = - mergedRanges(text, planFor(query, DEFAULT_NEAR).highlights(text)) - - override fun rangeSnippet(text: String, ranges: List, lineStart: Int, lineEnd: Int): String = - SnippetBuilder.build( - text, - ranges.map { Occurrence(it.first, it.last + 1, 0, EvidenceLevel.LITERAL) }, - lineStart, - lineEnd - ) - - // The highlight plans of the latest queries: a results page and its preview ask for the same one many times - private val planCache = object : LinkedHashMap(PLAN_CACHE_SIZE, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = - size > PLAN_CACHE_SIZE - } - - private fun planFor(query: String, near: Int): HighlightPlan { - synchronized(planCache) { planCache[query]?.let { return it } } - val plan = buildSearchContext(query, near, null, null, null, null)?.highlightPlan ?: HighlightPlan.EMPTY - synchronized(planCache) { planCache[query] = plan } - return plan - } - - /** - * What a query highlights: each word with its inflections, from the dictionary entries it truly is a form of and - * that aren't blacklisted for it. With ה׳ in the query, the lone ה is the divine name. - */ - private fun buildHighlightPlan( - tokens: List, - hasHashem: Boolean, - exactPhrases: List - ): HighlightPlan { - val forms = tokens.distinct().map { token -> - if (hasHashem && highlightKey(token) == "ה") { - return@map TokenForms(token, extraLiterals = loadHashemHighlightTerms()) - } - val key = highlightKey(token) - val inflections = magicDict?.expansionsFor(listOf(token)).orEmpty() - .filter { exp -> exp.inflections.any { highlightKey(it) == key } && !isHallucinatedExpansion(token, exp) } - .flatMap { it.inflections } - TokenForms(token, inflections) - } - val phrases = exactPhrases - .map { analyzeToTerms(stdAnalyzer, it).orEmpty().joinToString(" ") } - .filter { it.isNotBlank() } - return HighlightPlan.build(forms, phrases) - } - - override fun findInBookCandidates(query: String, bookId: Long): LongArray? { - val norm = HebrewTextUtils.normalizeHebrew(query) - if (norm.isBlank()) return LongArray(0) - val bookLines = BooleanQuery.Builder() - .add(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER) - .add(IntPoint.newExactQuery("book_id", bookId.toInt()), BooleanClause.Occur.FILTER) - .build() - // Nothing to filter on: every line would be a candidate, cheaper scanned from the DB - val clauses = literalPresenceClauses(analyzeToTerms(stdAnalyzer, norm).orEmpty()) - if (clauses.isEmpty()) return null - val builder = BooleanQuery.Builder().add(bookLines, BooleanClause.Occur.FILTER) - clauses.forEach { builder.add(it, BooleanClause.Occur.FILTER) } - - return withSearcher { searcher -> - // Only line_id is read per hit: no scoring, no other stored field. One collector per slice, as the - // slices may run concurrently - class LineIdCollector : Collector { - var ids = LongArray(256) - var count = 0 - - override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { - val storedFields = leafContext.reader().storedFields() - return object : LeafCollector { - override fun setScorer(scorer: Scorable) = Unit - - override fun collect(doc: Int) { - val lineId = storedFields.document(doc, setOf("line_id")) - .getField("line_id")?.numericValue()?.toLong() ?: return - if (count == ids.size) ids = ids.copyOf(count * 2) - ids[count++] = lineId - } - } - } - - override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES - } - val ids = searcher.search( - builder.build(), - object : CollectorManager { - override fun newCollector() = LineIdCollector() - - override fun reduce(collectors: Collection): LongArray { - val out = LongArray(collectors.sumOf { it.count }) - var at = 0 - for (c in collectors) { - c.ids.copyInto(out, at, 0, c.count) - at += c.count - } - return out - } - } - ) - val count = ids.size - // No candidate may mean a book the index lacks (older index, book added later): unknown, not "no match" - if (count == 0 && searcher.count(bookLines) == 0) null else ids - } - } - - /** - * Filters every line holding the analyzed [tokens] as one literal substring satisfies. The - * substring may start and end mid-word: the first token is only a word suffix, the last a - * word prefix, the inner ones whole words. A first or lone token is matched anywhere in a - * word through its 4-grams, so one under 4 letters can't filter (a leading wildcard would - * scan the whole term dictionary); a last one under 3 neither, its prefix being too common. - */ - private fun literalPresenceClauses(tokens: List): List = - tokens.mapIndexedNotNull { i, token -> - when { - i in 1 until tokens.lastIndex -> TermQuery(Term("text", token)) - i > 0 && token.length >= 3 -> PrefixQuery(Term("text", token)) - else -> buildNgramPresenceForToken(token) - } - } - - override fun close() { - if (searcherManagerLazy.isInitialized()) runCatching { searcherManager.close() } - } - - override fun computeFacets( - query: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean - ): SearchFacets? { - val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) - ?: return null - - return withSearcher { searcher -> - // Only book ids are read per hit (doc values when the index has them, else the first stored field); - // category counts follow from the book counts, as every line of a book shares its ancestors. One - // collector per slice, as the slices may run concurrently - class FacetCollector : Collector { - val bookCounts = HashMap() - var totalHits = 0L - - override fun getLeafCollector(leafContext: LeafReaderContext): LeafCollector { - val reader = leafContext.reader() - val storedFields = reader.storedFields() - val bookIdValues: NumericDocValues? = - reader.fieldInfos.fieldInfo(FIELD_BOOK_ID) - ?.takeIf { it.docValuesType == DocValuesType.NUMERIC } - ?.let { reader.getNumericDocValues(FIELD_BOOK_ID) } - // Lines of a book are mostly contiguous: count runs, then fold them into the map - var runBook = -1L - var runCount = 0 - - fun flushRun() { - if (runCount > 0) bookCounts.merge(runBook, runCount, Int::plus) - runCount = 0 - } - - return object : LeafCollector { - override fun setScorer(scorer: Scorable) = Unit - - override fun collect(doc: Int) { - totalHits++ - val bookId = - if (bookIdValues != null && bookIdValues.advanceExact(doc)) { - bookIdValues.longValue() - } else { - readStoredBookId(storedFields, doc) ?: return - } - if (bookId != runBook) { - flushRun() - runBook = bookId - if (!bookAncestors.containsKey(bookId)) loadAncestors(storedFields, doc, bookId) - } - runCount++ - } - - override fun finish() = flushRun() - } - } - - override fun scoreMode(): ScoreMode = ScoreMode.COMPLETE_NO_SCORES - } - - searcher.search( - context.matchQuery, - object : CollectorManager { - override fun newCollector() = FacetCollector() - - override fun reduce(collectors: Collection): SearchFacets { - val bookCounts = HashMap() - collectors.forEach { c -> c.bookCounts.forEach { (book, n) -> bookCounts.merge(book, n, Int::plus) } } - val categoryCounts = HashMap() - for ((bookId, count) in bookCounts) { - bookAncestors[bookId]?.forEach { categoryCounts.merge(it, count, Int::plus) } - } - return SearchFacets( - totalHits = collectors.sumOf { it.totalHits }, - categoryCounts = categoryCounts, - bookCounts = bookCounts - ) - } - } - ) - } - } - - private fun readStoredBookId(storedFields: StoredFields, doc: Int): Long? { - var bookId: Long? = null - storedFields.document( - doc, - object : StoredFieldVisitor() { - override fun needsField(fieldInfo: FieldInfo): Status = when { - bookId != null -> Status.STOP - fieldInfo.name == FIELD_BOOK_ID -> Status.YES - else -> Status.NO - } - - override fun longField(fieldInfo: FieldInfo, value: Long) { - bookId = value - } - - override fun intField(fieldInfo: FieldInfo, value: Int) { - bookId = value.toLong() - } - } - ) - return bookId - } - - private fun loadAncestors(storedFields: StoredFields, doc: Int, bookId: Long) { - val ancestors = storedFields.document(doc, setOf(FIELD_ANCESTOR_CATEGORY_IDS)) - .getField(FIELD_ANCESTOR_CATEGORY_IDS)?.stringValue() - ?.split(",") - ?.mapNotNull { it.trim().toLongOrNull() } - .orEmpty() - // Lines added by delta updates lack the field: try again on the book's next run - if (ancestors.isNotEmpty()) bookAncestors[bookId] = ancestors.toLongArray() - } - - // --- Inner SearchSession class --- - - private inner class LuceneSearchSession( - private val query: Query, - private val context: SearchContext, - private val searcher: IndexSearcher - ) : SearchSession { - private var after: ScoreDoc? = null - private var finished = false - private var totalHitsValue: Long? = null - - override suspend fun nextPage(limit: Int, snippets: Boolean): SearchPage? = withContext(Dispatchers.IO) { - if (finished) return@withContext null - val top = searcher.searchAfter(after, query, limit) - if (totalHitsValue == null) totalHitsValue = top.totalHits?.value - if (top.scoreDocs.isEmpty()) { - finished = true - return@withContext null - } - val stored = searcher.storedFields() - val hits = mapScoreDocs(stored, top.scoreDocs.asList(), context, snippets) - after = top.scoreDocs.last() - val isLast = top.scoreDocs.size < limit - if (isLast) finished = true - SearchPage( - hits = hits, - totalHits = totalHitsValue ?: hits.size.toLong(), - isLastPage = isLast - ) - } - - override fun close() { - searcherManager.release(searcher) - } - } - - // --- Additional public search methods --- - - suspend fun searchAllText(rawQuery: String, near: Int = 5, limit: Int, offset: Int = 0): List = - doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = null) - - suspend fun searchInBook(rawQuery: String, near: Int, bookId: Long, limit: Int, offset: Int = 0): List = - doSearch(rawQuery, near, limit, offset, bookFilter = bookId, categoryFilter = null) - - suspend fun searchInCategory(rawQuery: String, near: Int, categoryId: Long, limit: Int, offset: Int = 0): List = - doSearch(rawQuery, near, limit, offset, bookFilter = null, categoryFilter = categoryId) - - suspend fun searchInBooks(rawQuery: String, near: Int, bookIds: Collection, limit: Int, offset: Int = 0): List = - doSearchInBooks(rawQuery, near, limit, offset, bookIds) - - // --- Private implementation --- - - private class SearchContext( - val query: Query, - // The required clauses only: the same matches as [query], without the ranking-only clauses (fuzzy, n-grams, - // synonym boosts) that cost a rewrite but can't change the hit set - val matchQuery: Query, - private val plan: Lazy - ) { - val highlightPlan: HighlightPlan get() = plan.value - } - - private fun buildSearchContext( +class LuceneSearchEngine private constructor( + /** The Lucene index this engine reads, for the searches that share it (the semantic search's vectors). */ + val index: LuceneIndex, + private val engine: LexicalSearchEngine, +) : SearchEngine by engine { + private constructor( + index: LuceneIndex, + indexDir: Path, + snippetProvider: SnippetProvider?, + dictionaryPath: Path?, + ) : this( + index, + LexicalSearchEngine(index, snippetProvider, lazy { loadDictionary(indexDir, dictionaryPath) }), + ) + + constructor( + indexDir: Path, + snippetProvider: SnippetProvider? = null, + analyzer: Analyzer = StandardAnalyzer(), + dictionaryPath: Path? = null, + ) : this(LuceneIndexBackend(indexDir, analyzer = analyzer), indexDir, snippetProvider, dictionaryPath) + + suspend fun searchAllText( rawQuery: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean = false - ): SearchContext? { - // Split the raw query into exact phrases (wrapped in ASCII double quotes) and free text. - // Quoted segments are matched verbatim WITHOUT dictionary expansion (Google-style). - val parsed = SearchQueryParser.parse(rawQuery) - val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) - val exactPhrasesNorm = parsed.exactPhrases - .map { HebrewTextUtils.normalizeHebrew(it) } - .filter { it.isNotBlank() } - - if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null - - val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList() - - // Check if the original query contained ה׳ (Hashem) before normalization - val hasHashem = rawQuery.contains("ה׳") || rawQuery.contains("ה'") - - // Filter out single Hebrew letters and stop words BEFORE dictionary expansion - // BUT preserve "ה" if the original query had "ה׳" (Hashem) - val analyzedStd = analyzedRaw.filter { token -> - // Special case: if query has ה׳, keep "ה" token - if (token == "ה" && hasHashem) return@filter true - // Preserve numeric tokens (e.g., "6") so they can expand via MagicDictionary - if (token.any { it.isDigit() }) return@filter true - - token.length >= 2 && token !in setOf( - "א", "ב", "ג", "ד", "ה", "ו", "ז", "ח", "ט", "י", "כ", "ל", "מ", - "נ", "ס", "ע", "פ", "צ", "ק", "ר", "ש", "ת", - ) - } - - logger.d { "[DEBUG] Original query had Hashem (ה׳): $hasHashem" } - logger.d { "[DEBUG] Analyzed tokens: $analyzedStd" } - - // Get all possible expansions for each token (a token can belong to multiple bases) - // These expansions are used for SEARCH - we keep all of them for better recall - magicDict?.prefetch(analyzedStd) - val tokenExpansions: Map> = - analyzedStd.associateWith { token -> - // Get best expansion (prefers matching base, then largest) - val expansion = magicDict?.expansionFor(token) ?: return@associateWith emptyList() - listOf(expansion) - } - tokenExpansions.forEach { (token, exps) -> - exps.forEach { exp -> - logger.d { "[DEBUG] Token '$token' -> expansion: surface=${exp.surface.take(10)}..., variants=${exp.variants.take(10)}..., base=${exp.base}" } - } - } - - // What the snippets highlight: built on first use (facets need none) - val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhrasesNorm) } - - val builder = BooleanQuery.Builder() - // Mirrors every required clause of builder (the facets need only the hit set) - val matchBuilder = BooleanQuery.Builder() - fun require(query: Query, occur: BooleanClause.Occur) { - builder.add(query, occur) - matchBuilder.add(query, occur) - } - require(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER) - if (bookFilter != null) require(IntPoint.newExactQuery("book_id", bookFilter.toInt()), BooleanClause.Occur.FILTER) - if (categoryFilter != null) require(IntPoint.newExactQuery("category_id", categoryFilter.toInt()), BooleanClause.Occur.FILTER) - // Filter by base books only (is_base_book = 1) when baseBookOnly is true - if (baseBookOnly) require(IntPoint.newExactQuery("is_base_book", 1), BooleanClause.Occur.FILTER) - val bookIdsArray = bookIds?.map { it.toInt() }?.toIntArray() - if (bookIdsArray != null && bookIdsArray.isNotEmpty()) { - require(IntPoint.newSetQuery("book_id", *bookIdsArray), BooleanClause.Occur.FILTER) - } - val lineIdsArray = lineIds?.map { it.toInt() }?.toIntArray() - if (lineIdsArray != null && lineIdsArray.isNotEmpty()) { - require(IntPoint.newSetQuery("line_id", *lineIdsArray), BooleanClause.Occur.FILTER) - } - // Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase. - var exactClausesAdded = 0 - for (phrase in exactPhrasesNorm) { - val exactQuery = buildExactPhraseQuery(phrase) ?: continue - require(exactQuery, BooleanClause.Occur.MUST) - exactClausesAdded++ - logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" } - } - - // A fully-quoted query that produced no usable phrase clause (e.g. just punctuation) - // must not fall through to a match-all query. - if (norm.isBlank() && exactClausesAdded == 0) return null - - // Free (unquoted) text: dictionary-aware ranking + presence filtering. - if (norm.isNotBlank()) { - val rankedQuery = buildExpandedQuery(norm, near, analyzedStd, tokenExpansions) - val mustAllTokensQuery: Query? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions) - val phraseQuery: Query? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near) - if (mustAllTokensQuery != null) { - require(mustAllTokensQuery, BooleanClause.Occur.FILTER) - logger.d { "[DEBUG] Added mustAllTokensQuery as FILTER" } - } - if (phraseQuery != null && analyzedStd.size >= 2) { - if (near == 0) require(phraseQuery, BooleanClause.Occur.MUST) else builder.add(phraseQuery, BooleanClause.Occur.SHOULD) - logger.d { "[DEBUG] Added phraseQuery, near=$near" } - } - builder.add(rankedQuery, BooleanClause.Occur.SHOULD) - logger.d { "[DEBUG] Added rankedQuery as SHOULD" } - } - - val finalQuery = builder.build() - logger.d { "[DEBUG] Final query: $finalQuery" } - - return SearchContext( - query = finalQuery, - matchQuery = matchBuilder.build(), - plan = highlightPlan - ) - } - - // suspend because it calls snippetProvider.getSnippetSources(); - // callers are responsible for providing Dispatchers.IO context. - private suspend fun mapScoreDocs( - stored: StoredFields, - scoreDocs: List, - context: SearchContext, - snippets: Boolean = true - ): List { - if (scoreDocs.isEmpty()) return emptyList() - - // First pass: extract metadata from index - data class DocMeta( - val sd: ScoreDoc, - val bookId: Long, - val bookTitle: String, - val lineId: Long, - val lineIndex: Int, - val isBaseBook: Boolean, - val orderIndex: Int, - val indexedRaw: String // from text_raw field, may be empty if not stored - ) - - val docMetas = scoreDocs.map { sd -> - val doc = stored.document(sd.doc) - DocMeta( - sd = sd, - bookId = doc.getField("book_id").numericValue().toLong(), - bookTitle = doc.getField("book_title").stringValue() ?: "", - lineId = doc.getField("line_id").numericValue().toLong(), - lineIndex = doc.getField("line_index").numericValue().toInt(), - isBaseBook = doc.getField("is_base_book")?.numericValue()?.toInt() == 1, - orderIndex = doc.getField("order_index")?.numericValue()?.toInt() ?: 999, - indexedRaw = doc.getField("text_raw")?.stringValue() ?: "" - ) - } - - val snippetSources: Map = - if (snippets) snippetSources(docMetas.map { LineSnippetInfo(it.lineId, it.bookId, it.lineIndex) }) else emptyMap() - - val hits = docMetas.map { meta -> - val source = snippetSources[meta.lineId] ?: SnippetSource(meta.indexedRaw, 0, meta.indexedRaw.length) - val raw = source.text - val baseScore = meta.sd.score - - // Calculate boost: lower orderIndex = higher boost (only for base books) - val boostedScore = if (meta.isBaseBook) { - // Formula: boost = baseScore * (1 + (120 - orderIndex) / 60) - // orderIndex 1 gets ~3x boost, orderIndex 50 gets ~2.2x boost, orderIndex 100+ gets ~1.3x boost - val boostFactor = 1.0f + (120 - meta.orderIndex).coerceAtLeast(0) / 60.0f - baseScore * boostFactor - } else { - baseScore - } - - val snippet = if (snippets) SnippetBuilder.build(raw, context.highlightPlan, source.lineStart, source.lineEnd) else "" - LineHit( - bookId = meta.bookId, - bookTitle = meta.bookTitle, - lineId = meta.lineId, - lineIndex = meta.lineIndex, - snippet = snippet, - score = boostedScore, - rawText = raw, - isBaseBook = meta.isBaseBook - ) - } - // Re-sort by boosted score (descending) - return hits.sortedByDescending { it.score } - } - - // Snippet sources: from the provider if available, otherwise none (the indexed text_raw fallback) - private suspend fun snippetSources(lines: List): Map = - snippetProvider?.getSnippetSources(lines) ?: emptyMap() + near: Int = 5, + limit: Int, + offset: Int = 0, + ): List = engine.searchAllText(rawQuery, near, limit, offset) - private suspend fun doSearch( + suspend fun searchInBook( rawQuery: String, near: Int, + bookId: Long, limit: Int, - offset: Int, - bookFilter: Long?, - categoryFilter: Long? - ): List { - val context = buildSearchContext(rawQuery, near, bookFilter, categoryFilter, null, null) ?: return emptyList() - return withContext(Dispatchers.IO) { - withSearcher { searcher -> - val top = searcher.search(context.query, offset + limit) - val stored: StoredFields = searcher.storedFields() - val sliced = top.scoreDocs.drop(offset) - mapScoreDocs(stored, sliced, context) - } - } - } + offset: Int = 0, + ): List = engine.searchInBook(rawQuery, near, bookId, limit, offset) - private suspend fun doSearchInBooks( + suspend fun searchInCategory( rawQuery: String, near: Int, + categoryId: Long, limit: Int, - offset: Int, - bookIds: Collection - ): List { - if (bookIds.isEmpty()) return emptyList() - val context = buildSearchContext(rawQuery, near, bookFilter = null, categoryFilter = null, bookIds = bookIds, lineIds = null) ?: return emptyList() - return withContext(Dispatchers.IO) { - withSearcher { searcher -> - val top = searcher.search(context.query, offset + limit) - val stored: StoredFields = searcher.storedFields() - val sliced = top.scoreDocs.drop(offset) - mapScoreDocs(stored, sliced, context) - } - } - } - - private fun analyzeToTerms(analyzer: Analyzer, text: String): List? = try { - val out = mutableListOf() - val ts: TokenStream = analyzer.tokenStream("text", text) - val termAtt = ts.addAttribute(CharTermAttribute::class.java) - ts.reset() - while (ts.incrementToken()) { - val t = termAtt.toString() - if (t.isNotBlank()) out += t - } - ts.end(); ts.close() - out - } catch (_: Exception) { null } - - private fun buildNgramPresenceForToken(token: String): Query? { - if (token.length < 4) return null - val grams = mutableListOf() - var i = 0 - val L = token.length - while (i + 4 <= L) { - grams += token.substring(i, i + 4) - i += 1 - } - if (grams.isEmpty()) return null - val b = BooleanQuery.Builder() - for (g in grams.distinct()) { - b.add(TermQuery(Term("text_ng4", g)), BooleanClause.Occur.MUST) - } - return b.build() - } + offset: Int = 0, + ): List = engine.searchInCategory(rawQuery, near, categoryId, limit, offset) - private fun buildPresenceFilterForTokens( - tokens: List, + suspend fun searchInBooks( + rawQuery: String, near: Int, - expansionsByToken: Map> - ): Query? { - if (tokens.isEmpty()) return null - val outer = BooleanQuery.Builder() - for (t in tokens) { - val expansions = expansionsByToken[t] ?: emptyList() - val synonymTerms = buildLimitedTermsForToken(t, expansions) - val ngram = if (near > 0) buildNgramPresenceForToken(t) else null - val clause = BooleanQuery.Builder().apply { - add(TermQuery(Term("text", t)), BooleanClause.Occur.SHOULD) - if (ngram != null) add(ngram, BooleanClause.Occur.SHOULD) - for (term in synonymTerms) { - if (term != t) { - add(TermQuery(Term("text", term)), BooleanClause.Occur.SHOULD) - } + bookIds: Collection, + limit: Int, + offset: Int = 0, + ): List = engine.searchInBooks(rawQuery, near, bookIds, limit, offset) + + private companion object { + fun loadDictionary( + indexDir: Path, + dictionaryPath: Path?, + ): MagicDictionaryIndex? { + val candidates = + listOfNotNull( + dictionaryPath, + System.getProperty("magicDict")?.let { Path.of(it) }, + System.getenv("SEFORIM_MAGIC_DICT")?.let { Path.of(it) }, + indexDir.resolveSibling("lexical.db"), + indexDir.resolveSibling("seforim.db").resolveSibling("lexical.db"), + Path.of("SeforimLibrary/SeforimMagicIndexer/magicindexer/build/db/lexical.db"), + ).distinct() + val firstExisting = MagicDictionaryIndex.findValidDictionary(candidates.map { it.toString() }) + if (firstExisting == null) { + logger.d { + "[MagicDictionary] Missing lexical.db; search will run without dictionary expansions. " + + "Provide -DmagicDict=/path/lexical.db or SEFORIM_MAGIC_DICT. Checked: " + + candidates.joinToString() } - }.build() - outer.add(clause, BooleanClause.Occur.MUST) - } - return outer.build() - } - - private fun buildHebrewStdQuery(norm: String, near: Int): Query { - val qb = QueryBuilder(stdAnalyzer) - val phrase = qb.createPhraseQuery("text", norm, near) - if (phrase != null) return phrase - val bool = qb.createBooleanQuery("text", norm, BooleanClause.Occur.MUST) - return bool ?: BooleanQuery.Builder().build() - } - - /** - * Builds a strict, verbatim phrase query for a quoted segment: terms must appear in the - * given order and be adjacent (slop = 0), with NO dictionary expansion, fuzzy matching or - * n-gram substring matching. A single-token segment becomes an exact [TermQuery]. - * - * Nikud and final letters are still normalised (the segment is normalised by the caller and - * tokenised through [stdAnalyzer]) so that the phrase matches the indexed form. - */ - private fun buildExactPhraseQuery(phraseNorm: String): Query? { - if (phraseNorm.isBlank()) return null - val qb = QueryBuilder(stdAnalyzer) - qb.createPhraseQuery("text", phraseNorm, 0)?.let { return it } - return qb.createBooleanQuery("text", phraseNorm, BooleanClause.Occur.MUST) - } - - private fun buildMagicBoostQuery(expansions: List): Query? { - if (expansions.isEmpty()) return null - val surfaceTerms = LinkedHashSet() - val variantTerms = LinkedHashSet() - val baseTerms = LinkedHashSet() - for (exp in expansions) { - surfaceTerms.addAll(exp.surface) - variantTerms.addAll(exp.variants) - baseTerms.addAll(exp.base) - } - - val limitedSurfaces = surfaceTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedVariants = variantTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedBases = baseTerms.take(MAX_SYNONYM_BOOST_TERMS) - if (surfaceTerms.size > limitedSurfaces.size || - variantTerms.size > limitedVariants.size || - baseTerms.size > limitedBases.size - ) { - logger.d { - "[DEBUG] Capped magic boost terms: " + - "surface=${surfaceTerms.size}->${limitedSurfaces.size}, " + - "variants=${variantTerms.size}->${limitedVariants.size}, " + - "base=${baseTerms.size}->${limitedBases.size}" - } - } - - val b = BooleanQuery.Builder() - for (s in limitedSurfaces) { - b.add(BoostQuery(TermQuery(Term("text", s)), 2.0f), BooleanClause.Occur.SHOULD) - } - for (v in limitedVariants) { - b.add(BoostQuery(TermQuery(Term("text", v)), 1.5f), BooleanClause.Occur.SHOULD) - } - for (ba in limitedBases) { - b.add(BoostQuery(TermQuery(Term("text", ba)), 1.0f), BooleanClause.Occur.SHOULD) - } - return b.build() - } - - private fun buildSynonymBoostQuery(expansions: List): Query? { - if (expansions.isEmpty()) return null - val surfaceTerms = LinkedHashSet() - val variantTerms = LinkedHashSet() - val baseTerms = LinkedHashSet() - for (exp in expansions) { - surfaceTerms.addAll(exp.surface) - variantTerms.addAll(exp.variants) - baseTerms.addAll(exp.base) - } - - val limitedSurfaces = surfaceTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedVariants = variantTerms.take(MAX_SYNONYM_BOOST_TERMS) - val limitedBases = baseTerms.take(MAX_SYNONYM_BOOST_TERMS) - if (surfaceTerms.size > limitedSurfaces.size || - variantTerms.size > limitedVariants.size || - baseTerms.size > limitedBases.size - ) { - logger.d { - "[DEBUG] Capped synonym boost terms: " + - "surface=${surfaceTerms.size}->${limitedSurfaces.size}, " + - "variants=${variantTerms.size}->${limitedVariants.size}, " + - "base=${baseTerms.size}->${limitedBases.size}" + return null } - } - - val b = BooleanQuery.Builder() - for (s in limitedSurfaces) { - b.add(TermQuery(Term("text", s)), BooleanClause.Occur.SHOULD) - } - for (v in limitedVariants) { - b.add(TermQuery(Term("text", v)), BooleanClause.Occur.SHOULD) - } - for (ba in limitedBases) { - b.add(TermQuery(Term("text", ba)), BooleanClause.Occur.SHOULD) - } - return b.build() - } - - private fun buildSynonymPhrases( - tokens: List, - expansionsByToken: Map> - ): List> { - if (tokens.isEmpty()) return emptyList() - val termExpansions = buildTermAlternativesForTokens(tokens, expansionsByToken) - logger.d { "[DEBUG] buildSynonymPhrases - termExpansions sizes: ${termExpansions.map { it.size }}" } - fun buildMultiPhrase(slop: Int): Query { - val builder = org.apache.lucene.search.MultiPhraseQuery.Builder() - builder.setSlop(slop) - var pos = 0 - for (alts in termExpansions) { - builder.add(alts.map { Term("text", it) }.toTypedArray(), pos) - pos++ + logger.d { "[MagicDictionary] Loading lexical db from $firstExisting" } + return MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, firstExisting).also { + if (it == null) logger.d { "[MagicDictionary] Failed to load lexical db at $firstExisting; continuing without dictionary expansions" } } - return builder.build() } - return listOf( - buildMultiPhrase(0) to 50.0f, - buildMultiPhrase(3) to 20.0f, - buildMultiPhrase(8) to 5.0f - ) } +} - private fun buildSynonymPhraseQuery( - tokens: List, - expansionsByToken: Map>, - near: Int - ): Query? { - if (tokens.isEmpty()) return null - val termExpansions = buildTermAlternativesForTokens(tokens, expansionsByToken) - val builder = org.apache.lucene.search.MultiPhraseQuery.Builder() - builder.setSlop(near) - var position = 0 - for (alts in termExpansions) { - builder.add(alts.map { Term("text", it) }.toTypedArray(), position) - position++ - } - return builder.build() - } - - private fun buildNgram4Query(norm: String): Query? { - val tokens = norm.split("\\s+".toRegex()).map { it.trim() }.filter { it.length >= 4 } - if (tokens.isEmpty()) return null - val grams = mutableListOf() - for (t in tokens) { - val L = t.length - var i = 0 - while (i + 4 <= L) { - grams += t.substring(i, i + 4) - i += 1 - } - } - val uniq = grams.distinct() - if (uniq.isEmpty()) return null - val b = BooleanQuery.Builder() - for (g in uniq) { - b.add(TermQuery(Term("text_ng4", g)), BooleanClause.Occur.MUST) - } - return b.build() - } - - private fun buildExpandedQuery( - norm: String, - near: Int, - tokens: List, - expansionsByToken: Map> - ): Query { - val base = buildHebrewStdQuery(norm, near) - val allExpansions = expansionsByToken.values.flatten() - val synonymPhrases = buildSynonymPhrases(tokens, expansionsByToken) - val ngram = buildNgram4Query(norm) - val fuzzy = buildFuzzyQuery(norm, near) - val builder = BooleanQuery.Builder() - builder.add(base, BooleanClause.Occur.SHOULD) - for ((query, boost) in synonymPhrases) { - builder.add(BoostQuery(query, boost), BooleanClause.Occur.SHOULD) - } - if (ngram != null) builder.add(ngram, BooleanClause.Occur.SHOULD) - if (fuzzy != null) builder.add(fuzzy, BooleanClause.Occur.SHOULD) - val magic = buildMagicBoostQuery(allExpansions) - if (magic != null) builder.add(magic, BooleanClause.Occur.SHOULD) - val synonymBoost = buildSynonymBoostQuery(allExpansions) - if (synonymBoost != null) builder.add(synonymBoost, BooleanClause.Occur.SHOULD) - return builder.build() - } - - private fun buildFuzzyQuery(norm: String, near: Int): Query? { - if (near == 0) return null - if (norm.length < 4) return null - val tokens = analyzeToTerms(stdAnalyzer, norm)?.filter { it.length >= 4 } ?: emptyList() - if (tokens.isEmpty()) return null - val b = BooleanQuery.Builder() - for (t in tokens.distinct()) { - b.add(FuzzyQuery(Term("text", t), 1), BooleanClause.Occur.MUST) - } - return b.build() - } - - private fun buildLimitedTermsForToken( - token: String, - expansions: List - ): List { - if (expansions.isEmpty()) return listOf(token) - - val baseTerms = expansions.flatMap { it.base }.distinct() - val otherTerms = expansions.flatMap { it.surface + it.variants }.distinct() - - val ordered = LinkedHashSet() - if (token.isNotBlank()) { - ordered += token - } - baseTerms.forEach { ordered += it } - otherTerms.forEach { ordered += it } - - val totalSize = ordered.size - val limited = ordered.take(MAX_SYNONYM_TERMS_PER_TOKEN) - if (totalSize > limited.size) { - logger.d { - "[DEBUG] Capped synonym terms for token '$token' from $totalSize to ${limited.size}" - } - } - return limited - } - - private fun buildTermAlternativesForTokens( - tokens: List, - expansionsByToken: Map> - ): List> { - if (tokens.isEmpty()) return emptyList() - return tokens.map { token -> - val expansions = expansionsByToken[token] ?: emptyList() - buildLimitedTermsForToken(token, expansions) - } - } - - // The divine name's forms never change: read once from the dictionary, not on every search - private val hashemHighlightTerms: List by lazy { readHashemHighlightTerms() } - - private fun loadHashemHighlightTerms(): List = hashemHighlightTerms - - private fun readHashemHighlightTerms(): List { - val dict = magicDict ?: return emptyList() - val raw = dict.loadHashemSurfaces() - if (raw.isEmpty()) return emptyList() +/** [MagicDictionaryIndex.load] for a [Path]. */ +fun MagicDictionaryIndex.Companion.load( + norm: (String) -> String, + candidate: Path?, +): MagicDictionaryIndex? = load(norm, candidate?.toString()) - val terms = linkedSetOf() - raw.forEach { value -> - val trimmed = value.trim() - if (trimmed.isEmpty()) return@forEach - terms += trimmed - val stripped = HebrewTextUtils.stripDiacritics(trimmed).trim() - if (stripped.isNotEmpty()) terms += stripped - val normalized = HebrewTextUtils.normalizeHebrew(trimmed).trim() - if (normalized.isNotEmpty()) terms += normalized - } - - val out = terms.toList() - logger.d { "[DEBUG] Hashem highlight terms from lexical DB: ${out.take(20)}..." } - return out - } -} +/** [MagicDictionaryIndex.findValidDictionary] for [Path]s. */ +@JvmName("findValidDictionaryPath") +fun MagicDictionaryIndex.Companion.findValidDictionary(candidates: List): Path? = + findValidDictionary(candidates.map { it.toString() })?.let { Path.of(it) } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt new file mode 100644 index 00000000..57a46cb6 --- /dev/null +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt @@ -0,0 +1,42 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import java.sql.Connection +import java.sql.DriverManager +import java.util.concurrent.ConcurrentLinkedQueue + +/** Through JDBC (sqlite-jdbc), with a connection per thread: SQLite connections serialize their statements. */ +actual fun openReadOnlySqlite(path: String): SqliteReader = JdbcSqliteReader(path).also { it.query("SELECT 1") } + +private class JdbcSqliteReader( + path: String, +) : SqliteReader { + private val url = "jdbc:sqlite:${java.io.File(path).absolutePath}" + private val opened = ConcurrentLinkedQueue() + private val connection = + ThreadLocal.withInitial { + DriverManager.getConnection(url).apply { + autoCommit = false + // Enforce read-only queries without altering connection flags post-open + createStatement().use { it.execute("PRAGMA query_only=ON") } + opened += this + } + } + + override fun query( + sql: String, + args: List, + ): List> = + connection.get().prepareStatement(sql).use { stmt -> + args.forEachIndexed { i, arg -> stmt.setString(i + 1, arg) } + stmt.executeQuery().use { rs -> + val columns = (1..rs.metaData.columnCount).map { rs.metaData.getColumnLabel(it) } + val rows = ArrayList>() + while (rs.next()) rows += columns.associateWith { rs.getString(it) } + rows + } + } + + override fun close() { + generateSequence { opened.poll() }.forEach { runCatching { it.close() } } + } +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt new file mode 100644 index 00000000..80aa7175 --- /dev/null +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt @@ -0,0 +1,82 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.SearchFacets +import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor +import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc +import io.github.kdroidfilter.seforimlibrary.search.index.IndexPage +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend +import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.builtins.LongArraySerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.JsonNull +import java.nio.file.Path +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +/** + * The Java side of the bridge (see `BridgedLuceneIndex`): runs the calls of an app's [LuceneIndex]es on + * [LuceneIndexBackend]s, where Lucene can't run in the app itself: the iOS app calls [call] through JNI, from any + * thread, in the Java runtime it embeds. + */ +object IndexBridgeServer { + private val ids = AtomicInteger() + private val indexes = ConcurrentHashMap() + private val cursors = ConcurrentHashMap() + + /** Runs the call [request] (a [BridgeRequest] as UTF-8 JSON) and returns its [BridgeResponse] (UTF-8 JSON). */ + @JvmStatic + fun call(request: ByteArray): ByteArray { + val response = + try { + BridgeResponse(result = run(BridgeJson.decodeFromString(BridgeRequest.serializer(), request.decodeToString()))) + } catch (e: Throwable) { + BridgeResponse(error = e.stackTraceToString()) + } + return BridgeJson.encodeToString(BridgeResponse.serializer(), response).encodeToByteArray() + } + + private fun run(request: BridgeRequest) = + when (request) { + is BridgeRequest.Open -> { + val id = ids.incrementAndGet() + indexes[id] = LuceneIndexBackend(Path.of(request.indexDir), cacheQueries = request.cacheQueries) + BridgeJson.encodeToJsonElement(Int.serializer(), id) + } + is BridgeRequest.Close -> { + indexes.remove(request.index)?.close() + JsonNull + } + is BridgeRequest.Analyze -> + BridgeJson.encodeToJsonElement(ListSerializer(String.serializer()), index(request.index).analyze(request.text)) + is BridgeRequest.Search -> + page(index(request.index).search(request.query, request.limit, request.fields, request.sort)) + is BridgeRequest.OpenCursor -> { + val id = ids.incrementAndGet() + cursors[id] = index(request.index).openCursor(request.query, request.fields) + BridgeJson.encodeToJsonElement(Int.serializer(), id) + } + is BridgeRequest.CursorNext -> + page(cursors[request.cursor]?.next(request.limit) ?: error("Unknown cursor ${request.cursor}")) + is BridgeRequest.CloseCursor -> { + cursors.remove(request.cursor)?.close() + JsonNull + } + is BridgeRequest.Facets -> + BridgeJson.encodeToJsonElement(SearchFacets.serializer(), index(request.index).facets(request.query)) + is BridgeRequest.CollectLongs -> + BridgeJson.encodeToJsonElement(LongArraySerializer(), index(request.index).collectLongs(request.query, request.field)) + is BridgeRequest.Count -> + BridgeJson.encodeToJsonElement(Int.serializer(), index(request.index).count(request.query)) + is BridgeRequest.VectorEncodingOf -> + index(request.index).vectorEncoding(request.field)?.let { BridgeJson.encodeToJsonElement(VectorEncoding.serializer(), it) } + ?: JsonNull + is BridgeRequest.Knn -> + BridgeJson.encodeToJsonElement(ListSerializer(IndexDoc.serializer()), index(request.index).knn(request.request)) + } + + private fun index(id: Int): LuceneIndex = indexes[id] ?: error("Unknown index $id") + + private fun page(page: IndexPage) = BridgeJson.encodeToJsonElement(IndexPage.serializer(), page) +} diff --git a/search/src/jvmMain/resources/hallucination_blacklist.tsv b/search/src/jvmMain/resources/hallucination_blacklist.tsv deleted file mode 100644 index dd146a14..00000000 --- a/search/src/jvmMain/resources/hallucination_blacklist.tsv +++ /dev/null @@ -1,29 +0,0 @@ -# Hallucination Blacklist -# Format: tokenbase -# Lines starting with # are comments -# These are LLM-generated mappings that are incorrect and should be ignored -לחתוכ כנ -לחתוך כנ -לבקש רצה -מתפלל בעה -לשבת נתנ -לעבוד עשה -להעמיד קומ -כנ כנ -בנ בנ -לשימ שמ -לשים שמ -לדונ יד -לדון יד -אמ אמ -להרגיע יושב -לחייכ חוה -לחייך חוה -שנ שנ -עצ עצ -דנ דנ -להיכנס עלה -לשקול לקח -לבנות בנ -לשחרר שרה -להתיר שרה diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt new file mode 100644 index 00000000..1e7d286f --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/RealIndexParityTest.kt @@ -0,0 +1,23 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlinx.coroutines.runBlocking +import java.io.File +import java.nio.file.Path +import kotlin.test.Test + +/** + * Dumps what the engine answers on a real index (-PindexDir=) into build/parity/jvm.txt, for the + * parity with Android and iOS (see SearchDump). Skipped without an index. + */ +class RealIndexParityTest { + @Test + fun dumpRealIndex() { + val indexDir = System.getProperty("seforimIndexDir") ?: return + val out = File(System.getProperty("parityOut") ?: "build/parity/jvm.txt") + LuceneSearchEngine(Path.of(indexDir)).use { engine -> + out.parentFile.mkdirs() + out.writeText(runBlocking { SearchDump.dump(engine) }) + } + println("Parity dump: ${out.absolutePath}") + } +} diff --git a/search/src/nativeInterop/cinterop/jni.def b/search/src/nativeInterop/cinterop/jni.def new file mode 100644 index 00000000..0d5c1425 --- /dev/null +++ b/search/src/nativeInterop/cinterop/jni.def @@ -0,0 +1,3 @@ +# The JNI of the Java runtime the iOS app embeds (OpenJDK mobile, statically linked into the app: see IOS_RUNTIME.md) +headers = jni.h +package = io.github.kdroidfilter.seforimlibrary.search.jni diff --git a/search/src/nativeInterop/cinterop/sqlite3.def b/search/src/nativeInterop/cinterop/sqlite3.def new file mode 100644 index 00000000..f289e75f --- /dev/null +++ b/search/src/nativeInterop/cinterop/sqlite3.def @@ -0,0 +1,4 @@ +# The system SQLite of iOS, for the lexical dictionary +headers = sqlite3.h +package = io.github.kdroidfilter.seforimlibrary.search.sqlite3 +linkerOpts = -lsqlite3 diff --git a/settings.gradle.kts b/settings.gradle.kts index be36cb50..611f311d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -69,6 +69,7 @@ dependencyResolutionManagement { include(":core") include(":dao") include(":search") +include(":lucene-android") include(":cli") include(":catalog") include(":searchindex") From 86f0355ba50f378d1f879a2229bd84de3d6653a9 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 09:39:54 +0300 Subject: [PATCH 02/14] fix(search): release the iOS runtime's threads and close what the engine owns - iOS: threads attached to the embedded Java runtime are detached when they exit (pthread key destructor in C), or after each call when no destructor can be registered; static methods cached by class and method. - SQLite readers: iOS throws on step errors and refuses queries after close (no use of the freed handle); the JVM one keeps its statements prepared per thread and opens no connection after close. - The engine closes its dictionary, including one that finishes loading after close. - Bridge: closing an index closes its open cursors; a cursor's reads and close take turns. - Each quoted phrase is analyzed once; the bridged index caches its analyses. Signed-off-by: Elie Gambache --- .../seforimlibrary/search/Sqlite.android.kt | 4 +- .../search/LexicalSearchEngine.kt | 40 ++++++++--- .../search/index/bridge/BridgedLuceneIndex.kt | 9 ++- .../seforimlibrary/search/Sqlite.ios.kt | 32 ++++++++- .../search/index/bridge/EmbeddedJvm.kt | 46 +++++++----- .../seforimlibrary/search/Sqlite.jvm.kt | 51 +++++++++---- .../search/index/bridge/IndexBridgeServer.kt | 28 +++++++- .../seforimlibrary/search/SqliteReaderTest.kt | 52 ++++++++++++++ .../index/bridge/IndexBridgeServerTest.kt | 72 +++++++++++++++++++ search/src/nativeInterop/cinterop/jni.def | 32 +++++++++ 10 files changed, 317 insertions(+), 49 deletions(-) create mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt create mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt diff --git a/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt b/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt index a696b1fa..891591d2 100644 --- a/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt +++ b/search/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.android.kt @@ -21,5 +21,7 @@ private class AndroidSqliteReader( rows } - override fun close() = db.close() + override fun close() { + if (db.isOpen) db.close() + } } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt index 5529c24f..6dbbbe30 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt @@ -10,6 +10,7 @@ import io.github.kdroidfilter.seforimlibrary.search.index.Occur import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.withContext +import kotlin.concurrent.Volatile private val logger = Logger.withTag("LexicalSearchEngine") @@ -45,9 +46,21 @@ private val HIT_FIELDS = setOf("book_id", "book_title", "line_id", "line_index", class LexicalSearchEngine( private val index: LuceneIndex, private val snippetProvider: SnippetProvider? = null, - dictionary: Lazy = lazyOf(null), + private val dictionary: Lazy = lazyOf(null), ) : SearchEngine { - private val magicDict: MagicDictionaryIndex? by dictionary + @Volatile + private var closed = false + + // A dictionary that finished loading after close() is closed by its first reader: close() couldn't see it + private val magicDict: MagicDictionaryIndex? + get() { + val dict = dictionary.value + if (closed) { + dict?.close() + return null + } + return dict + } // --- SearchEngine interface implementation --- @@ -157,7 +170,7 @@ class LexicalSearchEngine( private fun buildHighlightPlan( tokens: List, hasHashem: Boolean, - exactPhrases: List, + exactPhrases: List>, ): HighlightPlan { val forms = tokens.distinct().map { token -> @@ -175,7 +188,7 @@ class LexicalSearchEngine( } val phrases = exactPhrases - .map { index.analyze(it).joinToString(" ") } + .map { it.joinToString(" ") } .filter { it.isNotBlank() } return HighlightPlan.build(forms, phrases) } @@ -220,7 +233,11 @@ class LexicalSearchEngine( } override fun close() { + closed = true index.close() + // The engine owns its dictionary (and its SQLite connections) once loaded; one still loading is closed by + // magicDict + if (dictionary.isInitialized()) dictionary.value?.close() } override fun computeFacets( @@ -339,6 +356,8 @@ class LexicalSearchEngine( parsed.exactPhrases .map { HebrewTextUtils.normalizeHebrew(it) } .filter { it.isNotBlank() } + // Analyzed once, for both the phrase clauses and the highlights (a bridge call on iOS) + val exactPhraseTokens = exactPhrasesNorm.map { index.analyze(it) } if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null @@ -373,7 +392,7 @@ class LexicalSearchEngine( } // What the snippets highlight: built on first use (facets need none) - val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhrasesNorm) } + val highlightPlan = lazy { buildHighlightPlan(analyzedStd, hasHashem, exactPhraseTokens) } val builder = BoolBuilder() // Mirrors every required clause of builder (the facets need only the hit set) @@ -401,8 +420,8 @@ class LexicalSearchEngine( } // Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase. var exactClausesAdded = 0 - for (phrase in exactPhrasesNorm) { - val exactQuery = buildExactPhraseQuery(phrase) ?: continue + for ((phrase, tokens) in exactPhrasesNorm.zip(exactPhraseTokens)) { + val exactQuery = buildExactPhraseQuery(phrase, tokens) ?: continue require(exactQuery, Occur.MUST) exactClausesAdded++ logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" } @@ -565,8 +584,11 @@ class LexicalSearchEngine( * Nikud and final letters are still normalised (the segment is normalised by the caller and * tokenised like the index) so that the phrase matches the indexed form. */ - private fun buildExactPhraseQuery(phraseNorm: String): IndexQuery? { - if (phraseNorm.isBlank() || index.analyze(phraseNorm).isEmpty()) return null + private fun buildExactPhraseQuery( + phraseNorm: String, + tokens: List, + ): IndexQuery? { + if (phraseNorm.isBlank() || tokens.isEmpty()) return null return IndexQuery.AnalyzedPhrase("text", phraseNorm, 0) } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt index 8783a1d5..a3c4ce44 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt @@ -1,5 +1,6 @@ package io.github.kdroidfilter.seforimlibrary.search.index.bridge +import io.github.kdroidfilter.seforimlibrary.search.LruCache import io.github.kdroidfilter.seforimlibrary.search.SearchFacets import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc @@ -25,6 +26,8 @@ fun interface BridgeTransport { fun call(request: ByteArray): ByteArray } +private const val ANALYZE_CACHE_SIZE = 64 + /** * The [LuceneIndex] at [indexDir] where Lucene can't run in process: in the Java runtime an iOS app embeds, in the Java * process an Android app before 15 starts. Lucene runs there with the same implementation as on the JVM and Android 15+ @@ -39,7 +42,11 @@ class BridgedLuceneIndex( ) : LuceneIndex { private val handle: Int = call(BridgeRequest.Open(indexDir, cacheQueries), Int.serializer()) - override fun analyze(text: String): List = call(BridgeRequest.Analyze(handle, text), ListSerializer(String.serializer())) + // Analysis depends on the text alone: the same query is analyzed by its facets, its session and its highlights + private val analyzed = LruCache>(ANALYZE_CACHE_SIZE) + + override fun analyze(text: String): List = + analyzed[text] ?: call(BridgeRequest.Analyze(handle, text), ListSerializer(String.serializer())).also { analyzed[text] = it } override fun search( query: IndexQuery, diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt index d44b8ac2..bb270b48 100644 --- a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.ios.kt @@ -2,6 +2,7 @@ package io.github.kdroidfilter.seforimlibrary.search +import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_DONE import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OK import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OPEN_FULLMUTEX import io.github.kdroidfilter.seforimlibrary.search.sqlite3.SQLITE_OPEN_READONLY @@ -18,6 +19,8 @@ import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_open_v2 import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_prepare_v2 import cnames.structs.sqlite3_stmt import io.github.kdroidfilter.seforimlibrary.search.sqlite3.sqlite3_step +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized import kotlinx.cinterop.CFunction import kotlinx.cinterop.COpaquePointer import kotlinx.cinterop.CPointer @@ -31,7 +34,7 @@ import kotlinx.cinterop.toCPointer import kotlinx.cinterop.toKString import kotlinx.cinterop.value -/** Through the system SQLite, one connection in serialized mode: its queries are safe from any thread. */ +/** Through the system SQLite, one connection whose queries take turns: safe from any thread. */ actual fun openReadOnlySqlite(path: String): SqliteReader = NativeSqliteReader(path) // SQLITE_TRANSIENT: SQLite copies the bound text (a C macro cinterop can't map) @@ -53,9 +56,23 @@ private class NativeSqliteReader( opened } + // Queries take turns (the connection serializes them anyway): an error message is then the failing query's, and + // close never frees the connection under a running query + private val lock = SynchronizedObject() + private var closed = false + override fun query( sql: String, args: List, + ): List> = + synchronized(lock) { + check(!closed) { "SQLite: the database is closed" } + queryLocked(sql, args) + } + + private fun queryLocked( + sql: String, + args: List, ): List> = memScoped { val out = alloc>() @@ -65,16 +82,25 @@ private class NativeSqliteReader( args.forEachIndexed { i, arg -> sqlite3_bind_text(stmt, i + 1, arg, -1, TRANSIENT) } val columns = (0 until sqlite3_column_count(stmt)).map { sqlite3_column_name(stmt, it)?.toKString() ?: "" } val rows = ArrayList>() - while (sqlite3_step(stmt) == SQLITE_ROW) { + var status = sqlite3_step(stmt) + while (status == SQLITE_ROW) { rows += columns.indices.associate { columns[it] to sqlite3_column_text(stmt, it)?.reinterpret()?.toKString() } + status = sqlite3_step(stmt) } + // Any other end is a failure (busy, I/O, corruption), not the last row: thrown, as the JVM and Android do + check(status == SQLITE_DONE) { "SQLite: ${sqlite3_errmsg(db)?.toKString()}" } rows } finally { sqlite3_finalize(stmt) } } + // Closing twice would free the connection twice override fun close() { - sqlite3_close_v2(db) + synchronized(lock) { + if (closed) return + closed = true + sqlite3_close_v2(db) + } } } diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt index d13f9495..12bbaaca 100644 --- a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt @@ -4,7 +4,6 @@ package io.github.kdroidfilter.seforimlibrary.search.index.bridge import io.github.kdroidfilter.seforimlibrary.search.jni.JNIEnvVar import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_CreateJavaVM -import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_EDETACHED import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_GetCreatedJavaVMs import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_OK import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_TRUE @@ -17,11 +16,13 @@ import io.github.kdroidfilter.seforimlibrary.search.jni.jclass import io.github.kdroidfilter.seforimlibrary.search.jni.jintVar import io.github.kdroidfilter.seforimlibrary.search.jni.jmethodID import io.github.kdroidfilter.seforimlibrary.search.jni.jvalue +import io.github.kdroidfilter.seforimlibrary.search.jni.seforim_attach_current_thread import kotlinx.atomicfu.locks.SynchronizedObject import kotlinx.atomicfu.locks.synchronized import kotlinx.cinterop.CPointer import kotlinx.cinterop.CPointerVar import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.IntVar import kotlinx.cinterop.addressOf import kotlinx.cinterop.alloc import kotlinx.cinterop.allocArray @@ -48,8 +49,8 @@ class EmbeddedJvm private constructor( ) { private val lock = SynchronizedObject() - // One bridge method per process (IndexBridgeServer.call) - private var bridge: Pair? = null + // The static methods called so far, by "class.method" + private val methods = HashMap>() /** * Calls the static method `byte[] method(byte[])` of [className] with [request], from the calling thread @@ -59,10 +60,19 @@ class EmbeddedJvm private constructor( className: String, method: String, request: ByteArray, + ): ByteArray = + withAttachedEnv { env -> + callStatic(env, className, method, request) + } + + private fun callStatic( + env: CPointer, + className: String, + method: String, + request: ByteArray, ): ByteArray { - val env = attachedEnv() val functions = env.pointed.pointed!! - val (clazz, methodId) = bridgeMethod(env, className, method) + val (clazz, methodId) = staticMethod(env, className, method) val array = functions.NewByteArray!!(env, request.size) ?: throwPending(env, "NewByteArray") try { if (request.isNotEmpty()) { @@ -95,36 +105,34 @@ class EmbeddedJvm private constructor( } } - // The class and method are resolved once: a global reference keeps the class - private fun bridgeMethod( + // Each class and method is resolved once: a global reference keeps the class + private fun staticMethod( env: CPointer, className: String, method: String, ): Pair = synchronized(lock) { - bridge ?: memScoped { + methods["$className.$method"] ?: memScoped { val functions = env.pointed.pointed!! val local = functions.FindClass!!(env, className.replace('.', '/').cstr.getPointer(this)) ?: throwPending(env, className) val global = functions.NewGlobalRef!!(env, local)!! functions.DeleteLocalRef!!(env, local) val methodId = functions.GetStaticMethodID!!(env, global, method.cstr.getPointer(this), "([B)[B".cstr.getPointer(this)) ?: throwPending(env, "$className.$method") - (global to methodId).also { bridge = it } + (global to methodId).also { methods["$className.$method"] = it } } } - private fun attachedEnv(): CPointer = + // Attached once per thread, detached when the thread exits (see jni.def), or after the call when that can't be + private inline fun withAttachedEnv(block: (CPointer) -> T): T = memScoped { - val env = alloc>() - val invoke = vm.pointed.pointed!! - val status = invoke.GetEnv!!(vm, env.ptr.reinterpret(), JNI_VERSION_21) - if (status == JNI_EDETACHED) { - // Daemon: an attached Kotlin worker never holds the runtime up at exit - check(invoke.AttachCurrentThreadAsDaemon!!(vm, env.ptr.reinterpret(), null) == JNI_OK) { "Can't attach the thread to the Java runtime" } - } else { - check(status == JNI_OK) { "GetEnv failed: $status" } + val detachAfter = alloc() + val env = seforim_attach_current_thread(vm, detachAfter.ptr) ?: error("Can't attach the thread to the Java runtime") + try { + block(env) + } finally { + if (detachAfter.value != 0) vm.pointed.pointed!!.DetachCurrentThread!!(vm) } - env.value!! } private fun throwPending( diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt index 57a46cb6..63182d68 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/Sqlite.jvm.kt @@ -2,6 +2,7 @@ package io.github.kdroidfilter.seforimlibrary.search import java.sql.Connection import java.sql.DriverManager +import java.sql.PreparedStatement import java.util.concurrent.ConcurrentLinkedQueue /** Through JDBC (sqlite-jdbc), with a connection per thread: SQLite connections serialize their statements. */ @@ -10,33 +11,55 @@ actual fun openReadOnlySqlite(path: String): SqliteReader = JdbcSqliteReader(pat private class JdbcSqliteReader( path: String, ) : SqliteReader { + // A thread's connection, with its statements prepared once: the dictionary runs the same lookup for every word + private class ThreadConnection( + val connection: Connection, + ) { + val statements = HashMap() + } + private val url = "jdbc:sqlite:${java.io.File(path).absolutePath}" private val opened = ConcurrentLinkedQueue() + + @Volatile + private var closed = false private val connection = ThreadLocal.withInitial { - DriverManager.getConnection(url).apply { - autoCommit = false - // Enforce read-only queries without altering connection flags post-open - createStatement().use { it.execute("PRAGMA query_only=ON") } - opened += this + val connection = + DriverManager.getConnection(url).apply { + autoCommit = false + // Enforce read-only queries without altering connection flags post-open + createStatement().use { it.execute("PRAGMA query_only=ON") } + } + opened += connection + // Opened by a thread racing close(): closed here, as close() may already have drained the queue + if (closed) { + connection.close() + error("SQLite: the database is closed") } + ThreadConnection(connection) } override fun query( sql: String, args: List, - ): List> = - connection.get().prepareStatement(sql).use { stmt -> - args.forEachIndexed { i, arg -> stmt.setString(i + 1, arg) } - stmt.executeQuery().use { rs -> - val columns = (1..rs.metaData.columnCount).map { rs.metaData.getColumnLabel(it) } - val rows = ArrayList>() - while (rs.next()) rows += columns.associateWith { rs.getString(it) } - rows - } + ): List> { + // A thread that never queried would otherwise open a connection no one closes + check(!closed) { "SQLite: the database is closed" } + val thread = connection.get() + val stmt = thread.statements.getOrPut(sql) { thread.connection.prepareStatement(sql) } + args.forEachIndexed { i, arg -> stmt.setString(i + 1, arg) } + return stmt.executeQuery().use { rs -> + val columns = (1..rs.metaData.columnCount).map { rs.metaData.getColumnLabel(it) } + val rows = ArrayList>() + while (rs.next()) rows += columns.associateWith { rs.getString(it) } + rows } + } + // Closing a connection closes its statements override fun close() { + closed = true generateSequence { opened.poll() }.forEach { runCatching { it.close() } } } } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt index 80aa7175..6d5bf9a8 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt @@ -23,7 +23,29 @@ import java.util.concurrent.atomic.AtomicInteger object IndexBridgeServer { private val ids = AtomicInteger() private val indexes = ConcurrentHashMap() - private val cursors = ConcurrentHashMap() + private val cursors = ConcurrentHashMap() + + // A cursor with the index it reads, closed with it. Its calls take turns: a page being read when the index + // closes finishes before its searcher is released + private class OpenCursor( + val index: Int, + private val cursor: IndexCursor, + ) { + private var closed = false + + @Synchronized + fun next(limit: Int): IndexPage { + check(!closed) { "Closed cursor" } + return cursor.next(limit) + } + + @Synchronized + fun close() { + if (closed) return + closed = true + cursor.close() + } + } /** Runs the call [request] (a [BridgeRequest] as UTF-8 JSON) and returns its [BridgeResponse] (UTF-8 JSON). */ @JvmStatic @@ -45,6 +67,8 @@ object IndexBridgeServer { BridgeJson.encodeToJsonElement(Int.serializer(), id) } is BridgeRequest.Close -> { + // The cursors left open hold the index's reader: closed first + cursors.filterValues { it.index == request.index }.keys.forEach { cursors.remove(it)?.close() } indexes.remove(request.index)?.close() JsonNull } @@ -54,7 +78,7 @@ object IndexBridgeServer { page(index(request.index).search(request.query, request.limit, request.fields, request.sort)) is BridgeRequest.OpenCursor -> { val id = ids.incrementAndGet() - cursors[id] = index(request.index).openCursor(request.query, request.fields) + cursors[id] = OpenCursor(request.index, index(request.index).openCursor(request.query, request.fields)) BridgeJson.encodeToJsonElement(Int.serializer(), id) } is BridgeRequest.CursorNext -> diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt new file mode 100644 index 00000000..4d9f37a6 --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SqliteReaderTest.kt @@ -0,0 +1,52 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import java.nio.file.Files +import java.sql.DriverManager +import kotlin.concurrent.thread +import kotlin.io.path.deleteIfExists +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertIs + +class SqliteReaderTest { + private fun withDatabase(block: (String) -> Unit) { + val file = Files.createTempFile("reader", ".db") + try { + DriverManager.getConnection("jdbc:sqlite:$file").use { conn -> + conn.createStatement().use { stmt -> + stmt.execute("CREATE TABLE word (value TEXT, base TEXT)") + stmt.execute("INSERT INTO word VALUES ('שלום', 'שלם'), ('שלם', 'שלם'), ('תורה', 'ירה')") + } + } + block(file.toString()) + } finally { + file.deleteIfExists() + } + } + + @Test + fun `a statement run again binds its new arguments`() = + withDatabase { path -> + openReadOnlySqlite(path).use { db -> + val sql = "SELECT value FROM word WHERE base = ? ORDER BY value" + assertEquals(listOf("שלום", "שלם"), db.query(sql, listOf("שלם")).map { it["value"] }) + assertEquals(listOf("תורה"), db.query(sql, listOf("ירה")).map { it["value"] }) + assertEquals(emptyList(), db.query(sql, listOf("אין"))) + } + } + + @Test + fun `a closed reader refuses queries, from any thread`() = + withDatabase { path -> + val db = openReadOnlySqlite(path) + db.query("SELECT 1 AS one") + db.close() + assertFailsWith { db.query("SELECT 1 AS one") } + // A thread that never queried doesn't open a connection either + var failure: Throwable? = null + thread { failure = runCatching { db.query("SELECT 1 AS one") }.exceptionOrNull() }.join() + assertIs(failure) + db.close() + } +} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt new file mode 100644 index 00000000..6de7c6c6 --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt @@ -0,0 +1,72 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import kotlinx.serialization.builtins.serializer +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.document.Document +import org.apache.lucene.document.Field +import org.apache.lucene.document.StringField +import org.apache.lucene.index.IndexWriter +import org.apache.lucene.index.IndexWriterConfig +import org.apache.lucene.store.FSDirectory +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.deleteRecursively +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class IndexBridgeServerTest { + private val indexDir: Path = + Files.createTempDirectory("bridge_index").also { dir -> + IndexWriter(FSDirectory.open(dir), IndexWriterConfig(StandardAnalyzer())).use { writer -> + repeat(3) { writer.addDocument(Document().apply { add(StringField("type", "line", Field.Store.YES)) }) } + } + } + + @OptIn(ExperimentalPathApi::class) + @AfterTest + fun cleanUp() { + indexDir.deleteRecursively() + } + + private fun call(request: BridgeRequest): BridgeResponse = + BridgeJson.decodeFromString( + BridgeResponse.serializer(), + IndexBridgeServer.call(BridgeJson.encodeToString(BridgeRequest.serializer(), request).encodeToByteArray()).decodeToString(), + ) + + private fun callInt(request: BridgeRequest): Int = BridgeJson.decodeFromJsonElement(Int.serializer(), assertNotNull(call(request).result)) + + @Test + fun `a cursor reads its pages until closed`() { + val index = callInt(BridgeRequest.Open(indexDir.toString(), cacheQueries = false)) + val cursor = callInt(BridgeRequest.OpenCursor(index, IndexQuery.MatchAll, emptySet())) + + assertNull(call(BridgeRequest.CursorNext(cursor, 2)).error) + call(BridgeRequest.CloseCursor(cursor)) + assertTrue(call(BridgeRequest.CursorNext(cursor, 2)).error.orEmpty().contains("Unknown cursor")) + call(BridgeRequest.Close(index)) + } + + @Test + fun `closing an index closes the cursors left open on it`() { + val index = callInt(BridgeRequest.Open(indexDir.toString(), cacheQueries = false)) + val other = callInt(BridgeRequest.Open(indexDir.toString(), cacheQueries = false)) + val cursor = callInt(BridgeRequest.OpenCursor(index, IndexQuery.MatchAll, emptySet())) + val otherCursor = callInt(BridgeRequest.OpenCursor(other, IndexQuery.MatchAll, emptySet())) + + call(BridgeRequest.Close(index)) + + assertTrue(call(BridgeRequest.CursorNext(cursor, 2)).error.orEmpty().contains("Unknown cursor")) + // Another index's cursors are left alone + val page = call(BridgeRequest.CursorNext(otherCursor, 5)) + assertNull(page.error) + call(BridgeRequest.Close(other)) + assertEquals(null, call(BridgeRequest.Close(other)).error) + } +} diff --git a/search/src/nativeInterop/cinterop/jni.def b/search/src/nativeInterop/cinterop/jni.def index 0d5c1425..9501d5e5 100644 --- a/search/src/nativeInterop/cinterop/jni.def +++ b/search/src/nativeInterop/cinterop/jni.def @@ -1,3 +1,35 @@ # The JNI of the Java runtime the iOS app embeds (OpenJDK mobile, statically linked into the app: see IOS_RUNTIME.md) headers = jni.h package = io.github.kdroidfilter.seforimlibrary.search.jni + +--- + +#include + +// A thread attached by seforim_attach_current_thread is detached when it exits, from this destructor: in plain C, as +// the Kotlin runtime of an exiting thread may already be gone +static pthread_key_t seforim_detach_key; +static int seforim_detach_key_ok = 0; +static pthread_once_t seforim_detach_once = PTHREAD_ONCE_INIT; + +static void seforim_detach_current_thread(void *vm) { + (*(JavaVM *) vm)->DetachCurrentThread((JavaVM *) vm); +} + +static void seforim_create_detach_key(void) { + seforim_detach_key_ok = pthread_key_create(&seforim_detach_key, seforim_detach_current_thread) == 0; +} + +// The JNIEnv of the calling thread, attached to vm (as a daemon) if it isn't yet; NULL when it can't be. *detach_after +// is set when the caller must detach the thread itself once done: no thread-exit destructor could be registered +static JNIEnv *seforim_attach_current_thread(JavaVM *vm, int *detach_after) { + JNIEnv *env = NULL; + *detach_after = 0; + jint status = (*vm)->GetEnv(vm, (void **) &env, JNI_VERSION_21); + if (status == JNI_OK) return env; + if (status != JNI_EDETACHED) return NULL; + if ((*vm)->AttachCurrentThreadAsDaemon(vm, (void **) &env, NULL) != JNI_OK) return NULL; + pthread_once(&seforim_detach_once, seforim_create_detach_key); + if (!seforim_detach_key_ok || pthread_setspecific(seforim_detach_key, vm) != 0) *detach_after = 1; + return env; +} From 063606b094f205fd279d93764c388f05eeb66901 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 09:57:31 +0300 Subject: [PATCH 03/14] refactor(search): carry the iOS bridge's calls in CBOR rather than JSON The same @Serializable requests and responses, the result encoded by its own serializer; kotlinx-serialization-cbor replaces -json in the search and the runtime's jars. Signed-off-by: Elie Gambache --- IOS_RUNTIME.md | 4 +- gradle/libs.versions.toml | 1 + search/build.gradle.kts | 4 +- .../search/index/LuceneIndexBackend.kt | 2 +- .../search/index/bridge/BridgeProtocol.kt | 21 +++++----- .../search/index/bridge/BridgedLuceneIndex.kt | 14 +++---- .../search/index/bridge/IndexBridgeServer.kt | 39 +++++++++++-------- .../index/bridge/IndexBridgeServerTest.kt | 6 +-- 8 files changed, 47 insertions(+), 44 deletions(-) diff --git a/IOS_RUNTIME.md b/IOS_RUNTIME.md index 6623c542..c9522e2f 100644 --- a/IOS_RUNTIME.md +++ b/IOS_RUNTIME.md @@ -14,7 +14,7 @@ snippets, highlights, hybrid fusion, semantic model (ONNX Runtime C API), view models... │ LuceneIndex ▼ -JvmLuceneIndex ── JSON over JNI (EmbeddedJvm.call) ──▶ IndexBridgeServer ──▶ LuceneIndexBackend (Lucene 10) +JvmLuceneIndex ── CBOR over JNI (EmbeddedJvm.call) ──▶ IndexBridgeServer ──▶ LuceneIndexBackend (Lucene 10) ``` - `search/src/commonMain/.../index/LuceneIndex.kt`: the only part of the search that runs Lucene: analysis, searches @@ -22,7 +22,7 @@ JvmLuceneIndex ── JSON over JNI (EmbeddedJvm.call) ──▶ IndexBridgeServ (`LuceneIndexBackend`, `androidJvmMain`); iOS through the bridge (`iosMain/.../bridge`), with the same implementation running in the embedded runtime. - `EmbeddedJvm` starts the runtime (`JNI_CreateJavaVM`), attaches the calling threads, calls - `IndexBridgeServer.call(byte[]): byte[]` (UTF-8 JSON of `BridgeRequest` / `BridgeResponse`). + `IndexBridgeServer.call(byte[]): byte[]` (CBOR of `BridgeRequest` / `BridgeResponse`). - Lucene runs unpatched there (Zero is HotSpot): no ART patch (`lucene-android`), `NIOFSDirectory` (no Panama mmap). Verified in the simulator on the real index (7M documents): the same facets, snippets, highlights, title suggestions diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f45cb89d..ba6d5398 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -35,6 +35,7 @@ kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutine kotlinx-coroutines-swing = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-swing", version.ref = "kotlinx-coroutines" } kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } +kotlinx-serialization-cbor = { module = "org.jetbrains.kotlinx:kotlinx-serialization-cbor", version.ref = "kotlinx-serialization" } kotlinx-serialization-protobuf = { module = "org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref = "kotlinx-serialization" } kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime" } kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 89b8274c..005a1ca1 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -128,7 +128,7 @@ kotlin { commonMain.dependencies { api(project(":core")) implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.json) + implementation(libs.kotlinx.serialization.cbor) implementation(libs.kotlinx.io.core) implementation(libs.atomicfu) implementation(libs.kermit) @@ -189,7 +189,7 @@ val iosJvmClasspath by tasks.registering(Sync::class) { from(tasks.named("jvmJar")) from( configurations.named("jvmRuntimeClasspath").map { classpath -> - classpath.filter { jar -> listOf("lucene-core", "kotlin-stdlib", "kotlinx-serialization-core", "kotlinx-serialization-json").any { jar.name.startsWith(it) } } + classpath.filter { jar -> listOf("lucene-core", "kotlin-stdlib", "kotlinx-serialization-core", "kotlinx-serialization-cbor").any { jar.name.startsWith(it) } } }, ) into(layout.buildDirectory.dir("iosJvmClasspath")) diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt index 710b5b4e..7d897441 100644 --- a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt +++ b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt @@ -459,7 +459,7 @@ class LuceneIndexBackend( } } } - // A sorted search has no score: NaN can't cross the iOS bridge's JSON + // A sorted search has no score (NaN): 0 instead, the same on every platform IndexDoc(if (sd.score.isNaN()) 0f else sd.score, longs, strings) } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt index 070e8f5b..b58f328b 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt @@ -3,14 +3,14 @@ package io.github.kdroidfilter.seforimlibrary.search.index.bridge import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest import io.github.kdroidfilter.seforimlibrary.search.index.SortField +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.cbor.Cbor /** * The calls of a [io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex] across the bridge between the iOS - * app and the Java runtime it embeds: JSON in UTF-8 both ways, each index and cursor named by a handle. + * app and the Java runtime it embeds: CBOR both ways, each index and cursor named by a handle. */ @Serializable internal sealed interface BridgeRequest { @@ -102,16 +102,15 @@ internal sealed interface BridgeRequest { ) : BridgeRequest } -/** The result of a call ([result], its type depending on the call), or the failure it raised ([error]). */ +/** + * The result of a call ([result]: CBOR of a type depending on the call, decoded by the caller; none for a call without + * one), or the failure it raised ([error]). + */ @Serializable internal class BridgeResponse( - val result: JsonElement? = null, + val result: ByteArray? = null, val error: String? = null, ) -internal val BridgeJson = - Json { - ignoreUnknownKeys = true - // Scores and vectors are plain floats; a NaN would mean a bug on the Lucene side, not data to carry - allowSpecialFloatingPointValues = false - } +@OptIn(ExperimentalSerializationApi::class) +internal val BridgeCbor = Cbor { ignoreUnknownKeys = true } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt index a3c4ce44..9f619424 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt @@ -15,11 +15,9 @@ import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.builtins.LongArraySerializer import kotlinx.serialization.builtins.nullable import kotlinx.serialization.builtins.serializer -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonNull /** - * How the calls reach the Java runtime Lucene runs in: one [BridgeRequest] in, its [BridgeResponse] out, both UTF-8 JSON. + * How the calls reach the Java runtime Lucene runs in: one [BridgeRequest] in, its [BridgeResponse] out, both CBOR. * Thread-safe: calls may overlap. */ fun interface BridgeTransport { @@ -29,8 +27,8 @@ fun interface BridgeTransport { private const val ANALYZE_CACHE_SIZE = 64 /** - * The [LuceneIndex] at [indexDir] where Lucene can't run in process: in the Java runtime an iOS app embeds, in the Java - * process an Android app before 15 starts. Lucene runs there with the same implementation as on the JVM and Android 15+ + * The [LuceneIndex] at [indexDir] where Lucene can't run in process: in the Java runtime an iOS app embeds. Lucene runs + * there with the same implementation as on the JVM and Android 15+ * (`LuceneIndexBackend`, behind `IndexBridgeServer`); this side only carries the calls, through [transport]. * * @param cacheQueries see `LuceneIndexBackend` @@ -95,10 +93,10 @@ class BridgedLuceneIndex( request: BridgeRequest, result: DeserializationStrategy?, ): T { - val bytes = transport.call(BridgeJson.encodeToString(BridgeRequest.serializer(), request).encodeToByteArray()) - val response = BridgeJson.decodeFromString(BridgeResponse.serializer(), bytes.decodeToString()) + val bytes = transport.call(BridgeCbor.encodeToByteArray(BridgeRequest.serializer(), request)) + val response = BridgeCbor.decodeFromByteArray(BridgeResponse.serializer(), bytes) response.error?.let { throw IllegalStateException("Lucene failed in the Java runtime: $it") } @Suppress("UNCHECKED_CAST") - return if (result == null) Unit as T else BridgeJson.decodeFromJsonElement(result, response.result ?: JsonNull as JsonElement) + return if (result == null) Unit as T else BridgeCbor.decodeFromByteArray(result, checkNotNull(response.result) { "No result" }) } } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt index 6d5bf9a8..bdd6d1b6 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt @@ -9,8 +9,9 @@ import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.builtins.LongArraySerializer +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.builtins.serializer -import kotlinx.serialization.json.JsonNull import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger @@ -47,60 +48,64 @@ object IndexBridgeServer { } } - /** Runs the call [request] (a [BridgeRequest] as UTF-8 JSON) and returns its [BridgeResponse] (UTF-8 JSON). */ + /** Runs the call [request] (a [BridgeRequest] in CBOR) and returns its [BridgeResponse] (CBOR). */ @JvmStatic fun call(request: ByteArray): ByteArray { val response = try { - BridgeResponse(result = run(BridgeJson.decodeFromString(BridgeRequest.serializer(), request.decodeToString()))) + BridgeResponse(result = run(BridgeCbor.decodeFromByteArray(BridgeRequest.serializer(), request))) } catch (e: Throwable) { BridgeResponse(error = e.stackTraceToString()) } - return BridgeJson.encodeToString(BridgeResponse.serializer(), response).encodeToByteArray() + return BridgeCbor.encodeToByteArray(BridgeResponse.serializer(), response) } - private fun run(request: BridgeRequest) = + private fun run(request: BridgeRequest): ByteArray? = when (request) { is BridgeRequest.Open -> { val id = ids.incrementAndGet() indexes[id] = LuceneIndexBackend(Path.of(request.indexDir), cacheQueries = request.cacheQueries) - BridgeJson.encodeToJsonElement(Int.serializer(), id) + encode(Int.serializer(), id) } is BridgeRequest.Close -> { // The cursors left open hold the index's reader: closed first cursors.filterValues { it.index == request.index }.keys.forEach { cursors.remove(it)?.close() } indexes.remove(request.index)?.close() - JsonNull + null } is BridgeRequest.Analyze -> - BridgeJson.encodeToJsonElement(ListSerializer(String.serializer()), index(request.index).analyze(request.text)) + encode(ListSerializer(String.serializer()), index(request.index).analyze(request.text)) is BridgeRequest.Search -> page(index(request.index).search(request.query, request.limit, request.fields, request.sort)) is BridgeRequest.OpenCursor -> { val id = ids.incrementAndGet() cursors[id] = OpenCursor(request.index, index(request.index).openCursor(request.query, request.fields)) - BridgeJson.encodeToJsonElement(Int.serializer(), id) + encode(Int.serializer(), id) } is BridgeRequest.CursorNext -> page(cursors[request.cursor]?.next(request.limit) ?: error("Unknown cursor ${request.cursor}")) is BridgeRequest.CloseCursor -> { cursors.remove(request.cursor)?.close() - JsonNull + null } is BridgeRequest.Facets -> - BridgeJson.encodeToJsonElement(SearchFacets.serializer(), index(request.index).facets(request.query)) + encode(SearchFacets.serializer(), index(request.index).facets(request.query)) is BridgeRequest.CollectLongs -> - BridgeJson.encodeToJsonElement(LongArraySerializer(), index(request.index).collectLongs(request.query, request.field)) + encode(LongArraySerializer(), index(request.index).collectLongs(request.query, request.field)) is BridgeRequest.Count -> - BridgeJson.encodeToJsonElement(Int.serializer(), index(request.index).count(request.query)) + encode(Int.serializer(), index(request.index).count(request.query)) is BridgeRequest.VectorEncodingOf -> - index(request.index).vectorEncoding(request.field)?.let { BridgeJson.encodeToJsonElement(VectorEncoding.serializer(), it) } - ?: JsonNull + encode(VectorEncoding.serializer().nullable, index(request.index).vectorEncoding(request.field)) is BridgeRequest.Knn -> - BridgeJson.encodeToJsonElement(ListSerializer(IndexDoc.serializer()), index(request.index).knn(request.request)) + encode(ListSerializer(IndexDoc.serializer()), index(request.index).knn(request.request)) } private fun index(id: Int): LuceneIndex = indexes[id] ?: error("Unknown index $id") - private fun page(page: IndexPage) = BridgeJson.encodeToJsonElement(IndexPage.serializer(), page) + private fun page(page: IndexPage) = encode(IndexPage.serializer(), page) + + private fun encode( + serializer: KSerializer, + value: T, + ): ByteArray = BridgeCbor.encodeToByteArray(serializer, value) } diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt index 6de7c6c6..0981f766 100644 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt @@ -35,12 +35,12 @@ class IndexBridgeServerTest { } private fun call(request: BridgeRequest): BridgeResponse = - BridgeJson.decodeFromString( + BridgeCbor.decodeFromByteArray( BridgeResponse.serializer(), - IndexBridgeServer.call(BridgeJson.encodeToString(BridgeRequest.serializer(), request).encodeToByteArray()).decodeToString(), + IndexBridgeServer.call(BridgeCbor.encodeToByteArray(BridgeRequest.serializer(), request)), ) - private fun callInt(request: BridgeRequest): Int = BridgeJson.decodeFromJsonElement(Int.serializer(), assertNotNull(call(request).result)) + private fun callInt(request: BridgeRequest): Int = BridgeCbor.decodeFromByteArray(Int.serializer(), assertNotNull(call(request).result)) @Test fun `a cursor reads its pages until closed`() { From 8545d40d09c9dfab1a6ff9ba7a1aa58ada4a5cf1 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 10:04:54 +0300 Subject: [PATCH 04/14] fix(search): send the bridge's bytes as CBOR byte strings and refuse NaN vectors - BridgeCbor writes byte arrays (results, byte vectors) as one byte string, not one integer per byte. - KnnRequest refuses a vector with a NaN or infinite component, the check the JSON encoding used to make. - BridgedLuceneIndexTest drives the bridged index through IndexBridgeServer end to end: pages with Hebrew text and scores, cursors, analysis, counts, ids and a null result. Signed-off-by: Elie Gambache --- .../search/index/LuceneIndex.kt | 7 +- .../search/index/bridge/BridgeProtocol.kt | 7 +- .../search/index/bridge/IndexBridgeServer.kt | 2 +- .../index/bridge/BridgedLuceneIndexTest.kt | 90 +++++++++++++++++++ 4 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt index 735cc39b..c74d17df 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt @@ -94,4 +94,9 @@ class KnnRequest( val byteVector: ByteArray? = null, val filter: IndexQuery? = null, val oversample: Int = 1, -) +) { + init { + // A NaN or infinite component means a broken embedding: its neighbors would be meaningless + require(floatVector?.all { it.isFinite() } != false) { "KNN vector with a NaN or infinite component" } + } +} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt index b58f328b..52d9296c 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt @@ -113,4 +113,9 @@ internal class BridgeResponse( ) @OptIn(ExperimentalSerializationApi::class) -internal val BridgeCbor = Cbor { ignoreUnknownKeys = true } +internal val BridgeCbor = + Cbor { + ignoreUnknownKeys = true + // Byte arrays (results, byte vectors) as one CBOR byte string, not an array of one integer per byte + alwaysUseByteString = true + } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt index bdd6d1b6..f6ef2bc1 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt @@ -7,9 +7,9 @@ import io.github.kdroidfilter.seforimlibrary.search.index.IndexPage import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding +import kotlinx.serialization.KSerializer import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.builtins.LongArraySerializer -import kotlinx.serialization.KSerializer import kotlinx.serialization.builtins.nullable import kotlinx.serialization.builtins.serializer import java.nio.file.Path diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt new file mode 100644 index 00000000..36f0e087 --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt @@ -0,0 +1,90 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.document.Document +import org.apache.lucene.document.Field +import org.apache.lucene.document.StoredField +import org.apache.lucene.document.StringField +import org.apache.lucene.document.TextField +import org.apache.lucene.index.IndexWriter +import org.apache.lucene.index.IndexWriterConfig +import org.apache.lucene.store.FSDirectory +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.deleteRecursively +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull + +/** The bridge's calls end to end, CBOR both ways: the bridged index answers as the in-process one. */ +class BridgedLuceneIndexTest { + private val texts = listOf("בראשית ברא אלהים", "ויאמר אלהים יהי אור", "ויהי ערב ויהי בקר") + + private val indexDir: Path = + Files.createTempDirectory("bridged_index").also { dir -> + IndexWriter(FSDirectory.open(dir), IndexWriterConfig(StandardAnalyzer())).use { writer -> + texts.forEachIndexed { i, text -> + writer.addDocument( + Document().apply { + add(StringField("type", "line", Field.Store.NO)) + add(TextField("text", text, Field.Store.YES)) + add(StoredField("line_id", 100L + i)) + }, + ) + } + } + } + + private val local = LuceneIndexBackend(indexDir) + private val bridged = BridgedLuceneIndex(BridgeTransport { IndexBridgeServer.call(it) }, indexDir.toString()) + + @OptIn(ExperimentalPathApi::class) + @AfterTest + fun cleanUp() { + bridged.close() + local.close() + indexDir.deleteRecursively() + } + + private val elohim = IndexQuery.Term("text", "אלהים") + + @Test + fun `pages carry Hebrew text, ids and scores`() { + val fields = setOf("text", "line_id") + val expected = local.search(elohim, 10, fields) + val actual = bridged.search(elohim, 10, fields) + + assertEquals(expected.totalHits, actual.totalHits) + assertEquals(expected.docs.map { it.strings["text"] }, actual.docs.map { it.strings["text"] }) + assertEquals(expected.docs.map { it.longs["line_id"] }, actual.docs.map { it.longs["line_id"] }) + assertEquals(expected.docs.map { it.score }, actual.docs.map { it.score }) + } + + @Test + fun `cursors, analysis, counts and ids cross the bridge`() { + assertEquals(local.analyze(texts[1]), bridged.analyze(texts[1])) + assertEquals(2, bridged.count(elohim)) + assertContentEquals(local.collectLongs(elohim, "line_id").sorted(), bridged.collectLongs(elohim, "line_id").sorted()) + bridged.openCursor(IndexQuery.MatchAll, setOf("line_id")).use { cursor -> + assertEquals(2, cursor.next(2).docs.size) + assertEquals(1, cursor.next(2).docs.size) + } + } + + @Test + fun `a null result crosses the bridge`() { + assertNull(bridged.vectorEncoding("embedding")) + } + + @Test + fun `a KNN vector with a NaN is refused`() { + assertFailsWith { KnnRequest("embedding", 1, emptySet(), floatVector = floatArrayOf(0f, Float.NaN)) } + } +} From 9c9a921b3d5784ed8cf963372a66738144406107 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 11:06:05 +0300 Subject: [PATCH 05/14] build(search): run the iOS search on OpenJDK 27 GA (jdk-mobile-27b35.1) The runtime now comes from kdroidFilter/openjdk-mobile's versioned releases: a stable OpenJDK 27 with openjdk/mobile's port, its JNI headers in the archive. Signed-off-by: Elie Gambache --- IOS_RUNTIME.md | 21 ++++--- search/build.gradle.kts | 123 ++++++++++++++++++++++------------------ 2 files changed, 81 insertions(+), 63 deletions(-) diff --git a/IOS_RUNTIME.md b/IOS_RUNTIME.md index c9522e2f..6822ab28 100644 --- a/IOS_RUNTIME.md +++ b/IOS_RUNTIME.md @@ -32,12 +32,15 @@ app search (Zayit's `searchkit`, hybrid with the semantic search) gives the same ## The runtime The runtimes are built by the CI of [kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) -(branch `mobile-runtime`: openjdk/mobile at the tested commit, its fixes, `mobile-runtime/build.sh`) and published as -the assets of its `mobile-runtime-*` releases; the build downloads them and checks their SHA-256 (`search`'s -`javaRuntime` tasks: `mobileRuntimeRelease`, `mobileRuntimeSha256`, in `search/build/javaRuntime/`). -A new runtime is a new release there and its tag and checksums here. +(branch `mobile-runtime`: the recipe, `openjdk-mobile.patch` and `build.sh`): a stable OpenJDK, 27 (`openjdk/jdk27u`'s GA +releases), with openjdk/mobile's port (which follows the JDK's mainline) ported to it. Every day, a new GA or recipe gets +its release, named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.1`: `jdk-27-ga`, the +JDK's build 35, the first release there). The build downloads one of them and checks its SHA-256 (`search`'s `javaRuntime` tasks: +`jdkMobileVersion`, `jdkMobileBuild`, `jdkMobileRevision`, `mobileRuntimeSha256`, extracted in +`search/build/javaRuntime/`; the iOS bindings only need its `include/`, extracted per target and release in +`search/build/jni`); moving to a newer runtime is changing those, then running the tests below. -The fixes, for Zero on iOS: +Besides openjdk/mobile's port, the patch carries two fixes for Zero on iOS: 1. `os_bsd_zero.cpp`: the W^X state (`DefaultWXWriteMode`, `os::_jit_exec_enabled`, `Thread::wx_enable_write`...) the shared code needs on Apple aarch64 is only defined for the C1/C2 port (`bsd_aarch64`): Zero gets its own, without page @@ -45,10 +48,10 @@ The fixes, for Zero on iOS: 2. `os_bsd.cpp`: `pd_commit_memory` asks for executable pages (the code cache), which iOS refuses without the JIT entitlement; Zero never executes them, so they are committed read-write. -`jdk-ios-simulator.tar.gz` and `jdk-ios-device.tar.gz` hold `static/` (`libjvm.a`, the JDK's native libraries, -`libffi.a`) and `lib/` (the runtime's `java.home`: `conf/`, `lib/`, `modules//...`). `make images` and -`make jmods` don't work for iOS (they link a dynamic libjvm): the runtime boots from an exploded image instead of a -`lib/modules` jimage. +The assets (`jdk-mobile-27-ios-simulator-aarch64-b35.1.tar.gz`, `…-ios-device-…`) hold `static/` (`libjvm.a`, the JDK's +native libraries, `libffi.a`), `include/` (`jni.h`, `jni_md.h`: the iOS bindings' headers) and `lib/` (the runtime's +`java.home`: `conf/`, `lib/`, `modules//...`). `make images` and `make jmods` don't work for iOS (they link a +dynamic libjvm): the runtime boots from an exploded image instead of a `lib/modules` jimage. ## Packaging it in the app diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 005a1ca1..4cff7e2c 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -13,73 +13,64 @@ plugins { group = "io.github.kdroidfilter.seforimlibrary" // The Java runtime Lucene runs in on iOS (OpenJDK's mobile port, see IOS_RUNTIME.md), built by the CI of -// kdroidFilter/openjdk-mobile (branch mobile-runtime) and downloaded from its release, checked -val mobileRuntimeRelease = "mobile-runtime-1" +// kdroidFilter/openjdk-mobile (branch mobile-runtime) and downloaded from its release, checked: OpenJDK 27 GA (the JDK's +// build 35) with openjdk/mobile's port, its first release there (jdk-mobile-27b35.1) +val jdkMobileVersion = "27" +val jdkMobileBuild = "35" +val jdkMobileRevision = "1" +val jdkMobileRelease = "jdk-mobile-${jdkMobileVersion}b$jdkMobileBuild.$jdkMobileRevision" val mobileRuntimeSha256 = mapOf( - "ios-simulator" to "5f392f51cece76b0bc318bced666738e3ce16fa559a7a385d345ab3955458384", - "ios-device" to "62ca2a74da57913613ae076905cb812d117afcec6472ffae3574488682fbf758", + "ios-simulator" to "0471359fb8711b9638c6f4432d170d74f8ece94994fe613c84406f690d661ded", + "ios-device" to "b5ca69777a83df475c2287ced5eb1df2fddefe2a9a235e9320ef0e488e509201", ) -// build/javaRuntime/: static/ (the libraries an app links) and lib/ (its java.home) -val javaRuntimes = +fun runtimeTaskSuffix(target: String): String = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } + +fun runtimeAsset(target: String): String = "jdk-mobile-$jdkMobileVersion-$target-aarch64-b$jdkMobileBuild.$jdkMobileRevision.tar.gz" + +fun runtimeArchive(target: String) = layout.buildDirectory.file("javaRuntime/${runtimeAsset(target)}") + +val downloadJavaRuntimes = mobileRuntimeSha256.mapValues { (target, sha256) -> - val suffix = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } - val archive = layout.buildDirectory.file("javaRuntime/jdk-$target.tar.gz") - val download = - tasks.register("downloadJavaRuntime$suffix") { - description = "Downloads the Java runtime for $target ($mobileRuntimeRelease)." - val url = "https://github.com/kdroidFilter/openjdk-mobile/releases/download/$mobileRuntimeRelease/jdk-$target.tar.gz" - inputs.property("url", url) - inputs.property("sha256", sha256) - outputs.file(archive) - doLast { - val file = archive.get().asFile.apply { parentFile.mkdirs() } - val digest = MessageDigest.getInstance("SHA-256") - DigestInputStream(URI(url).toURL().openStream(), digest).use { input -> - file.outputStream().use { input.copyTo(it) } - } - val actual = digest.digest().joinToString("") { "%02x".format(it) } - if (actual != sha256) { - file.delete() - error("Unexpected jdk-$target.tar.gz from $url (SHA-256 $actual)") - } + val asset = runtimeAsset(target) + val archive = runtimeArchive(target) + tasks.register("downloadJavaRuntime${runtimeTaskSuffix(target)}") { + description = "Downloads the Java runtime for $target ($jdkMobileRelease)." + val url = "https://github.com/kdroidFilter/openjdk-mobile/releases/download/$jdkMobileRelease/$asset" + inputs.property("url", url) + inputs.property("sha256", sha256) + outputs.file(archive) + doLast { + // Not a download for nothing: a release's checksums are set here once it is published + check(sha256.matches(Regex("[0-9a-f]{64}"))) { "No SHA-256 for $asset yet: set it in mobileRuntimeSha256 (see IOS_RUNTIME.md)" } + val file = archive.get().asFile.apply { parentFile.mkdirs() } + val digest = MessageDigest.getInstance("SHA-256") + DigestInputStream(URI(url).toURL().openStream(), digest).use { input -> + file.outputStream().use { input.copyTo(it) } + } + val actual = digest.digest().joinToString("") { "%02x".format(it) } + if (actual != sha256) { + file.delete() + error("Unexpected $asset from $url (SHA-256 $actual)") } } - tasks.register("javaRuntime$suffix") { + } + } + +// build/javaRuntime/: static/ (the libraries an app links), include/ (its JNI headers) and lib/ (its java.home) +val javaRuntimes = + downloadJavaRuntimes.mapValues { (target, download) -> + tasks.register("javaRuntime${runtimeTaskSuffix(target)}") { description = "Extracts the Java runtime for $target." dependsOn(download) - from(tarTree(archive)) + from(tarTree(runtimeArchive(target))) into(layout.buildDirectory.dir("javaRuntime/$target")) // Some of the JDK's files are read-only: the next sync couldn't replace them filePermissions { user { write = true } } } } -// The JNI headers of that runtime, for the iOS bindings (cinterop "jni"): from the sources of the same release -val jniHeaders = layout.buildDirectory.dir("jni/$mobileRuntimeRelease/include") -val downloadJniHeaders by tasks.registering { - description = "Downloads the JNI headers of the Java runtime ($mobileRuntimeRelease) for the iOS bindings." - val sources = "https://raw.githubusercontent.com/kdroidFilter/openjdk-mobile/$mobileRuntimeRelease/src/java.base" - val headers = - mapOf( - "jni.h" to ("$sources/share/native/include/jni.h" to "1e03bd83d1662ac00647f7a06c11d32333767ee537803ec1d153e89888414520"), - "jni_md.h" to ("$sources/unix/native/include/jni_md.h" to "8232a5a68d36060aaa426fbfdb4537983ecfdfc5fe07a43230005ddd1a4b986a"), - ) - val dir = jniHeaders - inputs.property("headers", headers.toString()) - outputs.dir(dir) - doLast { - headers.forEach { (name, source) -> - val (url, sha256) = source - val bytes = URI(url).toURL().readBytes() - val actual = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } - check(actual == sha256) { "Unexpected $name from $url (SHA-256 $actual)" } - dir.get().file(name).asFile.apply { parentFile.mkdirs() }.writeBytes(bytes) - } - } -} - // The runtime's static libraries an iOS binary links (force-loaded: the runtime finds its natives by name) val jdkStaticLibs = listOf( @@ -88,6 +79,23 @@ val jdkStaticLibs = ) val iosSimulatorRuntime = layout.buildDirectory.dir("javaRuntime/ios-simulator").get().asFile +// The JNI headers of each target's runtime, for its iOS bindings (cinterop "jni"): extracted alone (compiling needs +// nothing else of the runtime), in a directory per release, so a new runtime regenerates the bindings +val jniHeaders = + downloadJavaRuntimes.mapValues { (target, download) -> + tasks.register("jniHeaders${runtimeTaskSuffix(target)}") { + description = "Extracts the JNI headers of the Java runtime for $target." + dependsOn(download) + from(tarTree(runtimeArchive(target))) { include("include/**") } + into(layout.buildDirectory.dir("jni/$jdkMobileRelease/$target")) + } + } + +fun jniHeadersDir(target: String): File = layout.buildDirectory.dir("jni/$jdkMobileRelease/$target/include").get().asFile + +// The runtime target of each Kotlin/Native target +val iosRuntimeTargets = mapOf("iosArm64" to "ios-device", "iosSimulatorArm64" to "ios-simulator") + kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) @@ -116,7 +124,8 @@ kotlin { target.compilations.getByName("main").cinterops { create("jni") { definitionFile.set(project.file("src/nativeInterop/cinterop/jni.def")) - includeDirs(jniHeaders) + // The target's own runtime's + includeDirs(jniHeadersDir(iosRuntimeTargets.getValue(target.name))) } create("sqlite3") { definitionFile.set(project.file("src/nativeInterop/cinterop/sqlite3.def")) @@ -181,7 +190,13 @@ kotlin { } } -tasks.matching { it.name.startsWith("cinteropJni") }.configureEach { dependsOn(downloadJniHeaders) } +iosRuntimeTargets.forEach { (konanTarget, runtimeTarget) -> + tasks.matching { it.name == "cinteropJni${konanTarget.replaceFirstChar(Char::uppercase)}" }.configureEach { + dependsOn(jniHeaders.getValue(runtimeTarget)) + // The headers' content, not only their directory: the bindings follow them + inputs.dir(jniHeadersDir(runtimeTarget)).withPathSensitivity(PathSensitivity.RELATIVE) + } +} // The jars the embedded Java runtime runs: this module (its Lucene side and IndexBridgeServer), Lucene, Kotlin val iosJvmClasspath by tasks.registering(Sync::class) { From e4c1cada2630f5f791dd2ac59598fdf65fb337d4 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 12:55:53 +0300 Subject: [PATCH 06/14] feat(search): Lucene on a real JVM on Android too, from Android 10 ART can't keep up with the Java versions Lucene requires (21 for Lucene 10, 25 for Lucene 11, apache/lucene#14229), and already needed two of its classes patched. Android now runs Lucene as iOS does, behind the same bridge, in a Java process of the app's own (the Android runtime of the same jdk-mobile release: OpenJDK 27, server VM). - JavaProcessTransport (androidJvmMain) and IndexBridgeServer.main: the calls on the process's standard input and output; the calls sent to a process that ends fail without failing those of the next one. - LuceneIndexBackend moves to jvmMain; lucene-android and the ART device test go. - minSdk 29; the Android runtime is downloaded with the iOS ones; iosJvmClasspath becomes mobileJvmClasspath. - JavaProcessTransportTest runs the transport on the desktop, a java standing for the launcher. Signed-off-by: Elie Gambache --- IOS_RUNTIME.md | 31 ++- gradle/libs.versions.toml | 4 - lucene-android/build.gradle.kts | 77 ----- .../vectorization/VectorizationProvider.java | 262 ------------------ .../apache/lucene/util/HotspotVMOptions.java | 101 ------- search/build.gradle.kts | 40 +-- .../search/AndroidRealIndexParityTest.kt | 37 --- .../index/bridge/JavaProcessTransport.kt | 133 +++++++++ .../search/index/bridge/EmbeddedJvm.kt | 2 +- .../search/index/LuceneIndexBackend.kt | 7 +- .../search/index/bridge/IndexBridgeServer.kt | 51 +++- .../index/bridge/JavaProcessTransportTest.kt | 74 +++++ settings.gradle.kts | 1 - 13 files changed, 295 insertions(+), 525 deletions(-) delete mode 100644 lucene-android/build.gradle.kts delete mode 100644 lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java delete mode 100644 lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java delete mode 100644 search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt create mode 100644 search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt rename search/src/{androidJvmMain => jvmMain}/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt (98%) create mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt diff --git a/IOS_RUNTIME.md b/IOS_RUNTIME.md index 6822ab28..fdba210f 100644 --- a/IOS_RUNTIME.md +++ b/IOS_RUNTIME.md @@ -18,12 +18,13 @@ JvmLuceneIndex ── CBOR over JNI (EmbeddedJvm.call) ──▶ IndexBridgeServ ``` - `search/src/commonMain/.../index/LuceneIndex.kt`: the only part of the search that runs Lucene: analysis, searches - (`IndexQuery`, plain data), cursors, facets, KNN. The JVM and Android implement it in process - (`LuceneIndexBackend`, `androidJvmMain`); iOS through the bridge (`iosMain/.../bridge`), with the same implementation - running in the embedded runtime. + (`IndexQuery`, plain data), cursors, facets, KNN. Lucene runs on a JVM only (ART can't keep up with the Java versions + Lucene requires: Java 21 for 10, 25 for 11): the desktop implements it in process (`LuceneIndexBackend`), the mobile + apps through the bridge (`BridgedLuceneIndex`), with the same implementation running in the Java runtime they embed: + in the app on iOS (JNI), in a process of the app's own on Android. - `EmbeddedJvm` starts the runtime (`JNI_CreateJavaVM`), attaches the calling threads, calls `IndexBridgeServer.call(byte[]): byte[]` (CBOR of `BridgeRequest` / `BridgeResponse`). -- Lucene runs unpatched there (Zero is HotSpot): no ART patch (`lucene-android`), `NIOFSDirectory` (no Panama mmap). +- Lucene runs unpatched there (Zero is HotSpot), with `NIOFSDirectory` (no Panama mmap on Zero). Verified in the simulator on the real index (7M documents): the same facets, snippets, highlights, title suggestions and find-in-book candidates as the desktop, and the same hits but for Lucene's score noise (see `SearchDump`); the whole @@ -73,7 +74,7 @@ holds: ``` App.app/lib/{conf,lib,modules}/... (the runtime's lib/: java.base, java.logging, java.management, jdk.unsupported) -App.app/lib/app/*.jar (./gradlew :search:iosJvmClasspath: search-jvm, lucene-core, kotlin-stdlib, kotlinx-serialization) +App.app/lib/app/*.jar (./gradlew :search:mobileJvmClasspath: search-jvm, lucene-core, kotlin-stdlib, kotlinx-serialization) ``` About 50 MB uncompressed. Zayit's `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/` and `lib/` @@ -93,8 +94,18 @@ hashes), so the comparison allows score differences: facets, snippets and highli ## Android -Android 15+ (API 35) only: ART runs Lucene in process, with two classes patched (`lucene-android`: a guard in -`VectorizationProvider.lookup` and in `HotspotVMOptions`, checked against the upstream classes' SHA-256 so a Lucene bump -can't ship a stale patch). Before 15, ART lacks the Java 21 APIs Lucene 10 needs, and a Java runtime beside the app -would cost those older devices too much: they aren't supported. `:search:androidDeviceTest` runs the dump on a device -or an emulator. +Android 10+ (API 29: HotSpot's aarch64 code needs native ELF TLS). ART and HotSpot can't share a process (both define +`JNI_CreateJavaVM`, both handle `SIGSEGV`), so Lucene runs in a Java process of the app's own, the server VM (with its +JIT, which Android allows) of the same release's Android runtime (`jdk-mobile-27-android-aarch64-b35.1.tar.gz`), behind +the same bridge as iOS, its calls carried on the process's standard input and output (`JavaProcessTransport`, +`IndexBridgeServer.main`) rather than JNI. + +- The runtime holds the launcher, `jniLibs/arm64-v8a/libjvmlauncher.so`: an executable with the VM and the JDK's + natives in, packaged as a library (Android only executes native code from the APK; the app sets + `packaging.jniLibs.useLegacyPackaging = true` so it is installed as a file), and `lib/`, its `java.home`. The + launcher runs `IndexBridgeServer.main`; everything the VM prints goes to standard error, standard output carries the + answers. +- The process starts on the first call and again after it ends (Android may kill an app's child processes in the + background): the calls sent to a process that ends fail, the next ones start another. +- `JavaProcessTransportTest` runs the same transport on the desktop, a `java` standing for the launcher. On a device, + Zayit's `searchkit-check` app runs the whole search. diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ba6d5398..5e5b52c2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,8 +15,6 @@ kermit = "2.1.0" kotlinx-io = "0.9.1" cachemap = "0.3.0" atomicfu = "0.33.0" -androidx-test-runner = "1.7.0" -androidx-test-junit = "1.3.0" ksoup = "0.2.6" sqlDelight = "2.3.2" compose = "1.10.2" @@ -42,8 +40,6 @@ kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } cachemap = { module = "io.github.charlietap:cachemap", version.ref = "cachemap" } atomicfu = { module = "org.jetbrains.kotlinx:atomicfu", version.ref = "atomicfu" } -androidx-test-runner = { module = "androidx.test:runner", version.ref = "androidx-test-runner" } -androidx-test-junit = { module = "androidx.test.ext:junit", version.ref = "androidx-test-junit" } ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.ref = "lucene" } lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "lucene" } diff --git a/lucene-android/build.gradle.kts b/lucene-android/build.gradle.kts deleted file mode 100644 index a58bab66..00000000 --- a/lucene-android/build.gradle.kts +++ /dev/null @@ -1,77 +0,0 @@ -import java.security.MessageDigest -import java.util.zip.ZipFile -import javax.inject.Inject - -// lucene-core for Android (ART, API 35+): the upstream jar with two classes patched, as Lucene 10 assumes a HotSpot JVM -// in them (Runtime.version(), modules). The patched sources are the upstream ones plus a guard, marked "SeforimLibrary". -plugins { - `java-library` -} - -group = "io.github.kdroidfilter.seforimlibrary" - -java { - toolchain.languageVersion.set(JavaLanguageVersion.of(libs.versions.jvmToolchain.get().toInt())) -} - -tasks.withType().configureEach { - // Lucene's own level - options.release.set(21) -} - -val upstream: Configuration by configurations.creating { - isTransitive = false -} - -dependencies { - compileOnly(libs.lucene.core) - upstream(libs.lucene.core) -} - -// The upstream classes the patch replaces, by SHA-256: a Lucene bump that changes them fails the build, so the patch -// is diffed against the new sources and redone instead of shipping stale code -val replaced = - mapOf( - "org/apache/lucene/internal/vectorization/VectorizationProvider.class" to "317cbaed36506e22e6cfaec16eb2e1612465b55bb36d06125fd5b04e72d9e21c", - "org/apache/lucene/internal/vectorization/VectorizationProvider\$Holder.class" to "f416745bad90bfde50eaee0e21ec6ec3763e762114914700a8ad0915f54ff023", - "org/apache/lucene/util/HotspotVMOptions.class" to "beb595fb64a8cc73b602976d77ac2f84b2935b113305117074440ac706a301a3", - ) - -val checkUpstream by tasks.registering { - val jars: FileCollection = files(upstream) - val expected = replaced - inputs.files(jars) - doLast { - ZipFile(jars.singleFile).use { zip -> - for ((entry, sha) in expected) { - val bytes = zip.getInputStream(zip.getEntry(entry) ?: error("$entry is gone from ${jars.singleFile.name}")).readBytes() - val actual = MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } - check(actual == sha) { "$entry changed in ${jars.singleFile.name}: redo the patch of lucene-android on its new sources" } - } - } - } -} - -// ArchiveOperations, not the script's zipTree: the configuration cache can't keep references to the script -interface ArchiveAccess { - @get:Inject - val archives: ArchiveOperations -} -val archiveAccess = objects.newInstance() - -tasks.jar { - dependsOn(checkUpstream) - // The patched classes (compiled here) replace the upstream ones; everything else, services included, is upstream - val access = archiveAccess - from(upstream.elements.map { jars -> jars.map { access.archives.zipTree(it.asFile) } }) { - exclude(replaced.keys) - exclude("META-INF/MANIFEST.MF") - } - duplicatesStrategy = DuplicatesStrategy.FAIL -} - -// Consumers get the merged jar only: the secondary variants (classes, resources directories) would hand them the two -// patched classes without the rest of Lucene -listOf("apiElements", "runtimeElements").forEach { name -> - configurations.named(name) { outgoing.variants.clear() } -} diff --git a/lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java deleted file mode 100644 index 8d5f7dbb..00000000 --- a/lucene-android/src/main/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.lucene.internal.vectorization; - -import java.io.IOException; -import java.lang.StackWalker.StackFrame; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Set; -import java.util.function.Predicate; -import java.util.logging.Logger; -import java.util.stream.Stream; -import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.Constants; -import org.apache.lucene.util.VectorUtil; - -/** - * A provider of vectorization implementations. Depending on the Java version and availability of - * vectorization modules in the Java runtime this class provides optimized implementations (using - * SIMD) of several algorithms used throughout Apache Lucene. - * - *

Expert: set the {@value #UPPER_JAVA_FEATURE_VERSION_SYSPROP} system property to increase the - * set of Java versions this class will provide optimized implementations for. - * - * @lucene.internal - */ -public abstract class VectorizationProvider { - - static final OptionalInt TESTS_VECTOR_SIZE; - static final boolean TESTS_FORCE_INTEGER_VECTORS; - static final int UPPER_JAVA_FEATURE_VERSION = getUpperJavaFeatureVersion(); - - static { - var vs = OptionalInt.empty(); - try { - vs = - Stream.ofNullable(System.getProperty("tests.vectorsize")) - .filter(Predicate.not(Set.of("", "default")::contains)) - .mapToInt(Integer::parseInt) - .findAny(); - } catch ( - @SuppressWarnings("unused") - SecurityException se) { - // ignored - } - TESTS_VECTOR_SIZE = vs; - - boolean enforce = false; - try { - enforce = Boolean.getBoolean("tests.forceintegervectors"); - } catch ( - @SuppressWarnings("unused") - SecurityException se) { - // ignored - } - TESTS_FORCE_INTEGER_VECTORS = enforce; - } - - private static final String UPPER_JAVA_FEATURE_VERSION_SYSPROP = - "org.apache.lucene.vectorization.upperJavaFeatureVersion"; - private static final int DEFAULT_UPPER_JAVA_FEATURE_VERSION = 26; - - private static int getUpperJavaFeatureVersion() { - int runtimeVersion = DEFAULT_UPPER_JAVA_FEATURE_VERSION; - try { - String str = System.getProperty(UPPER_JAVA_FEATURE_VERSION_SYSPROP); - if (str != null) { - runtimeVersion = Math.max(Integer.parseInt(str), runtimeVersion); - } - } catch (@SuppressWarnings("unused") NumberFormatException | SecurityException ignored) { - Logger.getLogger(VectorizationProvider.class.getName()) - .warning( - "Cannot read sysprop " - + UPPER_JAVA_FEATURE_VERSION_SYSPROP - + ", so the default value will be used."); - } - return runtimeVersion; - } - - /** - * Returns the default instance of the provider matching vectorization possibilities of actual - * runtime. - * - * @throws IllegalCallerException if the singleton getter is not called from known Lucene classes. - */ - public static VectorizationProvider getInstance() { - ensureCaller(); - return Objects.requireNonNull( - Holder.INSTANCE, "call to getInstance() from subclass of VectorizationProvider"); - } - - VectorizationProvider() { - // no instance/subclass except from this package - } - - /** - * Returns a singleton (stateless) {@link VectorUtilSupport} to support SIMD usage in {@link - * VectorUtil}. - */ - public abstract VectorUtilSupport getVectorUtilSupport(); - - /** Returns a FlatVectorsScorer that supports the Lucene99 format. */ - public abstract FlatVectorsScorer getLucene99FlatVectorsScorer(); - - /** Returns a FlatVectorsScorer that supports the Lucene99 format. */ - public abstract FlatVectorsScorer getLucene99ScalarQuantizedVectorsScorer(); - - /** Create a new {@link PostingDecodingUtil} for the given {@link IndexInput}. */ - public abstract PostingDecodingUtil newPostingDecodingUtil(IndexInput input) throws IOException; - - /** Returns a {@link DocValuesRangeSupport} for SIMD-accelerated range evaluation. */ - public DocValuesRangeSupport getDocValuesRangeSupport() { - return DefaultDocValuesRangeSupport.INSTANCE; - } - - /** - * Returns a {@link DocValuesBulkDecodeSupport} instance for bulk numeric value decode. The - * returned instance uses SIMD when available (Panama Vector API), falling back to a scalar loop - * otherwise. - */ - public abstract DocValuesBulkDecodeSupport getDocValuesBulkDecodeSupport(); - - // *** Lookup mechanism: *** - - private static final Logger LOG = Logger.getLogger(VectorizationProvider.class.getName()); - - // visible for tests - static VectorizationProvider lookup(boolean testMode) { - // SeforimLibrary (Android): ART has no Runtime.version() nor Panama; the scalar provider is the only one there - if (!Constants.IS_HOTSPOT_VM) { - return new DefaultVectorizationProvider(); - } - final int runtimeVersion = Runtime.version().feature(); - assert runtimeVersion >= 21; - if (runtimeVersion <= UPPER_JAVA_FEATURE_VERSION) { - // only use vector module with Hotspot VM - if (!Constants.IS_HOTSPOT_VM) { - LOG.warning( - "Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled."); - return new DefaultVectorizationProvider(); - } - // don't use vector module with JVMCI (it does not work) - if (Constants.IS_JVMCI_VM) { - LOG.warning( - "Java runtime is using JVMCI Compiler; Java vector incubator API can't be enabled."); - return new DefaultVectorizationProvider(); - } - // is the incubator module present and readable (JVM providers may to exclude them or it is - // build with jlink) - final var vectorMod = lookupVectorModule(); - if (vectorMod.isEmpty()) { - LOG.warning( - "Java vector incubator module is not readable. For optimal vector performance, pass '--add-modules jdk.incubator.vector' to enable Vector API."); - return new DefaultVectorizationProvider(); - } - vectorMod.ifPresent(VectorizationProvider.class.getModule()::addReads); - // check for testMode and otherwise fallback to default if slowness could happen - if (!testMode) { - if (TESTS_VECTOR_SIZE.isPresent() || TESTS_FORCE_INTEGER_VECTORS) { - LOG.warning( - "Vector bitsize and/or integer vectors enforcement; using default vectorization provider outside of testMode"); - return new DefaultVectorizationProvider(); - } - if (Constants.IS_CLIENT_VM) { - LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled"); - return new DefaultVectorizationProvider(); - } - } - try { - // we use method handles with lookup, so we do not need to deal with setAccessible as we - // have private access through the lookup: - final var lookup = MethodHandles.lookup(); - final var cls = - lookup.findClass( - "org.apache.lucene.internal.vectorization.PanamaVectorizationProvider"); - final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class)); - try { - return (VectorizationProvider) constr.invoke(); - } catch (UnsupportedOperationException uoe) { - // not supported because preferred vector size too small or similar - LOG.warning("Java vector incubator API was not enabled. " + uoe.getMessage()); - return new DefaultVectorizationProvider(); - } catch (RuntimeException | Error e) { - throw e; - } catch (Throwable th) { - throw new AssertionError(th); - } - } catch (NoSuchMethodException | IllegalAccessException e) { - throw new LinkageError( - "PanamaVectorizationProvider is missing correctly typed constructor", e); - } catch (ClassNotFoundException cnfe) { - throw new LinkageError("PanamaVectorizationProvider is missing in Lucene JAR file", cnfe); - } - } else { - LOG.warning( - "You are running with unsupported Java " - + runtimeVersion - + ". To make full use of the Vector API, please update Apache Lucene."); - } - return new DefaultVectorizationProvider(); - } - - /** - * Looks up the vector module from Lucene's {@link ModuleLayer} or the root layer (if unnamed). - */ - private static Optional lookupVectorModule() { - return Optional.ofNullable(VectorizationProvider.class.getModule().getLayer()) - .orElse(ModuleLayer.boot()) - .findModule("jdk.incubator.vector"); - } - - // add all possible callers here as FQCN: - private static final Set VALID_CALLERS = - Set.of( - "org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil", - "org.apache.lucene.util.VectorUtil", - "org.apache.lucene.codecs.lucene104.Lucene104PostingsReader", - "org.apache.lucene.codecs.lucene104.PostingIndexInput", - "org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducer", - "org.apache.lucene.tests.util.TestSysoutsLimits"); - - private static void ensureCaller() { - final boolean validCaller = - StackWalker.getInstance() - .walk( - s -> - s.skip(2) - .limit(1) - .map(StackFrame::getClassName) - .anyMatch(VALID_CALLERS::contains)); - if (!validCaller) { - throw new IllegalCallerException( - "VectorizationProvider is internal and can only be used by known Lucene classes."); - } - } - - /** This static holder class prevents classloading deadlock. */ - private static final class Holder { - private Holder() {} - - static final VectorizationProvider INSTANCE = lookup(false); - } -} diff --git a/lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java b/lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java deleted file mode 100644 index 7eddee21..00000000 --- a/lucene-android/src/main/java/org/apache/lucene/util/HotspotVMOptions.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.util; - -import java.lang.reflect.Method; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Function; -import java.util.logging.Logger; - -/** Accessor to get Hotspot VM Options (if available). */ -final class HotspotVMOptions { - private HotspotVMOptions() {} // can't construct - - /** True iff the Java VM is based on Hotspot and has the Hotspot MX bean readable by Lucene */ - public static final boolean IS_HOTSPOT_VM; - - /** - * Returns an optional with the value of a Hotspot VM option. If the VM option does not exist or - * is not readable, returns an empty optional. - */ - public static Optional get(String name) { - return ACCESSOR.apply(Objects.requireNonNull(name, "name")); - } - - private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory"; - private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean"; - private static final Function> ACCESSOR; - - static { - boolean isHotspot = false; - Function> accessor = name -> Optional.empty(); - try { - final Class beanClazz = Class.forName(HOTSPOT_BEAN_CLASS); - // we use reflection for this, because the management factory is not part - // of java.base module: - final Object hotSpotBean = - Class.forName(MANAGEMENT_FACTORY_CLASS) - .getMethod("getPlatformMXBean", Class.class) - .invoke(null, beanClazz); - if (hotSpotBean != null) { - final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class); - final Method getValueMethod = getVMOptionMethod.getReturnType().getMethod("getValue"); - isHotspot = true; - accessor = - name -> { - try { - final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, name); - return Optional.of(getValueMethod.invoke(vmOption).toString()); - } catch (@SuppressWarnings("unused") - ReflectiveOperationException - | RuntimeException e) { - return Optional.empty(); - } - }; - } - } catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) { - isHotspot = false; - final Logger log = Logger.getLogger(HotspotVMOptions.class.getName()); - // SeforimLibrary (Android): ART has no modules (Class.getModule, ModuleLayer) - if (jdkManagementUnreadable()) { - log.warning( - "Lucene cannot access JVM internals to optimize algorithms or calculate object sizes, unless the 'jdk.management' Java module " - + "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor]."); - } else { - log.warning( - "Lucene cannot optimize algorithms or calculate object sizes for JVMs that are not based on Hotspot or a compatible implementation."); - } - } - IS_HOTSPOT_VM = isHotspot; - ACCESSOR = accessor; - } - - // Whether a module layer holds this class without the jdk.management module readable; false where there are no - // modules (classpath, ART) - private static boolean jdkManagementUnreadable() { - try { - final Module module = HotspotVMOptions.class.getModule(); - final ModuleLayer layer = module.getLayer(); - // classpath / unnamed module has no layer, so we need to check: - return layer != null - && layer.findModule("jdk.management").map(module::canRead).orElse(false) == false; - } catch (@SuppressWarnings("unused") LinkageError e) { - return false; - } - } -} diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 4cff7e2c..bdd84e6e 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -23,6 +23,7 @@ val mobileRuntimeSha256 = mapOf( "ios-simulator" to "0471359fb8711b9638c6f4432d170d74f8ece94994fe613c84406f690d661ded", "ios-device" to "b5ca69777a83df475c2287ced5eb1df2fddefe2a9a235e9320ef0e488e509201", + "android" to "290039f7149558cb7eb7d2a317b24002928d0a65296b2582adf65c11d9fbc5b8", ) fun runtimeTaskSuffix(target: String): String = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } @@ -58,7 +59,8 @@ val downloadJavaRuntimes = } } -// build/javaRuntime/: static/ (the libraries an app links), include/ (its JNI headers) and lib/ (its java.home) +// build/javaRuntime/: include/ (its JNI headers), lib/ (its java.home) and, on iOS, static/ (the libraries an app +// links), on Android jniLibs/ (the launcher of the Java process) val javaRuntimes = downloadJavaRuntimes.mapValues { (target, download) -> tasks.register("javaRuntime${runtimeTaskSuffix(target)}") { @@ -103,12 +105,8 @@ kotlin { android { namespace = "io.github.kdroidfilter.seforimlibrary.search" compileSdk = 35 - // Lucene 10 needs the Java 21 APIs ART only has from Android 15 - minSdk = 35 - // The search on ART, on a device or an emulator (the parity dump on a real index, see SearchDump) - withDeviceTestBuilder { sourceSetTreeName = "test" }.configure { - instrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - } + // Lucene runs in a Java process of the app's own (JavaProcessTransport), from Android 10 + minSdk = 29 } // The iOS tests link the runtime and run the search end to end in the simulator iosSimulatorArm64 { @@ -143,11 +141,11 @@ kotlin { implementation(libs.kermit) } - // Lucene runs in process on the JVM and Android (ART, API 35+): the same code serves both + // Lucene runs on a JVM only: in process on the desktop, in the Java runtime the mobile apps embed (a process of + // the app's own on Android, in the app on iOS), behind the same bridge. The JVM and Android share the rest val androidJvmMain by creating { dependsOn(commonMain.get()) dependencies { - compileOnly(libs.lucene.core) implementation(libs.jsoup) } } @@ -159,13 +157,7 @@ kotlin { implementation(libs.sqlite.jdbc) } } - androidMain { - dependsOn(androidJvmMain) - dependencies { - // lucene-core patched for ART - implementation(project(":lucene-android")) - } - } + androidMain { dependsOn(androidJvmMain) } // The manual source set above turns off the default hierarchy: iOS shares its own one by hand val iosMain by creating { @@ -177,12 +169,6 @@ kotlin { iosArm64Main { dependsOn(iosMain) } iosSimulatorArm64Main { dependsOn(iosMain) } - getByName("androidDeviceTest").dependencies { - implementation(kotlin("test")) - implementation(libs.androidx.test.runner) - implementation(libs.androidx.test.junit) - } - jvmTest.dependencies { implementation(kotlin("test")) implementation(libs.kotlinx.coroutines.test) @@ -199,15 +185,15 @@ iosRuntimeTargets.forEach { (konanTarget, runtimeTarget) -> } // The jars the embedded Java runtime runs: this module (its Lucene side and IndexBridgeServer), Lucene, Kotlin -val iosJvmClasspath by tasks.registering(Sync::class) { - description = "Collects the jars the Java runtime of an iOS app runs for the search." +val mobileJvmClasspath by tasks.registering(Sync::class) { + description = "Collects the jars the Java runtime of a mobile app runs for the search (iOS, Android)." from(tasks.named("jvmJar")) from( configurations.named("jvmRuntimeClasspath").map { classpath -> classpath.filter { jar -> listOf("lucene-core", "kotlin-stdlib", "kotlinx-serialization-core", "kotlinx-serialization-cbor").any { jar.name.startsWith(it) } } }, ) - into(layout.buildDirectory.dir("iosJvmClasspath")) + into(layout.buildDirectory.dir("mobileJvmClasspath")) } run { @@ -217,7 +203,7 @@ run { val iosTestRuntime by tasks.registering(Sync::class) { dependsOn(javaRuntimes.getValue("ios-simulator")) from(iosSimulatorRuntime.resolve("lib")) - from(iosJvmClasspath) { into("app") } + from(mobileJvmClasspath) { into("app") } into(testBinaryDir.map { it.dir("lib") }) filePermissions { user { write = true } } } @@ -225,7 +211,7 @@ run { tasks.withType().configureEach { dependsOn(iosTestRuntime) device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) - val jars = iosJvmClasspath.map { sync -> sync.destinationDir.listFiles().orEmpty().map { testBinaryDir.get().dir("lib/app/${it.name}").asFile.path } } + val jars = mobileJvmClasspath.map { sync -> sync.destinationDir.listFiles().orEmpty().map { testBinaryDir.get().dir("lib/app/${it.name}").asFile.path } } val indexDir = providers.gradleProperty("indexDir") val parityOut = layout.buildDirectory.file("parity/ios.txt") val luceneQueryCache = providers.gradleProperty("luceneQueryCache") diff --git a/search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt b/search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt deleted file mode 100644 index 38adfa24..00000000 --- a/search/src/androidDeviceTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/AndroidRealIndexParityTest.kt +++ /dev/null @@ -1,37 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search - -import android.os.SystemClock -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend -import kotlinx.coroutines.runBlocking -import org.junit.Test -import org.junit.runner.RunWith -import java.io.File - -/** - * The search on ART (Android 15+), with Lucene in process: dumps what it answers on a real index, like - * RealIndexParityTest on the JVM. The index and lexical.db go in the test app's files dir, under seforim/ (adb root, push, - * chown to the app); the dump comes back as parity-android.txt there. Skipped without an index. - */ -@RunWith(AndroidJUnit4::class) -class AndroidRealIndexParityTest { - @Test - fun dumpRealIndex() { - val files = InstrumentationRegistry.getInstrumentation().targetContext.filesDir - val indexDir = File(files, "seforim/seforim.db.lucene") - println("Parity index: $indexDir exists=${indexDir.exists()} files=${indexDir.list()?.size}") - if (!indexDir.isDirectory) return - val arguments = InstrumentationRegistry.getArguments() - arguments.getString("luceneQueryCache")?.let { System.setProperty("seforim.lucene.queryCache", it) } - val engine = - LexicalSearchEngine( - index = LuceneIndexBackend(indexDir.toPath()), - dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, File(indexDir.parentFile, "lexical.db").path) }, - ) - val start = SystemClock.elapsedRealtime() - val dump = engine.use { runBlocking { SearchDump.dump(it) } } - println("Parity dump in ${SystemClock.elapsedRealtime() - start} ms") - File(files, "parity-android.txt").writeText(dump) - } -} diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt new file mode 100644 index 00000000..09a42478 --- /dev/null +++ b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt @@ -0,0 +1,133 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import co.touchlab.kermit.Logger +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.DataInputStream +import java.io.DataOutputStream +import java.io.File +import java.io.IOException +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutionException +import java.util.concurrent.atomic.AtomicInteger + +private const val SERVER_CLASS = "io.github.kdroidfilter.seforimlibrary.search.index.bridge.IndexBridgeServer" +private const val STDERR_LINES_KEPT = 40 + +// Far above any answer (a page of hits, facets), far below a misread frame +private const val MAX_ANSWER_BYTES = 64 * 1024 * 1024 + +/** + * The calls to a Java process of the app's own, for Android, whose ART can't run Lucene: OpenJDK's mobile port (built + * for Android, see IOS_RUNTIME.md) runs `IndexBridgeServer.main`, which serves the calls on its standard input and + * output. A separate process, as ART and HotSpot can't share one. It starts on the first call and again after it dies + * (Android may kill an app's child processes in the background). + * + * Android only executes the native code an APK holds: [launcher] is the runtime's launcher, an executable packaged as a + * library (`libjvmlauncher.so`, from the app's native library dir), the VM linked in; the JDK's classes and data are in + * the app's files ([javaHome], an exploded image: `modules/`, `conf/`, `lib/`). + * + * @param classPath the jars of the search's Java side (`mobileJvmClasspath`, the same as iOS) + * @param options the Java process's options (`-Xmx...`) + */ +class JavaProcessTransport( + private val launcher: File, + private val javaHome: File, + private val classPath: List, + private val options: List = listOf("-Xmx512m"), +) : BridgeTransport, + AutoCloseable { + private val lock = Any() + private var running: Running? = null + private val ids = AtomicInteger() + private val logger = Logger.withTag("SeforimJava") + + // A started process with the calls sent to it: when it ends, they fail, and those of the next process don't + private class Running( + val process: Process, + ) { + val input = DataOutputStream(BufferedOutputStream(process.outputStream)) + val pending = ConcurrentHashMap>() + val stderr = ArrayDeque() + } + + override fun call(request: ByteArray): ByteArray { + val id = ids.incrementAndGet() + val answer = CompletableFuture() + synchronized(lock) { + val run = started() + run.pending[id] = answer + try { + run.input.writeInt(id) + run.input.writeInt(request.size) + run.input.write(request) + run.input.flush() + } catch (e: IOException) { + // The process is gone: its reader fails the calls sent to it, this one included + run.pending.remove(id) + throw IllegalStateException("The Java process can't be reached", e) + } + } + return try { + answer.get() + } catch (e: ExecutionException) { + throw e.cause ?: e + } + } + + // The running process, starting it when there is none + private fun started(): Running { + running?.let { return it } + val command = listOf(launcher.path, javaHome.path, classPath.joinToString(":") { it.path }, SERVER_CLASS) + options + val run = Running(ProcessBuilder(command).start()) + running = run + Thread({ readAnswers(run) }, "seforim-java-answers").apply { isDaemon = true }.start() + Thread({ readErrors(run) }, "seforim-java-stderr").apply { isDaemon = true }.start() + return run + } + + private fun readAnswers(run: Running) { + val answers = DataInputStream(BufferedInputStream(run.process.inputStream)) + try { + while (true) { + val id = answers.readInt() + val size = answers.readInt() + // Anything but a frame here means the process wrote something else on its output: stop reading it + check(size in 0..MAX_ANSWER_BYTES) { "Not an answer from the Java process (size $size)" } + val bytes = ByteArray(size).also { answers.readFully(it) } + run.pending.remove(id)?.complete(bytes) + } + } catch (failure: Throwable) { + logger.w { "Java process answers: $failure" } + run.process.destroy() + // The process ended: the calls sent to it fail, the next one starts another + val exit = runCatching { run.process.waitFor() }.getOrNull() + val reason = IllegalStateException("The Java process ended (exit $exit): ${synchronized(run.stderr) { run.stderr.joinToString("\n") }}") + synchronized(lock) { + if (running === run) running = null + } + run.pending.keys.toList().forEach { id -> run.pending.remove(id)?.completeExceptionally(reason) } + } + } + + private fun readErrors(run: Running) { + run.process.errorStream.bufferedReader().forEachLine { line -> + logger.w { line } + synchronized(run.stderr) { + run.stderr.addLast(line) + if (run.stderr.size > STDERR_LINES_KEPT) run.stderr.removeFirst() + } + } + } + + override fun close() { + synchronized(lock) { + running?.let { run -> + runCatching { run.input.close() } + run.process.destroy() + } + running = null + } + } +} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt index 12bbaaca..fb2a3ea9 100644 --- a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt +++ b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt @@ -42,7 +42,7 @@ import kotlinx.cinterop.value * linked statically into the app. Lucene runs there; the app calls it through [call]. * * The runtime reads its classes from `/lib` (its `java.home`, set by the VM itself); [classPath] - * names the jars of the search (see the `iosJvmClasspath` task of SeforimLibrary's `search` module). + * names the jars of the search (see the `mobileJvmClasspath` task of SeforimLibrary's `search` module). */ class EmbeddedJvm private constructor( private val vm: CPointer, diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt similarity index 98% rename from search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt rename to search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt index 7d897441..c709ee42 100644 --- a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt @@ -61,8 +61,8 @@ private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" private const val MIN_DOCS_PER_SLICE = 100_000 /** - * The [LuceneIndex] of a Lucene directory, read with Lucene in this process: on the JVM, on Android (ART, API 35+), - * and inside the Java runtime the iOS app embeds (see `IndexBridgeServer`). + * The [LuceneIndex] of a Lucene directory, read with Lucene in this process: on the JVM, and inside the Java runtimes + * the mobile apps embed (see `IndexBridgeServer`). * * @param indexDir the index directory * @param cacheQueries whether to cache any query seen often (small indexes: the lookup index); else only the filters @@ -516,7 +516,7 @@ class LuceneIndexBackend( companion object { /** * The directory of [path]: memory-mapped where Lucene can (HotSpot), read through NIO where its Panama-based - * mapping isn't available (GraalVM native image, Android, the iOS Java runtime). + * mapping isn't available (GraalVM native image, the iOS Java runtime). */ fun openDirectory(path: Path): Directory = if (mmapAvailable) FSDirectory.open(path) else NIOFSDirectory(path) @@ -524,7 +524,6 @@ class LuceneIndexBackend( private val mmapAvailable: Boolean by lazy { System.getProperty("org.graalvm.nativeimage.imagecode") == null && - System.getProperty("java.vm.vendor")?.contains("Android") != true && System.getProperty("os.name") != "Darwin" && System.getProperty("java.vm.name")?.contains("Zero") != true } diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt index f6ef2bc1..bae786a5 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt @@ -12,16 +12,65 @@ import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.builtins.LongArraySerializer import kotlinx.serialization.builtins.nullable import kotlinx.serialization.builtins.serializer +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.DataInputStream +import java.io.DataOutputStream +import java.io.EOFException +import java.io.FileDescriptor +import java.io.FileOutputStream +import java.io.PrintStream import java.nio.file.Path import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger /** * The Java side of the bridge (see `BridgedLuceneIndex`): runs the calls of an app's [LuceneIndex]es on * [LuceneIndexBackend]s, where Lucene can't run in the app itself: the iOS app calls [call] through JNI, from any - * thread, in the Java runtime it embeds. + * thread, in the Java runtime it embeds; an Android app runs [main] in a Java process of its own. */ object IndexBridgeServer { + /** + * Serves the calls of standard input on standard output, until the input closes: each call is a frame (its id, its + * length, the request), each answer a frame with the same id; calls run concurrently, answered as they complete. + * Anything else printed goes to standard error, which keeps the output for the frames alone. + */ + @JvmStatic + fun main(args: Array) { + val frames = DataOutputStream(BufferedOutputStream(FileOutputStream(FileDescriptor.out))) + System.setOut(PrintStream(FileOutputStream(FileDescriptor.err), true)) + val input = DataInputStream(BufferedInputStream(System.`in`)) + val workers = + Executors.newFixedThreadPool(maxOf(2, Runtime.getRuntime().availableProcessors())) { task -> + Thread(task, "bridge-${threadIds.incrementAndGet()}").apply { isDaemon = true } + } + while (true) { + val id: Int + val request: ByteArray + try { + id = input.readInt() + request = ByteArray(input.readInt()).also { input.readFully(it) } + } catch (_: EOFException) { + break + } + workers.execute { + val response = call(request) + synchronized(frames) { + frames.writeInt(id) + frames.writeInt(response.size) + frames.write(response) + frames.flush() + } + } + } + // The calls already read are answered before the process ends + workers.shutdown() + workers.awaitTermination(1, TimeUnit.MINUTES) + } + + private val threadIds = AtomicInteger() private val ids = AtomicInteger() private val indexes = ConcurrentHashMap() private val cursors = ConcurrentHashMap() diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt new file mode 100644 index 00000000..f4a0e7d3 --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt @@ -0,0 +1,74 @@ +package io.github.kdroidfilter.seforimlibrary.search.index.bridge + +import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery +import org.apache.lucene.analysis.standard.StandardAnalyzer +import org.apache.lucene.document.Document +import org.apache.lucene.document.Field +import org.apache.lucene.document.StringField +import org.apache.lucene.index.IndexWriter +import org.apache.lucene.index.IndexWriterConfig +import org.apache.lucene.store.FSDirectory +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.deleteRecursively +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * The Android path of the bridge, on the desktop: a Java process serving [IndexBridgeServer.main], started by a script + * with the arguments of the Android launcher (`

[Java options...]`). + */ +class JavaProcessTransportTest { + private val dir: Path = Files.createTempDirectory("process_bridge") + private val indexDir: Path = + dir.resolve("index").also { index -> + IndexWriter(FSDirectory.open(index), IndexWriterConfig(StandardAnalyzer())).use { writer -> + repeat(3) { writer.addDocument(Document().apply { add(StringField("type", "line", Field.Store.YES)) }) } + } + } + private val launcher: File = + dir.resolve("launcher.sh").toFile().apply { + writeText("#!/bin/sh\njavaHome=$1; classPath=$2; main=$3; shift 3\nexec \"\$javaHome/bin/java\" -cp \"\$classPath\" \"$@\" \"\$main\"\n") + setExecutable(true) + } + + @OptIn(ExperimentalPathApi::class) + @AfterTest + fun cleanUp() { + dir.deleteRecursively() + } + + private fun transport() = + JavaProcessTransport( + launcher = launcher, + javaHome = File(System.getProperty("java.home")), + classPath = System.getProperty("java.class.path").split(File.pathSeparator).map(::File), + ) + + @Test + fun `the index answers through the Java process`() { + transport().use { transport -> + BridgedLuceneIndex(transport, indexDir.toString()).use { index -> + assertEquals(3, index.count(IndexQuery.MatchAll)) + assertEquals(2, index.search(IndexQuery.MatchAll, limit = 2, fields = setOf("type")).docs.size) + } + } + } + + @Test + fun `a process that died is started again on the next call`() { + transport().use { transport -> + BridgedLuceneIndex(transport, indexDir.toString()).use { index -> + assertEquals(3, index.count(IndexQuery.MatchAll)) + } + // Closing the transport ends its process; the next index starts a new one + transport.close() + BridgedLuceneIndex(transport, indexDir.toString()).use { index -> + assertEquals(3, index.count(IndexQuery.MatchAll)) + } + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 611f311d..be36cb50 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -69,7 +69,6 @@ dependencyResolutionManagement { include(":core") include(":dao") include(":search") -include(":lucene-android") include(":cli") include(":catalog") include(":searchindex") From 043e435ee8269ff038409d290d0f89d08bc1bf73 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 13:31:25 +0300 Subject: [PATCH 07/14] build(search): jdk-mobile-27b35.2, with the Android runtime for x86_64 too The Android runtimes are now downloaded per architecture (android-aarch64, android-x64: their java.home differs, UnixConstants among others), for devices and x86_64 emulators; the release is built on Linux for Android and smoke-tested on an x86_64 emulator. Signed-off-by: Elie Gambache --- IOS_RUNTIME.md | 14 ++++++++------ search/build.gradle.kts | 17 +++++++++++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/IOS_RUNTIME.md b/IOS_RUNTIME.md index fdba210f..5097d58f 100644 --- a/IOS_RUNTIME.md +++ b/IOS_RUNTIME.md @@ -35,8 +35,8 @@ app search (Zayit's `searchkit`, hybrid with the semantic search) gives the same The runtimes are built by the CI of [kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) (branch `mobile-runtime`: the recipe, `openjdk-mobile.patch` and `build.sh`): a stable OpenJDK, 27 (`openjdk/jdk27u`'s GA releases), with openjdk/mobile's port (which follows the JDK's mainline) ported to it. Every day, a new GA or recipe gets -its release, named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.1`: `jdk-27-ga`, the -JDK's build 35, the first release there). The build downloads one of them and checks its SHA-256 (`search`'s `javaRuntime` tasks: +its release, named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.2`: `jdk-27-ga`, the +JDK's build 35, the second release there). The build downloads one of them and checks its SHA-256 (`search`'s `javaRuntime` tasks: `jdkMobileVersion`, `jdkMobileBuild`, `jdkMobileRevision`, `mobileRuntimeSha256`, extracted in `search/build/javaRuntime/`; the iOS bindings only need its `include/`, extracted per target and release in `search/build/jni`); moving to a newer runtime is changing those, then running the tests below. @@ -49,7 +49,7 @@ Besides openjdk/mobile's port, the patch carries two fixes for Zero on iOS: 2. `os_bsd.cpp`: `pd_commit_memory` asks for executable pages (the code cache), which iOS refuses without the JIT entitlement; Zero never executes them, so they are committed read-write. -The assets (`jdk-mobile-27-ios-simulator-aarch64-b35.1.tar.gz`, `…-ios-device-…`) hold `static/` (`libjvm.a`, the JDK's +The assets (`jdk-mobile-27-ios-simulator-aarch64-b35.2.tar.gz`, `…-ios-device-…`) hold `static/` (`libjvm.a`, the JDK's native libraries, `libffi.a`), `include/` (`jni.h`, `jni_md.h`: the iOS bindings' headers) and `lib/` (the runtime's `java.home`: `conf/`, `lib/`, `modules//...`). `make images` and `make jmods` don't work for iOS (they link a dynamic libjvm): the runtime boots from an exploded image instead of a `lib/modules` jimage. @@ -96,13 +96,15 @@ hashes), so the comparison allows score differences: facets, snippets and highli Android 10+ (API 29: HotSpot's aarch64 code needs native ELF TLS). ART and HotSpot can't share a process (both define `JNI_CreateJavaVM`, both handle `SIGSEGV`), so Lucene runs in a Java process of the app's own, the server VM (with its -JIT, which Android allows) of the same release's Android runtime (`jdk-mobile-27-android-aarch64-b35.1.tar.gz`), behind +JIT, which Android allows) of the same release's Android runtimes (`jdk-mobile-27-android-aarch64-b35.2.tar.gz` and +`…-android-x64-…`: devices and x86_64 emulators, the CI smoke-testing the latter on one), behind the same bridge as iOS, its calls carried on the process's standard input and output (`JavaProcessTransport`, `IndexBridgeServer.main`) rather than JNI. -- The runtime holds the launcher, `jniLibs/arm64-v8a/libjvmlauncher.so`: an executable with the VM and the JDK's +- A runtime holds the launcher, `jniLibs//libjvmlauncher.so`: an executable with the VM and the JDK's natives in, packaged as a library (Android only executes native code from the APK; the app sets - `packaging.jniLibs.useLegacyPackaging = true` so it is installed as a file), and `lib/`, its `java.home`. The + `packaging.jniLibs.useLegacyPackaging = true` so it is installed as a file), and `lib/`, its `java.home` (one per + architecture: some of its generated classes, `UnixConstants` among them, depend on it). The launcher runs `IndexBridgeServer.main`; everything the VM prints goes to standard error, standard output carries the answers. - The process starts on the first call and again after it ends (Android may kill an app's child processes in the diff --git a/search/build.gradle.kts b/search/build.gradle.kts index bdd84e6e..4b0a14b5 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -14,21 +14,26 @@ group = "io.github.kdroidfilter.seforimlibrary" // The Java runtime Lucene runs in on iOS (OpenJDK's mobile port, see IOS_RUNTIME.md), built by the CI of // kdroidFilter/openjdk-mobile (branch mobile-runtime) and downloaded from its release, checked: OpenJDK 27 GA (the JDK's -// build 35) with openjdk/mobile's port, its first release there (jdk-mobile-27b35.1) +// build 35) with openjdk/mobile's port, its second release there (jdk-mobile-27b35.2: x86_64 Android added) val jdkMobileVersion = "27" val jdkMobileBuild = "35" -val jdkMobileRevision = "1" +val jdkMobileRevision = "2" val jdkMobileRelease = "jdk-mobile-${jdkMobileVersion}b$jdkMobileBuild.$jdkMobileRevision" val mobileRuntimeSha256 = mapOf( - "ios-simulator" to "0471359fb8711b9638c6f4432d170d74f8ece94994fe613c84406f690d661ded", - "ios-device" to "b5ca69777a83df475c2287ced5eb1df2fddefe2a9a235e9320ef0e488e509201", - "android" to "290039f7149558cb7eb7d2a317b24002928d0a65296b2582adf65c11d9fbc5b8", + "ios-simulator" to "5cebc9571c2162832ded028a4c5ab916983f12b290c67ba03d314d64f6611361", + "ios-device" to "b8d81c33fe36f73f70ed319500fdf00b5ff140f297764afa5b015891913c4456", + "android-aarch64" to "0f5d13d3726896a30423f2bdae7d60083ec4a4f3bf71b72c4706677cf2a418e5", + "android-x64" to "05e432d86b9f12ffa236b1c476b10102b520f7e085034dc0ffb8aa11f5d0cdd4", ) fun runtimeTaskSuffix(target: String): String = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } -fun runtimeAsset(target: String): String = "jdk-mobile-$jdkMobileVersion-$target-aarch64-b$jdkMobileBuild.$jdkMobileRevision.tar.gz" +// The asset of a target, named like the JBR's archives: - (the iOS runtimes are arm64 only) +fun runtimeAsset(target: String): String { + val platform = if (target.startsWith("ios-")) "$target-aarch64" else target + return "jdk-mobile-$jdkMobileVersion-$platform-b$jdkMobileBuild.$jdkMobileRevision.tar.gz" +} fun runtimeArchive(target: String) = layout.buildDirectory.file("javaRuntime/${runtimeAsset(target)}") From 81d97254c65fe17a80f54ea2148f1305d590cb6a Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Fri, 16 Oct 2026 17:41:12 +0300 Subject: [PATCH 08/14] refactor(search): Lucene through Nucleus JVM Access, on every platform MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-written bridge (the CBOR protocol, BridgedLuceneIndex, IndexBridgeServer, EmbeddedJvm, the Java process and its transport) gives way to Nucleus JVM Access (NucleusFramework/NucleusJvmAccess): - search-facade, a Kotlin/JVM module, is the façade: LuceneIndex (analyze, search, cursors, facets, collectLongs, count, KNN) and the value types, IndexQuery a sealed value; LuceneIndexBackend, internal, runs Lucene. - search applies NJA: its common code uses the proxies and value types NJA generates from the façade, in io.github.kdroidfilter.seforimlibrary.search.index (the package it used); the query helpers stay common. The desktop calls the façade in its own JVM, Android runs HotSpot in the app's process (libnja.so and the runtime in the library's native libraries and assets), iOS links Zero. - The runtimes are jdk-mobile-27b35.3's (NucleusFramework/openjdk-mobile: the Android runtimes' static libraries), downloaded, checked and staged for NJA (build/jdkMobile). - -Pnja.local= builds NJA from its sources until a release has what the search needs. - IOS_RUNTIME.md becomes JVM_RUNTIME.md. The same dumps as before on the real index: the desktop (through NJA) identical, iOS and Android 11 the same but for Lucene's score noise and hit totals. Signed-off-by: Elie Gambache --- IOS_RUNTIME.md | 113 ---------- JVM_RUNTIME.md | 72 ++++++ gradle/libs.versions.toml | 6 +- search-facade/build.gradle.kts | 23 ++ .../seforimlibrary/search/lucene/IndexApi.kt | 88 ++++---- .../search/lucene/LuceneIndex.kt | 67 ++++++ .../search/lucene}/LuceneIndexBackend.kt | 46 ++-- search/build.gradle.kts | 211 ++++++------------ .../index/bridge/JavaProcessTransport.kt | 133 ----------- .../search/LexicalSearchEngine.kt | 2 +- .../search/index/LuceneIndex.kt | 102 --------- .../search/index/QueryBuilders.kt | 15 ++ .../search/index/bridge/BridgeProtocol.kt | 121 ---------- .../search/index/bridge/BridgedLuceneIndex.kt | 102 --------- .../search/index/bridge/EmbeddedJvm.kt | 193 ---------------- .../search/index/bridge/JvmLuceneIndex.kt | 16 -- .../search/IosRealIndexParityTest.kt | 12 +- .../search/LuceneSearchEngine.kt | 10 +- .../search/index/bridge/IndexBridgeServer.kt | 160 ------------- .../index/bridge/BridgedLuceneIndexTest.kt | 90 -------- .../index/bridge/IndexBridgeServerTest.kt | 72 ------ .../index/bridge/JavaProcessTransportTest.kt | 74 ------ search/src/nativeInterop/cinterop/jni.def | 35 --- settings.gradle.kts | 7 + 24 files changed, 339 insertions(+), 1431 deletions(-) delete mode 100644 IOS_RUNTIME.md create mode 100644 JVM_RUNTIME.md create mode 100644 search-facade/build.gradle.kts rename search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt => search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt (51%) create mode 100644 search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt rename {search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index => search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene}/LuceneIndexBackend.kt (95%) delete mode 100644 search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt delete mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt create mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt delete mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt delete mode 100644 search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt delete mode 100644 search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt delete mode 100644 search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt delete mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt delete mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt delete mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt delete mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt delete mode 100644 search/src/nativeInterop/cinterop/jni.def diff --git a/IOS_RUNTIME.md b/IOS_RUNTIME.md deleted file mode 100644 index 5097d58f..00000000 --- a/IOS_RUNTIME.md +++ /dev/null @@ -1,113 +0,0 @@ -# The search on iOS: Lucene in an embedded Java runtime - -Lucene needs a JVM. On iOS, the app embeds one: OpenJDK's mobile port ([openjdk/mobile](https://github.com/openjdk/mobile)), -built as static libraries with the Zero VM (an interpreter, as iOS forbids generated code). Only the Lucene part of the -search runs there; everything else is Kotlin/Native. - -## Architecture - -``` -iOS app (Kotlin/Native) Java runtime in the same process (OpenJDK Zero) -─────────────────────── ───────────────────────────────────────────────── -LexicalSearchEngine, MagicDictionaryIndex (sqlite3), -snippets, highlights, hybrid fusion, semantic model -(ONNX Runtime C API), view models... - │ LuceneIndex - ▼ -JvmLuceneIndex ── CBOR over JNI (EmbeddedJvm.call) ──▶ IndexBridgeServer ──▶ LuceneIndexBackend (Lucene 10) -``` - -- `search/src/commonMain/.../index/LuceneIndex.kt`: the only part of the search that runs Lucene: analysis, searches - (`IndexQuery`, plain data), cursors, facets, KNN. Lucene runs on a JVM only (ART can't keep up with the Java versions - Lucene requires: Java 21 for 10, 25 for 11): the desktop implements it in process (`LuceneIndexBackend`), the mobile - apps through the bridge (`BridgedLuceneIndex`), with the same implementation running in the Java runtime they embed: - in the app on iOS (JNI), in a process of the app's own on Android. -- `EmbeddedJvm` starts the runtime (`JNI_CreateJavaVM`), attaches the calling threads, calls - `IndexBridgeServer.call(byte[]): byte[]` (CBOR of `BridgeRequest` / `BridgeResponse`). -- Lucene runs unpatched there (Zero is HotSpot), with `NIOFSDirectory` (no Panama mmap on Zero). - -Verified in the simulator on the real index (7M documents): the same facets, snippets, highlights, title suggestions -and find-in-book candidates as the desktop, and the same hits but for Lucene's score noise (see `SearchDump`); the whole -app search (Zayit's `searchkit`, hybrid with the semantic search) gives the same dump as the desktop. - -## The runtime - -The runtimes are built by the CI of [kdroidFilter/openjdk-mobile](https://github.com/kdroidFilter/openjdk-mobile) -(branch `mobile-runtime`: the recipe, `openjdk-mobile.patch` and `build.sh`): a stable OpenJDK, 27 (`openjdk/jdk27u`'s GA -releases), with openjdk/mobile's port (which follows the JDK's mainline) ported to it. Every day, a new GA or recipe gets -its release, named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.2`: `jdk-27-ga`, the -JDK's build 35, the second release there). The build downloads one of them and checks its SHA-256 (`search`'s `javaRuntime` tasks: -`jdkMobileVersion`, `jdkMobileBuild`, `jdkMobileRevision`, `mobileRuntimeSha256`, extracted in -`search/build/javaRuntime/`; the iOS bindings only need its `include/`, extracted per target and release in -`search/build/jni`); moving to a newer runtime is changing those, then running the tests below. - -Besides openjdk/mobile's port, the patch carries two fixes for Zero on iOS: - -1. `os_bsd_zero.cpp`: the W^X state (`DefaultWXWriteMode`, `os::_jit_exec_enabled`, `Thread::wx_enable_write`...) the - shared code needs on Apple aarch64 is only defined for the C1/C2 port (`bsd_aarch64`): Zero gets its own, without page - protection switches (it generates no code). -2. `os_bsd.cpp`: `pd_commit_memory` asks for executable pages (the code cache), which iOS refuses without the JIT - entitlement; Zero never executes them, so they are committed read-write. - -The assets (`jdk-mobile-27-ios-simulator-aarch64-b35.2.tar.gz`, `…-ios-device-…`) hold `static/` (`libjvm.a`, the JDK's -native libraries, `libffi.a`), `include/` (`jni.h`, `jni_md.h`: the iOS bindings' headers) and `lib/` (the runtime's -`java.home`: `conf/`, `lib/`, `modules//...`). `make images` and `make jmods` don't work for iOS (they link a -dynamic libjvm): the runtime boots from an exploded image instead of a `lib/modules` jimage. - -## Packaging it in the app - -**Linking** (the app target, or a test binary): - -``` --force_load static/libjvm.a --force_load static/{libjava,libnio,libnet,libzip,libjimage,libverify,libextnet,libfallbackLinker,libsyslookup}.a -static/libffi.a -export_dynamic -lz -lc++ -lsqlite3 --framework Foundation -framework CoreFoundation -framework Security -framework SystemConfiguration -``` - -`-force_load` and `-export_dynamic` are required: the runtime finds its natives by name (`dlsym`: `JIMAGE_Open`, -`Java_java_lang_...`, `JNI_OnLoad_*`), which the linker would otherwise drop or hide. The semantic search adds ONNX -Runtime's `onnxruntime.xcframework` (`onnxruntime-c` 1.20.0, the version of the desktop and Android builds). - -**Files** — the VM sets its `java.home` to `/lib` itself (`-Djava.home` is ignored), so the app bundle -holds: - -``` -App.app/lib/{conf,lib,modules}/... (the runtime's lib/: java.base, java.logging, java.management, jdk.unsupported) -App.app/lib/app/*.jar (./gradlew :search:mobileJvmClasspath: search-jvm, lucene-core, kotlin-stdlib, kotlinx-serialization) -``` - -About 50 MB uncompressed. Zayit's `./gradlew :searchkit:assembleIosJavaRuntimeDevice` assembles `static/` and `lib/` -(with the jars); `searchkit`'s `IosJavaRuntime` finds the jars in `lib/app` by default. - -## Tests - -```bash -./gradlew :search:iosSimulatorArm64Test -PindexDir= -PluceneQueryCache=off -./gradlew :search:jvmTest --tests '*RealIndexParityTest*' -PindexDir= -PluceneQueryCache=off -``` - -They write `search/build/parity/ios.txt` and `jvm.txt` (`SearchDump`). `-PluceneQueryCache=off` turns off Lucene's -query cache: it changes the last digits of the scores from one search to the next. Two runs on the same platform differ -anyway by Lucene's process-dependent clause order (`BooleanQuery` rewrites through a `HashMap` keyed by class identity -hashes), so the comparison allows score differences: facets, snippets and highlights must be identical. - -## Android - -Android 10+ (API 29: HotSpot's aarch64 code needs native ELF TLS). ART and HotSpot can't share a process (both define -`JNI_CreateJavaVM`, both handle `SIGSEGV`), so Lucene runs in a Java process of the app's own, the server VM (with its -JIT, which Android allows) of the same release's Android runtimes (`jdk-mobile-27-android-aarch64-b35.2.tar.gz` and -`…-android-x64-…`: devices and x86_64 emulators, the CI smoke-testing the latter on one), behind -the same bridge as iOS, its calls carried on the process's standard input and output (`JavaProcessTransport`, -`IndexBridgeServer.main`) rather than JNI. - -- A runtime holds the launcher, `jniLibs//libjvmlauncher.so`: an executable with the VM and the JDK's - natives in, packaged as a library (Android only executes native code from the APK; the app sets - `packaging.jniLibs.useLegacyPackaging = true` so it is installed as a file), and `lib/`, its `java.home` (one per - architecture: some of its generated classes, `UnixConstants` among them, depend on it). The - launcher runs `IndexBridgeServer.main`; everything the VM prints goes to standard error, standard output carries the - answers. -- The process starts on the first call and again after it ends (Android may kill an app's child processes in the - background): the calls sent to a process that ends fail, the next ones start another. -- `JavaProcessTransportTest` runs the same transport on the desktop, a `java` standing for the launcher. On a device, - Zayit's `searchkit-check` app runs the whole search. diff --git a/JVM_RUNTIME.md b/JVM_RUNTIME.md new file mode 100644 index 00000000..b11b9649 --- /dev/null +++ b/JVM_RUNTIME.md @@ -0,0 +1,72 @@ +# Lucene on a JVM, on every platform + +Lucene needs a JVM, and keeps up with the newest Java (21 for Lucene 10, 25 for Lucene 11): ART can't, so the search +runs Lucene on a real JVM everywhere, and everything else (query building, dictionary, ranking, snippets, highlights, +fusion, the semantic model) in common Kotlin. The bridge between both is +[Nucleus JVM Access](https://github.com/NucleusFramework/NucleusJvmAccess) (NJA). + +``` +search (Kotlin Multiplatform) search-facade (Kotlin/JVM): what runs on the JVM +────────────────────────────── ───────────────────────────────────────────────── +LexicalSearchEngine, MagicDictionaryIndex, snippets, +highlights... use the façade's API as if it were LuceneIndex(indexDir, cacheQueries): analyze, search, +local: io.github...search.index.LuceneIndex, ──▶ cursors, facets, collectLongs, count, KNN +IndexQuery, IndexPage, KnnRequest... (generated by IndexQuery (a sealed value), IndexPage, KnnRequest... +NJA from the façade's sources) LuceneIndexBackend (internal): Lucene 10 +``` + +| | Desktop (JVM) | Android 10+ | iOS | +|---|---|---|---| +| The JVM | the app's own | HotSpot (server VM, JIT), in the app's process next to ART | Zero (interpreter: iOS forbids generated code), linked into the app | +| A call | a plain call into NJA's dispatcher, on the calling thread | handed to a HotSpot thread paired with the calling one | JNI, on the calling thread | + +- `search-facade` holds the façade: its public API (`LuceneIndex`, `IndexCursor` and the value types: data classes, + enums, the sealed `IndexQuery`) is what NJA exports; `LuceneIndexBackend` is internal. +- NJA generates, from the façade's sources, the proxies and value types `search`'s common code uses, in + `io.github.kdroidfilter.seforimlibrary.search.index` (`njaFacade.clientPackage`: apart from the façade's own package, + as the desktop has both on its class path). Query helpers built in the app (`bool { }`) stay in common code. +- `LuceneIndex(path)` is the same constructor on every platform; `close()` closes the JVM's index. + +## The runtimes + +The mobile JVMs are jdk-mobile's: a stable OpenJDK (27, `openjdk/jdk27u`'s GA releases) with openjdk/mobile's port, +built by the CI of [NucleusFramework/openjdk-mobile](https://github.com/NucleusFramework/openjdk-mobile) and published +as releases named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.3`: `jdk-27-ga`, the +JDK's build 35, its third release there). `search` downloads the four runtimes (`android-aarch64`, `android-x64`, +`ios-simulator-aarch64`, `ios-device-aarch64`), checks their SHA-256 and stages them in `search/build/jdkMobile`, where +NJA reads them (`nucleusJvm.jdkMobile`); moving to a newer runtime is changing `jdkMobileVersion`, `jdkMobileBuild`, +`jdkMobileRevision` and `jdkMobileSha256`, then running the tests below. + +- **Android**: NJA links HotSpot and the JDK's natives into `libnja.so` (one per ABI: arm64-v8a, x86_64) and puts the + runtime (`java.home` per ABI, the façade's class path, the VM's options) in `search`'s assets, extracted once per + version on the device: an app depending on `search` gets both through the library, with nothing to configure. +- **iOS**: the binaries link jdk-mobile's static libraries (`-force_load`: the runtime finds its natives by name, + `-export_dynamic`) and carry NJA's bundle next to the executable: `lib/` (the VM's `java.home`, which it sets to + `/lib`) and `nja/` (the class path and the VM's options). For `search`'s own tests NJA does both; an + app does them as Zayit's `searchkit` shows (`assembleIosJavaRuntimeDevice`: `static/` to link, `bundle/` to copy into + `App.app`). About 50 MB uncompressed. +- **The desktop**: the façade is on the app's class path; nothing else. +- The VM's options are `nucleusJvm.jvmOptions` (a 128 MB heap: about 140 MB of memory on Android, 160 MB on iOS, on + real data). + +## Building with NJA's sources + +Until a release of NJA has what the search needs (JVM targets, primitive arrays, sealed value hierarchies), +`-Pnja.local=` builds its plugins and runtimes from its sources (settings.gradle.kts). + +## Tests + +```bash +./gradlew :search:jvmTest -PindexDir= -PluceneQueryCache=off # build/parity/jvm.txt +./gradlew :search:iosSimulatorArm64Test -PindexDir= -PluceneQueryCache=off # build/parity/ios.txt +``` + +They dump what the search answers on a real index (`SearchDump`). `-PluceneQueryCache=off` turns off Lucene's query +cache: it changes the last digits of the scores from one search to the next. Two runs differ anyway by Lucene's +process-dependent clause order (`BooleanQuery` rewrites through a `HashMap` keyed by class identity hashes), so the +comparison allows score differences: facets, snippets and highlights must be identical. On Android, Zayit's +`searchkit-check` app runs the whole search on a device or an emulator. + +Verified on the real index (7M documents): the desktop, the iOS simulator and Android 11 emulator give the same facets, +snippets, highlights, titles and find-in-book candidates (Zayit's `searchkit`, hybrid with the semantic search: the +same dump as the desktop, but for a hit total Lucene counts as a lower bound). diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5e5b52c2..1a8cbac1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,8 @@ filekitCore = "0.13.0" kotlin = "2.4.20" +# Nucleus JVM Access: Lucene on a JVM on every platform (the search's façade, search-facade) +nja = "0.2.0" agp = "9.1.1" jvmToolchain = "25" lucene = "10.5.2" @@ -33,7 +35,6 @@ kotlinx-coroutines-android = { module = "org.jetbrains.kotlinx:kotlinx-coroutine kotlinx-coroutines-swing = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-swing", version.ref = "kotlinx-coroutines" } kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } -kotlinx-serialization-cbor = { module = "org.jetbrains.kotlinx:kotlinx-serialization-cbor", version.ref = "kotlinx-serialization" } kotlinx-serialization-protobuf = { module = "org.jetbrains.kotlinx:kotlinx-serialization-protobuf", version.ref = "kotlinx-serialization" } kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime" } kermit = { module = "co.touchlab:kermit", version.ref = "kermit" } @@ -63,6 +64,9 @@ compose-runtime = { module = "org.jetbrains.compose.runtime:runtime", version.re [plugins] multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } +kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } +nja = { id = "dev.nucleusframework.nja", version.ref = "nja" } +nja-facade = { id = "dev.nucleusframework.nja.facade", version.ref = "nja" } android-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } maven-publish = { id = "com.vanniktech.maven.publish", version.ref = "maven-publish" } kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } diff --git a/search-facade/build.gradle.kts b/search-facade/build.gradle.kts new file mode 100644 index 00000000..458a6c82 --- /dev/null +++ b/search-facade/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { + id("org.jetbrains.kotlin.jvm") + alias(libs.plugins.nja.facade) +} + +group = "io.github.kdroidfilter.seforimlibrary" + +// The search's Lucene side, its façade for Nucleus JVM Access: it runs on a JVM (the desktop's own, the mobile apps' +// embedded one), and search's common code uses it through the proxies NJA generates (see JVM_RUNTIME.md) +kotlin { + jvmToolchain(libs.versions.jvmToolchain.get().toInt()) +} + +njaFacade { + libName = "search" + // The package the common code knows these types by (the façade's own is ...search.lucene: the desktop has both) + clientPackage = "io.github.kdroidfilter.seforimlibrary.search.index" +} + +dependencies { + implementation(libs.lucene.core) + testImplementation(kotlin("test")) +} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt similarity index 51% rename from search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt rename to search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt index c695249d..7da91c71 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/IndexQuery.kt +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/IndexApi.kt @@ -1,55 +1,39 @@ -package io.github.kdroidfilter.seforimlibrary.search.index - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable +package io.github.kdroidfilter.seforimlibrary.search.lucene /** * A Lucene query as plain data: the engines build it in common code, a [LuceneIndex] turns it into Lucene's own - * query, in this process (JVM, Android) or in the Java runtime the iOS app embeds. + * query, on a JVM (the desktop's own, the mobile apps' embedded one). */ -@Serializable sealed interface IndexQuery { /** Documents whose [field] holds the term [text]. */ - @Serializable - @SerialName("term") data class Term( val field: String, val text: String, ) : IndexQuery /** Documents whose [field] holds a term starting with [text]. */ - @Serializable - @SerialName("prefix") data class Prefix( val field: String, val text: String, ) : IndexQuery /** Documents whose [field] holds a term within [maxEdits] edits of [text]. */ - @Serializable - @SerialName("fuzzy") data class Fuzzy( val field: String, val text: String, val maxEdits: Int, ) : IndexQuery - @Serializable - @SerialName("bool") data class Bool( val clauses: List, ) : IndexQuery - @Serializable - @SerialName("boost") data class Boost( val query: IndexQuery, val boost: Float, ) : IndexQuery /** Each position matches any of its terms, within [slop] moves. */ - @Serializable - @SerialName("multiPhrase") data class MultiPhrase( val field: String, val positions: List>, @@ -60,8 +44,6 @@ sealed interface IndexQuery { * [text] analyzed like the index (StandardAnalyzer) into a phrase within [slop] moves (a term query for one term), * every term required when the analysis can't build a phrase, and nothing when it yields no term. */ - @Serializable - @SerialName("analyzedPhrase") data class AnalyzedPhrase( val field: String, val text: String, @@ -69,46 +51,74 @@ sealed interface IndexQuery { ) : IndexQuery /** Documents whose int point [field] is [value]. */ - @Serializable - @SerialName("intExact") data class IntExact( val field: String, val value: Int, ) : IndexQuery /** Documents whose int point [field] is one of [values]. */ - @Serializable - @SerialName("intSet") - class IntSet( + data class IntSet( val field: String, val values: IntArray, ) : IndexQuery /** Every document. */ - @Serializable - @SerialName("all") data object MatchAll : IndexQuery } -@Serializable data class Clause( val query: IndexQuery, val occur: Occur, ) -@Serializable enum class Occur { MUST, SHOULD, FILTER, MUST_NOT } -/** A [IndexQuery.Bool] built clause by clause. */ -class BoolBuilder { - private val clauses = ArrayList() +/** A document found: its score and the stored fields asked for, numeric ones in [longs], the others in [strings]. */ +data class IndexDoc( + val score: Float, + val longs: Map = emptyMap(), + val strings: Map = emptyMap(), +) + +/** Documents found, with the number of matches ([totalHits] may be a lower bound, as Lucene counts them). */ +data class IndexPage( + val docs: List, + val totalHits: Long, +) - fun add( - query: IndexQuery, - occur: Occur, - ): BoolBuilder = apply { clauses += Clause(query, occur) } +/** How the documents of a query spread over the books and categories (ancestors included). */ +data class IndexFacets( + val totalHits: Long, + val categoryCounts: Map, + val bookCounts: Map, +) - fun build(): IndexQuery.Bool = IndexQuery.Bool(clauses.toList()) -} +enum class SortType { INT, LONG } -inline fun bool(block: BoolBuilder.() -> Unit): IndexQuery.Bool = BoolBuilder().apply(block).build() +data class SortField( + val field: String, + val type: SortType, + val reverse: Boolean = false, +) + +enum class VectorEncoding { FLOAT32, BYTE } + +/** + * A nearest neighbors search of [field]: with [byteVector] on a byte field, else with [floatVector]; on a float field + * quantized by Lucene, [oversample] times more candidates are gathered, then the best [k] kept by their exact + * similarity. [filter] restricts the documents considered. + */ +data class KnnRequest( + val field: String, + val k: Int, + val fields: Set, + val floatVector: FloatArray? = null, + val byteVector: ByteArray? = null, + val filter: IndexQuery? = null, + val oversample: Int = 1, +) { + init { + // A NaN or infinite component means a broken embedding: its neighbors would be meaningless + require(floatVector?.all { it.isFinite() } != false) { "KNN vector with a NaN or infinite component" } + } +} diff --git a/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt new file mode 100644 index 00000000..d7a487db --- /dev/null +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndex.kt @@ -0,0 +1,67 @@ +package io.github.kdroidfilter.seforimlibrary.search.lucene + +import java.nio.file.Path + +/** + * The Lucene index of a search, read only: the only part of the search that runs Lucene, on a JVM (the desktop's own, + * the mobile apps' embedded one, through Nucleus JVM Access). Everything else (query building, dictionary, ranking, + * snippets, highlighting, fusion) is common code. + * + * Thread-safe; an [IndexCursor] is used by one caller at a time. + * + * @param indexDir the index directory + * @param cacheQueries whether to cache any query seen often (small indexes: the lookup index); else only the filters + * every search repeats + */ +class LuceneIndex( + indexDir: String, + cacheQueries: Boolean = false, +) : AutoCloseable { + private val backend = LuceneIndexBackend(Path.of(indexDir), cacheQueries) + + /** The terms of [text] as the index's analyzer (StandardAnalyzer, the `text` field) splits it. */ + fun analyze(text: String): List = backend.analyze(text) + + /** The best [limit] documents for [query] (by score, or by [sort]), with their stored [fields]. */ + fun search( + query: IndexQuery, + limit: Int, + fields: Set, + sort: List? = null, + ): IndexPage = backend.search(query, limit, fields, sort) + + /** Pages through the documents of [query] by score, on the index as it is now, with their stored [fields]. */ + fun openCursor( + query: IndexQuery, + fields: Set, + ): IndexCursor = IndexCursor(backend.openCursor(query, fields)) + + /** How the documents of [query] spread over the books and categories (`book_id`, `ancestor_category_ids`). */ + fun facets(query: IndexQuery): IndexFacets = backend.facets(query) + + /** The stored numeric [field] of every document of [query], unscored and in no particular order. */ + fun collectLongs( + query: IndexQuery, + field: String, + ): LongArray = backend.collectLongs(query, field) + + fun count(query: IndexQuery): Int = backend.count(query) + + /** How the vectors of [field] are stored, null when no segment has the field. */ + fun vectorEncoding(field: String): VectorEncoding? = backend.vectorEncoding(field) + + /** The nearest documents to a vector (see [KnnRequest]), best first, with their stored [KnnRequest.fields]. */ + fun knn(request: KnnRequest): List = backend.knn(request) + + override fun close() = backend.close() +} + +/** A page of a [LuceneIndex.openCursor]. */ +class IndexCursor internal constructor( + private val cursor: BackendCursor, +) : AutoCloseable { + /** The next [limit] documents, none once all were read. */ + fun next(limit: Int): IndexPage = cursor.next(limit) + + override fun close() = cursor.close() +} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt similarity index 95% rename from search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt rename to search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt index c709ee42..234a6bee 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndexBackend.kt +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt @@ -1,6 +1,5 @@ -package io.github.kdroidfilter.seforimlibrary.search.index +package io.github.kdroidfilter.seforimlibrary.search.lucene -import io.github.kdroidfilter.seforimlibrary.search.SearchFacets import org.apache.lucene.analysis.Analyzer import org.apache.lucene.analysis.standard.StandardAnalyzer import org.apache.lucene.analysis.tokenattributes.CharTermAttribute @@ -61,19 +60,19 @@ private const val FIELD_ANCESTOR_CATEGORY_IDS = "ancestor_category_ids" private const val MIN_DOCS_PER_SLICE = 100_000 /** - * The [LuceneIndex] of a Lucene directory, read with Lucene in this process: on the JVM, and inside the Java runtimes - * the mobile apps embed (see `IndexBridgeServer`). + * What [LuceneIndex] runs: a Lucene directory, read with Lucene in this JVM (the desktop's, the mobile apps' embedded + * one). * * @param indexDir the index directory * @param cacheQueries whether to cache any query seen often (small indexes: the lookup index); else only the filters * every search repeats * @param analyzer the analyzer the index was written with */ -class LuceneIndexBackend( +internal class LuceneIndexBackend( private val indexDir: Path, private val cacheQueries: Boolean = false, private val analyzer: Analyzer = StandardAnalyzer(), -) : LuceneIndex { +) : AutoCloseable { // Open lazily to avoid any I/O at app startup private val dir: Directory by lazy { openDirectory(indexDir) } @@ -138,7 +137,7 @@ class LuceneIndexBackend( } } - override fun analyze(text: String): List = + fun analyze(text: String): List = try { val out = mutableListOf() analyzer.tokenStream("text", text).use { ts -> @@ -155,7 +154,7 @@ class LuceneIndexBackend( emptyList() } - override fun search( + fun search( query: IndexQuery, limit: Int, fields: Set, @@ -172,16 +171,16 @@ class LuceneIndexBackend( top.toPage(searcher.storedFields(), fields) } - override fun openCursor( + fun openCursor( query: IndexQuery, fields: Set, - ): IndexCursor = Cursor(toLucene(query), fields, acquire()) + ): BackendCursor = Cursor(toLucene(query), fields, acquire()) private inner class Cursor( private val query: Query, private val fields: Set, private val searcher: IndexSearcher, - ) : IndexCursor { + ) : BackendCursor { private var after: ScoreDoc? = null private var totalHits: Long? = null private var closed = false @@ -200,7 +199,7 @@ class LuceneIndexBackend( } } - override fun facets(query: IndexQuery): SearchFacets = + fun facets(query: IndexQuery): IndexFacets = withSearcher { searcher -> // Only book ids are read per hit (doc values when the index has them, else the first stored field); // category counts follow from the book counts, as every line of a book shares its ancestors. One @@ -254,17 +253,17 @@ class LuceneIndexBackend( searcher.search( toLucene(query), - object : CollectorManager { + object : CollectorManager { override fun newCollector() = FacetCollector() - override fun reduce(collectors: Collection): SearchFacets { + override fun reduce(collectors: Collection): IndexFacets { val bookCounts = HashMap() collectors.forEach { c -> c.bookCounts.forEach { (book, n) -> bookCounts.merge(book, n, Int::plus) } } val categoryCounts = HashMap() for ((bookId, count) in bookCounts) { bookAncestors[bookId]?.forEach { categoryCounts.merge(it, count, Int::plus) } } - return SearchFacets( + return IndexFacets( totalHits = collectors.sumOf { it.totalHits }, categoryCounts = categoryCounts, bookCounts = bookCounts, @@ -274,7 +273,7 @@ class LuceneIndexBackend( ) } - override fun collectLongs( + fun collectLongs( query: IndexQuery, field: String, ): LongArray = @@ -323,9 +322,9 @@ class LuceneIndexBackend( ) } - override fun count(query: IndexQuery): Int = withSearcher { it.count(toLucene(query)) } + fun count(query: IndexQuery): Int = withSearcher { it.count(toLucene(query)) } - override fun vectorEncoding(field: String): VectorEncoding? = + fun vectorEncoding(field: String): VectorEncoding? = withSearcher { searcher -> searcher.indexReader .leaves() @@ -333,7 +332,7 @@ class LuceneIndexBackend( ?.let { if (it.vectorEncoding == LuceneVectorEncoding.BYTE) VectorEncoding.BYTE else VectorEncoding.FLOAT32 } } - override fun knn(request: KnnRequest): List = + fun knn(request: KnnRequest): List = withSearcher { searcher -> val filter = request.filter?.let { toLucene(it) } val byteVector = request.byteVector @@ -430,8 +429,8 @@ class LuceneIndexBackend( LuceneSortField( field, when (type) { - SortField.Type.INT -> LuceneSortField.Type.INT - SortField.Type.LONG -> LuceneSortField.Type.LONG + SortType.INT -> LuceneSortField.Type.INT + SortType.LONG -> LuceneSortField.Type.LONG }, reverse, ) @@ -575,3 +574,8 @@ private object StableFiltersCachingPolicy : QueryCachingPolicy { else -> false } } + +/** A cursor of the backend, behind an [IndexCursor]. */ +internal interface BackendCursor : AutoCloseable { + fun next(limit: Int): IndexPage +} diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 4b0a14b5..12642dd1 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -1,4 +1,3 @@ -import org.jetbrains.kotlin.gradle.plugin.mpp.TestExecutable import org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeSimulatorTest import java.net.URI import java.security.DigestInputStream @@ -8,101 +7,63 @@ plugins { alias(libs.plugins.multiplatform) alias(libs.plugins.android.library) alias(libs.plugins.kotlinx.serialization) + alias(libs.plugins.nja) } group = "io.github.kdroidfilter.seforimlibrary" -// The Java runtime Lucene runs in on iOS (OpenJDK's mobile port, see IOS_RUNTIME.md), built by the CI of -// kdroidFilter/openjdk-mobile (branch mobile-runtime) and downloaded from its release, checked: OpenJDK 27 GA (the JDK's -// build 35) with openjdk/mobile's port, its second release there (jdk-mobile-27b35.2: x86_64 Android added) +// The Java runtimes Lucene runs in on the mobile platforms (see JVM_RUNTIME.md): jdk-mobile's releases, built by the CI +// of NucleusFramework/openjdk-mobile, downloaded and checked: OpenJDK 27 GA (the JDK's build 35) with openjdk/mobile's +// port, its third release there (jdk-mobile-27b35.3: the Android runtimes' static libraries, for NJA's in-process VM) val jdkMobileVersion = "27" val jdkMobileBuild = "35" -val jdkMobileRevision = "2" +val jdkMobileRevision = "3" val jdkMobileRelease = "jdk-mobile-${jdkMobileVersion}b$jdkMobileBuild.$jdkMobileRevision" -val mobileRuntimeSha256 = +val jdkMobileSha256 = mapOf( - "ios-simulator" to "5cebc9571c2162832ded028a4c5ab916983f12b290c67ba03d314d64f6611361", - "ios-device" to "b8d81c33fe36f73f70ed319500fdf00b5ff140f297764afa5b015891913c4456", - "android-aarch64" to "0f5d13d3726896a30423f2bdae7d60083ec4a4f3bf71b72c4706677cf2a418e5", - "android-x64" to "05e432d86b9f12ffa236b1c476b10102b520f7e085034dc0ffb8aa11f5d0cdd4", + "android-aarch64" to "625d0dcdb36f1e935b06ae5720169e77145500c23987a38e85600a4093ea4396", + "android-x64" to "e6f69c83dd4c04a1822a53815d1a961d510b5b471c96313ee39e7a257148c654", + "ios-simulator-aarch64" to "14043e85df2a019cba5ad55ac19979f4cacbfbd18ada8b2aeb895b52be0d2174", + "ios-device-aarch64" to "e38dc41f3a85b2f0263692030e03228b4d3127d1da07f52a77215b5ce7d962d6", ) -fun runtimeTaskSuffix(target: String): String = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } - -// The asset of a target, named like the JBR's archives: - (the iOS runtimes are arm64 only) -fun runtimeAsset(target: String): String { - val platform = if (target.startsWith("ios-")) "$target-aarch64" else target - return "jdk-mobile-$jdkMobileVersion-$platform-b$jdkMobileBuild.$jdkMobileRevision.tar.gz" -} - -fun runtimeArchive(target: String) = layout.buildDirectory.file("javaRuntime/${runtimeAsset(target)}") - -val downloadJavaRuntimes = - mobileRuntimeSha256.mapValues { (target, sha256) -> - val asset = runtimeAsset(target) - val archive = runtimeArchive(target) - tasks.register("downloadJavaRuntime${runtimeTaskSuffix(target)}") { - description = "Downloads the Java runtime for $target ($jdkMobileRelease)." - val url = "https://github.com/kdroidFilter/openjdk-mobile/releases/download/$jdkMobileRelease/$asset" - inputs.property("url", url) - inputs.property("sha256", sha256) - outputs.file(archive) - doLast { - // Not a download for nothing: a release's checksums are set here once it is published - check(sha256.matches(Regex("[0-9a-f]{64}"))) { "No SHA-256 for $asset yet: set it in mobileRuntimeSha256 (see IOS_RUNTIME.md)" } - val file = archive.get().asFile.apply { parentFile.mkdirs() } - val digest = MessageDigest.getInstance("SHA-256") - DigestInputStream(URI(url).toURL().openStream(), digest).use { input -> - file.outputStream().use { input.copyTo(it) } - } - val actual = digest.digest().joinToString("") { "%02x".format(it) } - if (actual != sha256) { - file.delete() - error("Unexpected $asset from $url (SHA-256 $actual)") +// build/jdkMobile/, the layout NJA reads (nucleusJvm.jdkMobile): static/, include/ and lib/ (java.home) +val jdkMobileDir = layout.buildDirectory.dir("jdkMobile") +val stageJdkMobile = + jdkMobileSha256.map { (target, sha256) -> + val asset = "jdk-mobile-$jdkMobileVersion-$target-b$jdkMobileBuild.$jdkMobileRevision.tar.gz" + val archive = layout.buildDirectory.file("jdkMobileDownloads/$asset") + val suffix = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } + val download = + tasks.register("downloadJdkMobile$suffix") { + description = "Downloads jdk-mobile's runtime for $target ($jdkMobileRelease)." + val url = "https://github.com/NucleusFramework/openjdk-mobile/releases/download/$jdkMobileRelease/$asset" + inputs.property("url", url) + inputs.property("sha256", sha256) + outputs.file(archive) + doLast { + val file = archive.get().asFile.apply { parentFile.mkdirs() } + val digest = MessageDigest.getInstance("SHA-256") + DigestInputStream(URI(url).toURL().openStream(), digest).use { input -> + file.outputStream().use { input.copyTo(it) } + } + val actual = digest.digest().joinToString("") { "%02x".format(it) } + if (actual != sha256) { + file.delete() + error("Unexpected $asset from $url (SHA-256 $actual)") + } } } - } - } - -// build/javaRuntime/: include/ (its JNI headers), lib/ (its java.home) and, on iOS, static/ (the libraries an app -// links), on Android jniLibs/ (the launcher of the Java process) -val javaRuntimes = - downloadJavaRuntimes.mapValues { (target, download) -> - tasks.register("javaRuntime${runtimeTaskSuffix(target)}") { - description = "Extracts the Java runtime for $target." + tasks.register("stageJdkMobile$suffix") { + description = "Extracts jdk-mobile's runtime for $target." dependsOn(download) - from(tarTree(runtimeArchive(target))) - into(layout.buildDirectory.dir("javaRuntime/$target")) + from(tarTree(archive)) + into(jdkMobileDir.map { it.dir(target) }) // Some of the JDK's files are read-only: the next sync couldn't replace them filePermissions { user { write = true } } } } -// The runtime's static libraries an iOS binary links (force-loaded: the runtime finds its natives by name) -val jdkStaticLibs = - listOf( - "libjvm.a", "libjava.a", "libnio.a", "libnet.a", "libzip.a", "libjimage.a", "libverify.a", "libextnet.a", - "libfallbackLinker.a", "libsyslookup.a", - ) -val iosSimulatorRuntime = layout.buildDirectory.dir("javaRuntime/ios-simulator").get().asFile - -// The JNI headers of each target's runtime, for its iOS bindings (cinterop "jni"): extracted alone (compiling needs -// nothing else of the runtime), in a directory per release, so a new runtime regenerates the bindings -val jniHeaders = - downloadJavaRuntimes.mapValues { (target, download) -> - tasks.register("jniHeaders${runtimeTaskSuffix(target)}") { - description = "Extracts the JNI headers of the Java runtime for $target." - dependsOn(download) - from(tarTree(runtimeArchive(target))) { include("include/**") } - into(layout.buildDirectory.dir("jni/$jdkMobileRelease/$target")) - } - } - -fun jniHeadersDir(target: String): File = layout.buildDirectory.dir("jni/$jdkMobileRelease/$target/include").get().asFile - -// The runtime target of each Kotlin/Native target -val iosRuntimeTargets = mapOf("iosArm64" to "ios-device", "iosSimulatorArm64" to "ios-simulator") - kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) @@ -110,26 +71,11 @@ kotlin { android { namespace = "io.github.kdroidfilter.seforimlibrary.search" compileSdk = 35 - // Lucene runs in a Java process of the app's own (JavaProcessTransport), from Android 10 + // NJA's bridge runs from Android 10 minSdk = 29 } - // The iOS tests link the runtime and run the search end to end in the simulator - iosSimulatorArm64 { - binaries.withType().configureEach { - val static = iosSimulatorRuntime.resolve("static") - linkerOpts(jdkStaticLibs.flatMap { listOf("-force_load", static.resolve(it).path) }) - linkerOpts(static.resolve("libffi.a").path, "-export_dynamic", "-lz", "-lc++") - listOf("Foundation", "CoreFoundation", "Security", "SystemConfiguration").forEach { linkerOpts("-framework", it) } - linkTaskProvider.configure { dependsOn(javaRuntimes.getValue("ios-simulator")) } - } - } listOf(iosArm64(), iosSimulatorArm64()).forEach { target -> target.compilations.getByName("main").cinterops { - create("jni") { - definitionFile.set(project.file("src/nativeInterop/cinterop/jni.def")) - // The target's own runtime's - includeDirs(jniHeadersDir(iosRuntimeTargets.getValue(target.name))) - } create("sqlite3") { definitionFile.set(project.file("src/nativeInterop/cinterop/sqlite3.def")) } @@ -140,14 +86,14 @@ kotlin { commonMain.dependencies { api(project(":core")) implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.cbor) + // SearchFacets, which the apps keep + api(libs.kotlinx.serialization.json) implementation(libs.kotlinx.io.core) implementation(libs.atomicfu) implementation(libs.kermit) } - // Lucene runs on a JVM only: in process on the desktop, in the Java runtime the mobile apps embed (a process of - // the app's own on Android, in the app on iOS), behind the same bridge. The JVM and Android share the rest + // The JVM and Android share the HTML and coroutine helpers (Lucene runs behind the façade, on a JVM) val androidJvmMain by creating { dependsOn(commonMain.get()) dependencies { @@ -157,8 +103,6 @@ kotlin { jvmMain { dependsOn(androidJvmMain) dependencies { - implementation(libs.lucene.core) - implementation(libs.lucene.analysis.common) implementation(libs.sqlite.jdbc) } } @@ -177,56 +121,37 @@ kotlin { jvmTest.dependencies { implementation(kotlin("test")) implementation(libs.kotlinx.coroutines.test) + // The test indexes are written with Lucene directly + implementation(libs.lucene.core) + implementation(libs.lucene.analysis.common) } } } -iosRuntimeTargets.forEach { (konanTarget, runtimeTarget) -> - tasks.matching { it.name == "cinteropJni${konanTarget.replaceFirstChar(Char::uppercase)}" }.configureEach { - dependsOn(jniHeaders.getValue(runtimeTarget)) - // The headers' content, not only their directory: the bindings follow them - inputs.dir(jniHeadersDir(runtimeTarget)).withPathSensitivity(PathSensitivity.RELATIVE) - } +// Lucene, through Nucleus JVM Access: the façade (search-facade) runs on a JVM, the desktop's own or the one the mobile +// apps embed (iOS: Zero linked into the app; Android: HotSpot in the app's process), and common code uses its proxies +nucleusJvm { + facade(project(":search-facade")) + jdkMobile.set(jdkMobileDir) + androidAbis.set(setOf("arm64-v8a", "x86_64")) + // Lucene maps its files with Panama (madvise): native access, without the warning + jvmOptions.addAll("-Xmx128m", "--enable-native-access=ALL-UNNAMED") + // Lucene's query cache off: the same search ranks the same every time (the parity tests across platforms) + providers.gradleProperty("luceneQueryCache").orNull?.let { jvmOptions.add("-Dseforim.lucene.queryCache=$it") } } -// The jars the embedded Java runtime runs: this module (its Lucene side and IndexBridgeServer), Lucene, Kotlin -val mobileJvmClasspath by tasks.registering(Sync::class) { - description = "Collects the jars the Java runtime of a mobile app runs for the search (iOS, Android)." - from(tasks.named("jvmJar")) - from( - configurations.named("jvmRuntimeClasspath").map { classpath -> - classpath.filter { jar -> listOf("lucene-core", "kotlin-stdlib", "kotlinx-serialization-core", "kotlinx-serialization-cbor").any { jar.name.startsWith(it) } } - }, - ) - into(layout.buildDirectory.dir("mobileJvmClasspath")) -} - -run { - val testBinaryDir = layout.buildDirectory.dir("bin/iosSimulatorArm64/debugTest") - - // java.home of the runtime is /lib: its classes (an exploded image), conf and data, the jars beside - val iosTestRuntime by tasks.registering(Sync::class) { - dependsOn(javaRuntimes.getValue("ios-simulator")) - from(iosSimulatorRuntime.resolve("lib")) - from(mobileJvmClasspath) { into("app") } - into(testBinaryDir.map { it.dir("lib") }) - filePermissions { user { write = true } } - } - - tasks.withType().configureEach { - dependsOn(iosTestRuntime) - device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) - val jars = mobileJvmClasspath.map { sync -> sync.destinationDir.listFiles().orEmpty().map { testBinaryDir.get().dir("lib/app/${it.name}").asFile.path } } - val indexDir = providers.gradleProperty("indexDir") - val parityOut = layout.buildDirectory.file("parity/ios.txt") - val luceneQueryCache = providers.gradleProperty("luceneQueryCache") - doFirst { - // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix - environment("SIMCTL_CHILD_SEFORIM_JVM_CLASSPATH", jars.get().joinToString(":")) - indexDir.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_INDEX_DIR", it) } - environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", parityOut.get().asFile.path) - luceneQueryCache.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_JVM_OPTIONS", "-Dseforim.lucene.queryCache=$it") } - } +// NJA reads the runtimes from jdkMobile, and the iOS binaries link them: staged first +tasks.matching { it.name.startsWith("nja") }.configureEach { dependsOn(stageJdkMobile) } +tasks.withType().configureEach { dependsOn(stageJdkMobile) } + +tasks.withType().configureEach { + device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) + val indexDir = providers.gradleProperty("indexDir") + val parityOut = layout.buildDirectory.file("parity/ios.txt") + doFirst { + // The simulator passes the SIMCTL_CHILD_ variables to the test process, without the prefix + indexDir.orNull?.let { environment("SIMCTL_CHILD_SEFORIM_INDEX_DIR", it) } + environment("SIMCTL_CHILD_SEFORIM_PARITY_OUT", parityOut.get().asFile.path) } } diff --git a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt b/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt deleted file mode 100644 index 09a42478..00000000 --- a/search/src/androidJvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransport.kt +++ /dev/null @@ -1,133 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import co.touchlab.kermit.Logger -import java.io.BufferedInputStream -import java.io.BufferedOutputStream -import java.io.DataInputStream -import java.io.DataOutputStream -import java.io.File -import java.io.IOException -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ExecutionException -import java.util.concurrent.atomic.AtomicInteger - -private const val SERVER_CLASS = "io.github.kdroidfilter.seforimlibrary.search.index.bridge.IndexBridgeServer" -private const val STDERR_LINES_KEPT = 40 - -// Far above any answer (a page of hits, facets), far below a misread frame -private const val MAX_ANSWER_BYTES = 64 * 1024 * 1024 - -/** - * The calls to a Java process of the app's own, for Android, whose ART can't run Lucene: OpenJDK's mobile port (built - * for Android, see IOS_RUNTIME.md) runs `IndexBridgeServer.main`, which serves the calls on its standard input and - * output. A separate process, as ART and HotSpot can't share one. It starts on the first call and again after it dies - * (Android may kill an app's child processes in the background). - * - * Android only executes the native code an APK holds: [launcher] is the runtime's launcher, an executable packaged as a - * library (`libjvmlauncher.so`, from the app's native library dir), the VM linked in; the JDK's classes and data are in - * the app's files ([javaHome], an exploded image: `modules/`, `conf/`, `lib/`). - * - * @param classPath the jars of the search's Java side (`mobileJvmClasspath`, the same as iOS) - * @param options the Java process's options (`-Xmx...`) - */ -class JavaProcessTransport( - private val launcher: File, - private val javaHome: File, - private val classPath: List, - private val options: List = listOf("-Xmx512m"), -) : BridgeTransport, - AutoCloseable { - private val lock = Any() - private var running: Running? = null - private val ids = AtomicInteger() - private val logger = Logger.withTag("SeforimJava") - - // A started process with the calls sent to it: when it ends, they fail, and those of the next process don't - private class Running( - val process: Process, - ) { - val input = DataOutputStream(BufferedOutputStream(process.outputStream)) - val pending = ConcurrentHashMap>() - val stderr = ArrayDeque() - } - - override fun call(request: ByteArray): ByteArray { - val id = ids.incrementAndGet() - val answer = CompletableFuture() - synchronized(lock) { - val run = started() - run.pending[id] = answer - try { - run.input.writeInt(id) - run.input.writeInt(request.size) - run.input.write(request) - run.input.flush() - } catch (e: IOException) { - // The process is gone: its reader fails the calls sent to it, this one included - run.pending.remove(id) - throw IllegalStateException("The Java process can't be reached", e) - } - } - return try { - answer.get() - } catch (e: ExecutionException) { - throw e.cause ?: e - } - } - - // The running process, starting it when there is none - private fun started(): Running { - running?.let { return it } - val command = listOf(launcher.path, javaHome.path, classPath.joinToString(":") { it.path }, SERVER_CLASS) + options - val run = Running(ProcessBuilder(command).start()) - running = run - Thread({ readAnswers(run) }, "seforim-java-answers").apply { isDaemon = true }.start() - Thread({ readErrors(run) }, "seforim-java-stderr").apply { isDaemon = true }.start() - return run - } - - private fun readAnswers(run: Running) { - val answers = DataInputStream(BufferedInputStream(run.process.inputStream)) - try { - while (true) { - val id = answers.readInt() - val size = answers.readInt() - // Anything but a frame here means the process wrote something else on its output: stop reading it - check(size in 0..MAX_ANSWER_BYTES) { "Not an answer from the Java process (size $size)" } - val bytes = ByteArray(size).also { answers.readFully(it) } - run.pending.remove(id)?.complete(bytes) - } - } catch (failure: Throwable) { - logger.w { "Java process answers: $failure" } - run.process.destroy() - // The process ended: the calls sent to it fail, the next one starts another - val exit = runCatching { run.process.waitFor() }.getOrNull() - val reason = IllegalStateException("The Java process ended (exit $exit): ${synchronized(run.stderr) { run.stderr.joinToString("\n") }}") - synchronized(lock) { - if (running === run) running = null - } - run.pending.keys.toList().forEach { id -> run.pending.remove(id)?.completeExceptionally(reason) } - } - } - - private fun readErrors(run: Running) { - run.process.errorStream.bufferedReader().forEachLine { line -> - logger.w { line } - synchronized(run.stderr) { - run.stderr.addLast(line) - if (run.stderr.size > STDERR_LINES_KEPT) run.stderr.removeFirst() - } - } - } - - override fun close() { - synchronized(lock) { - running?.let { run -> - runCatching { run.input.close() } - run.process.destroy() - } - running = null - } - } -} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt index 6dbbbe30..d914392f 100644 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LexicalSearchEngine.kt @@ -252,7 +252,7 @@ class LexicalSearchEngine( val context = buildSearchContext(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) ?: return null - return index.facets(context.matchQuery) + return index.facets(context.matchQuery).let { SearchFacets(it.totalHits, it.categoryCounts, it.bookCounts) } } // --- Inner SearchSession class --- diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt deleted file mode 100644 index c74d17df..00000000 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/LuceneIndex.kt +++ /dev/null @@ -1,102 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index - -import io.github.kdroidfilter.seforimlibrary.search.SearchFacets -import kotlinx.serialization.Serializable - -/** - * The Lucene index of a search, read only: the only part of the search that runs Lucene. The JVM and Android read it in - * process; iOS through the Java runtime it embeds (Lucene needs a JVM), with the same implementation. Everything else - * (query building, dictionary, ranking, snippets, highlighting, fusion) is common code. - * - * Implementations are thread-safe; a [IndexCursor] is used by one caller at a time. - */ -interface LuceneIndex : AutoCloseable { - /** The terms of [text] as the index's analyzer (StandardAnalyzer, the `text` field) splits it. */ - fun analyze(text: String): List - - /** The best [limit] documents for [query] (by score, or by [sort]), with their stored [fields]. */ - fun search( - query: IndexQuery, - limit: Int, - fields: Set, - sort: List? = null, - ): IndexPage - - /** Pages through the documents of [query] by score, on the index as it is now, with their stored [fields]. */ - fun openCursor( - query: IndexQuery, - fields: Set, - ): IndexCursor - - /** How the documents of [query] spread over the books and categories (`book_id`, `ancestor_category_ids`). */ - fun facets(query: IndexQuery): SearchFacets - - /** The stored numeric [field] of every document of [query], unscored and in no particular order. */ - fun collectLongs( - query: IndexQuery, - field: String, - ): LongArray - - fun count(query: IndexQuery): Int - - /** How the vectors of [field] are stored, null when no segment has the field. */ - fun vectorEncoding(field: String): VectorEncoding? - - /** The nearest documents to a vector (see [KnnRequest]), best first, with their stored [KnnRequest.fields]. */ - fun knn(request: KnnRequest): List -} - -/** A page of a [LuceneIndex.openCursor]. */ -interface IndexCursor : AutoCloseable { - /** The next [limit] documents, none once all were read. */ - fun next(limit: Int): IndexPage -} - -/** A document found: its score and the stored fields asked for, numeric ones in [longs], the others in [strings]. */ -@Serializable -class IndexDoc( - val score: Float, - val longs: Map = emptyMap(), - val strings: Map = emptyMap(), -) - -/** Documents found, with the number of matches ([totalHits] may be a lower bound, as Lucene counts them). */ -@Serializable -class IndexPage( - val docs: List, - val totalHits: Long, -) - -@Serializable -data class SortField( - val field: String, - val type: Type, - val reverse: Boolean = false, -) { - @Serializable - enum class Type { INT, LONG } -} - -@Serializable -enum class VectorEncoding { FLOAT32, BYTE } - -/** - * A nearest neighbors search of [field]: with [byteVector] on a byte field, else with [floatVector]; on a float field - * quantized by Lucene, [oversample] times more candidates are gathered, then the best [k] kept by their exact - * similarity. [filter] restricts the documents considered. - */ -@Serializable -class KnnRequest( - val field: String, - val k: Int, - val fields: Set, - val floatVector: FloatArray? = null, - val byteVector: ByteArray? = null, - val filter: IndexQuery? = null, - val oversample: Int = 1, -) { - init { - // A NaN or infinite component means a broken embedding: its neighbors would be meaningless - require(floatVector?.all { it.isFinite() } != false) { "KNN vector with a NaN or infinite component" } - } -} diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt new file mode 100644 index 00000000..9dc2c981 --- /dev/null +++ b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/QueryBuilders.kt @@ -0,0 +1,15 @@ +package io.github.kdroidfilter.seforimlibrary.search.index + +/** A [IndexQuery.Bool] built clause by clause. */ +class BoolBuilder { + private val clauses = ArrayList() + + fun add( + query: IndexQuery, + occur: Occur, + ): BoolBuilder = apply { clauses += Clause(query, occur) } + + fun build(): IndexQuery.Bool = IndexQuery.Bool(clauses.toList()) +} + +inline fun bool(block: BoolBuilder.() -> Unit): IndexQuery.Bool = BoolBuilder().apply(block).build() diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt deleted file mode 100644 index 52d9296c..00000000 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgeProtocol.kt +++ /dev/null @@ -1,121 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery -import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest -import io.github.kdroidfilter.seforimlibrary.search.index.SortField -import kotlinx.serialization.ExperimentalSerializationApi -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import kotlinx.serialization.cbor.Cbor - -/** - * The calls of a [io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex] across the bridge between the iOS - * app and the Java runtime it embeds: CBOR both ways, each index and cursor named by a handle. - */ -@Serializable -internal sealed interface BridgeRequest { - @Serializable - @SerialName("open") - data class Open( - val indexDir: String, - val cacheQueries: Boolean, - ) : BridgeRequest - - @Serializable - @SerialName("close") - data class Close( - val index: Int, - ) : BridgeRequest - - @Serializable - @SerialName("analyze") - data class Analyze( - val index: Int, - val text: String, - ) : BridgeRequest - - @Serializable - @SerialName("search") - data class Search( - val index: Int, - val query: IndexQuery, - val limit: Int, - val fields: Set, - val sort: List? = null, - ) : BridgeRequest - - @Serializable - @SerialName("openCursor") - data class OpenCursor( - val index: Int, - val query: IndexQuery, - val fields: Set, - ) : BridgeRequest - - @Serializable - @SerialName("next") - data class CursorNext( - val cursor: Int, - val limit: Int, - ) : BridgeRequest - - @Serializable - @SerialName("closeCursor") - data class CloseCursor( - val cursor: Int, - ) : BridgeRequest - - @Serializable - @SerialName("facets") - data class Facets( - val index: Int, - val query: IndexQuery, - ) : BridgeRequest - - @Serializable - @SerialName("collectLongs") - data class CollectLongs( - val index: Int, - val query: IndexQuery, - val field: String, - ) : BridgeRequest - - @Serializable - @SerialName("count") - data class Count( - val index: Int, - val query: IndexQuery, - ) : BridgeRequest - - @Serializable - @SerialName("vectorEncoding") - data class VectorEncodingOf( - val index: Int, - val field: String, - ) : BridgeRequest - - @Serializable - @SerialName("knn") - class Knn( - val index: Int, - val request: KnnRequest, - ) : BridgeRequest -} - -/** - * The result of a call ([result]: CBOR of a type depending on the call, decoded by the caller; none for a call without - * one), or the failure it raised ([error]). - */ -@Serializable -internal class BridgeResponse( - val result: ByteArray? = null, - val error: String? = null, -) - -@OptIn(ExperimentalSerializationApi::class) -internal val BridgeCbor = - Cbor { - ignoreUnknownKeys = true - // Byte arrays (results, byte vectors) as one CBOR byte string, not an array of one integer per byte - alwaysUseByteString = true - } diff --git a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt b/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt deleted file mode 100644 index 9f619424..00000000 --- a/search/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndex.kt +++ /dev/null @@ -1,102 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.LruCache -import io.github.kdroidfilter.seforimlibrary.search.SearchFacets -import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor -import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc -import io.github.kdroidfilter.seforimlibrary.search.index.IndexPage -import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery -import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.SortField -import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding -import kotlinx.serialization.DeserializationStrategy -import kotlinx.serialization.builtins.ListSerializer -import kotlinx.serialization.builtins.LongArraySerializer -import kotlinx.serialization.builtins.nullable -import kotlinx.serialization.builtins.serializer - -/** - * How the calls reach the Java runtime Lucene runs in: one [BridgeRequest] in, its [BridgeResponse] out, both CBOR. - * Thread-safe: calls may overlap. - */ -fun interface BridgeTransport { - fun call(request: ByteArray): ByteArray -} - -private const val ANALYZE_CACHE_SIZE = 64 - -/** - * The [LuceneIndex] at [indexDir] where Lucene can't run in process: in the Java runtime an iOS app embeds. Lucene runs - * there with the same implementation as on the JVM and Android 15+ - * (`LuceneIndexBackend`, behind `IndexBridgeServer`); this side only carries the calls, through [transport]. - * - * @param cacheQueries see `LuceneIndexBackend` - */ -class BridgedLuceneIndex( - private val transport: BridgeTransport, - indexDir: String, - cacheQueries: Boolean = false, -) : LuceneIndex { - private val handle: Int = call(BridgeRequest.Open(indexDir, cacheQueries), Int.serializer()) - - // Analysis depends on the text alone: the same query is analyzed by its facets, its session and its highlights - private val analyzed = LruCache>(ANALYZE_CACHE_SIZE) - - override fun analyze(text: String): List = - analyzed[text] ?: call(BridgeRequest.Analyze(handle, text), ListSerializer(String.serializer())).also { analyzed[text] = it } - - override fun search( - query: IndexQuery, - limit: Int, - fields: Set, - sort: List?, - ): IndexPage = call(BridgeRequest.Search(handle, query, limit, fields, sort), IndexPage.serializer()) - - override fun openCursor( - query: IndexQuery, - fields: Set, - ): IndexCursor { - val cursor = call(BridgeRequest.OpenCursor(handle, query, fields), Int.serializer()) - return object : IndexCursor { - private var closed = false - - override fun next(limit: Int): IndexPage = call(BridgeRequest.CursorNext(cursor, limit), IndexPage.serializer()) - - override fun close() { - if (closed) return - closed = true - call(BridgeRequest.CloseCursor(cursor), null) - } - } - } - - override fun facets(query: IndexQuery): SearchFacets = call(BridgeRequest.Facets(handle, query), SearchFacets.serializer()) - - override fun collectLongs( - query: IndexQuery, - field: String, - ): LongArray = call(BridgeRequest.CollectLongs(handle, query, field), LongArraySerializer()) - - override fun count(query: IndexQuery): Int = call(BridgeRequest.Count(handle, query), Int.serializer()) - - override fun vectorEncoding(field: String): VectorEncoding? = - call(BridgeRequest.VectorEncodingOf(handle, field), VectorEncoding.serializer().nullable) - - override fun knn(request: KnnRequest): List = call(BridgeRequest.Knn(handle, request), ListSerializer(IndexDoc.serializer())) - - override fun close() { - call(BridgeRequest.Close(handle), null) - } - - private fun call( - request: BridgeRequest, - result: DeserializationStrategy?, - ): T { - val bytes = transport.call(BridgeCbor.encodeToByteArray(BridgeRequest.serializer(), request)) - val response = BridgeCbor.decodeFromByteArray(BridgeResponse.serializer(), bytes) - response.error?.let { throw IllegalStateException("Lucene failed in the Java runtime: $it") } - @Suppress("UNCHECKED_CAST") - return if (result == null) Unit as T else BridgeCbor.decodeFromByteArray(result, checkNotNull(response.result) { "No result" }) - } -} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt deleted file mode 100644 index fb2a3ea9..00000000 --- a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/EmbeddedJvm.kt +++ /dev/null @@ -1,193 +0,0 @@ -@file:OptIn(ExperimentalForeignApi::class) - -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.jni.JNIEnvVar -import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_CreateJavaVM -import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_GetCreatedJavaVMs -import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_OK -import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_TRUE -import io.github.kdroidfilter.seforimlibrary.search.jni.JNI_VERSION_21 -import io.github.kdroidfilter.seforimlibrary.search.jni.JavaVMInitArgs -import io.github.kdroidfilter.seforimlibrary.search.jni.JavaVMOption -import io.github.kdroidfilter.seforimlibrary.search.jni.JavaVMVar -import io.github.kdroidfilter.seforimlibrary.search.jni.jbyteVar -import io.github.kdroidfilter.seforimlibrary.search.jni.jclass -import io.github.kdroidfilter.seforimlibrary.search.jni.jintVar -import io.github.kdroidfilter.seforimlibrary.search.jni.jmethodID -import io.github.kdroidfilter.seforimlibrary.search.jni.jvalue -import io.github.kdroidfilter.seforimlibrary.search.jni.seforim_attach_current_thread -import kotlinx.atomicfu.locks.SynchronizedObject -import kotlinx.atomicfu.locks.synchronized -import kotlinx.cinterop.CPointer -import kotlinx.cinterop.CPointerVar -import kotlinx.cinterop.ExperimentalForeignApi -import kotlinx.cinterop.IntVar -import kotlinx.cinterop.addressOf -import kotlinx.cinterop.alloc -import kotlinx.cinterop.allocArray -import kotlinx.cinterop.convert -import kotlinx.cinterop.cstr -import kotlinx.cinterop.get -import kotlinx.cinterop.invoke -import kotlinx.cinterop.memScoped -import kotlinx.cinterop.pointed -import kotlinx.cinterop.ptr -import kotlinx.cinterop.reinterpret -import kotlinx.cinterop.usePinned -import kotlinx.cinterop.value - -/** - * The Java runtime the iOS app embeds: OpenJDK's mobile port (Zero, the interpreter, as iOS forbids generated code), - * linked statically into the app. Lucene runs there; the app calls it through [call]. - * - * The runtime reads its classes from `/lib` (its `java.home`, set by the VM itself); [classPath] - * names the jars of the search (see the `mobileJvmClasspath` task of SeforimLibrary's `search` module). - */ -class EmbeddedJvm private constructor( - private val vm: CPointer, -) { - private val lock = SynchronizedObject() - - // The static methods called so far, by "class.method" - private val methods = HashMap>() - - /** - * Calls the static method `byte[] method(byte[])` of [className] with [request], from the calling thread - * (attached to the runtime on its first call), and returns its result; throws when it threw. - */ - fun call( - className: String, - method: String, - request: ByteArray, - ): ByteArray = - withAttachedEnv { env -> - callStatic(env, className, method, request) - } - - private fun callStatic( - env: CPointer, - className: String, - method: String, - request: ByteArray, - ): ByteArray { - val functions = env.pointed.pointed!! - val (clazz, methodId) = staticMethod(env, className, method) - val array = functions.NewByteArray!!(env, request.size) ?: throwPending(env, "NewByteArray") - try { - if (request.isNotEmpty()) { - request.usePinned { pinned -> - functions.SetByteArrayRegion!!(env, array, 0, request.size, pinned.addressOf(0).reinterpret()) - } - } - val result = - memScoped { - val args = allocArray(1) - args[0].l = array - functions.CallStaticObjectMethodA!!(env, clazz, methodId, args) - } - if (functions.ExceptionCheck!!(env).toInt() != 0) throwPending(env, "$className.$method") - result ?: error("$className.$method returned null") - try { - val size = functions.GetArrayLength!!(env, result) - val bytes = ByteArray(size) - if (size > 0) { - bytes.usePinned { pinned -> - functions.GetByteArrayRegion!!(env, result, 0, size, pinned.addressOf(0).reinterpret()) - } - } - return bytes - } finally { - functions.DeleteLocalRef!!(env, result) - } - } finally { - functions.DeleteLocalRef!!(env, array) - } - } - - // Each class and method is resolved once: a global reference keeps the class - private fun staticMethod( - env: CPointer, - className: String, - method: String, - ): Pair = - synchronized(lock) { - methods["$className.$method"] ?: memScoped { - val functions = env.pointed.pointed!! - val local = functions.FindClass!!(env, className.replace('.', '/').cstr.getPointer(this)) ?: throwPending(env, className) - val global = functions.NewGlobalRef!!(env, local)!! - functions.DeleteLocalRef!!(env, local) - val methodId = functions.GetStaticMethodID!!(env, global, method.cstr.getPointer(this), "([B)[B".cstr.getPointer(this)) - ?: throwPending(env, "$className.$method") - (global to methodId).also { methods["$className.$method"] = it } - } - } - - // Attached once per thread, detached when the thread exits (see jni.def), or after the call when that can't be - private inline fun withAttachedEnv(block: (CPointer) -> T): T = - memScoped { - val detachAfter = alloc() - val env = seforim_attach_current_thread(vm, detachAfter.ptr) ?: error("Can't attach the thread to the Java runtime") - try { - block(env) - } finally { - if (detachAfter.value != 0) vm.pointed.pointed!!.DetachCurrentThread!!(vm) - } - } - - private fun throwPending( - env: CPointer, - what: String, - ): Nothing { - val functions = env.pointed.pointed!! - if (functions.ExceptionCheck!!(env).toInt() != 0) { - functions.ExceptionDescribe!!(env) - functions.ExceptionClear!!(env) - } - error("Java call failed: $what") - } - - companion object { - private val lock = SynchronizedObject() - private var instance: EmbeddedJvm? = null - - /** - * The Java runtime of this process, started on the first call with [classPath] and [options] (`-Xmx...`, - * `-D...`); a process has one, so later calls get it as it is. - */ - fun start( - classPath: List, - options: List = emptyList(), - ): EmbeddedJvm = - synchronized(lock) { - instance ?: (existing() ?: create(classPath, options)).also { instance = it } - } - - private fun existing(): EmbeddedJvm? = - memScoped { - val vms = allocArray>(1) - val count = alloc() - if (JNI_GetCreatedJavaVMs(vms, 1, count.ptr) == JNI_OK && count.value > 0) vms[0]?.let { EmbeddedJvm(it) } else null - } - - private fun create( - classPath: List, - options: List, - ): EmbeddedJvm = - memScoped { - val all = listOf("-Djava.class.path=" + classPath.joinToString(":")) + options - val vmOptions = allocArray(all.size) - all.forEachIndexed { i, option -> vmOptions[i].optionString = option.cstr.getPointer(this) } - val args = alloc() - args.version = JNI_VERSION_21 - args.nOptions = all.size - args.options = vmOptions - args.ignoreUnrecognized = JNI_TRUE.convert() - val vm = alloc>() - val env = alloc>() - val status = JNI_CreateJavaVM(vm.ptr, env.ptr.reinterpret(), args.ptr) - check(status == JNI_OK) { "JNI_CreateJavaVM failed: $status" } - EmbeddedJvm(vm.value!!) - } - } -} diff --git a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt b/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt deleted file mode 100644 index 7e5b535e..00000000 --- a/search/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JvmLuceneIndex.kt +++ /dev/null @@ -1,16 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex - -private const val SERVER_CLASS = "io.github.kdroidfilter.seforimlibrary.search.index.bridge.IndexBridgeServer" -private const val SERVER_METHOD = "call" - -/** The calls to the Java runtime the iOS app embeds: `IndexBridgeServer.call`, through JNI, from the calling thread. */ -fun EmbeddedJvm.bridgeTransport(): BridgeTransport = BridgeTransport { call(SERVER_CLASS, SERVER_METHOD, it) } - -/** The [LuceneIndex] at [indexDir] on iOS: Lucene runs in the Java runtime the app embeds ([jvm]). */ -fun JvmLuceneIndex( - jvm: EmbeddedJvm, - indexDir: String, - cacheQueries: Boolean = false, -): LuceneIndex = BridgedLuceneIndex(jvm.bridgeTransport(), indexDir, cacheQueries) diff --git a/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt b/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt index e99e60d9..6ac8f7f8 100644 --- a/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt +++ b/search/src/iosSimulatorArm64Test/kotlin/io/github/kdroidfilter/seforimlibrary/search/IosRealIndexParityTest.kt @@ -1,7 +1,6 @@ package io.github.kdroidfilter.seforimlibrary.search -import io.github.kdroidfilter.seforimlibrary.search.index.bridge.EmbeddedJvm -import io.github.kdroidfilter.seforimlibrary.search.index.bridge.JvmLuceneIndex +import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex import kotlinx.coroutines.runBlocking import kotlinx.io.buffered import kotlinx.io.files.Path @@ -12,8 +11,8 @@ import kotlin.time.measureTimedValue import kotlin.test.Test /** - * The search on iOS, end to end in the simulator: Lucene in the embedded Java runtime, everything else here. Dumps what - * it answers on a real index like RealIndexParityTest does on the JVM (SEFORIM_INDEX_DIR, SEFORIM_JVM_CLASSPATH and + * The search on iOS, end to end in the simulator: Lucene in the embedded Java runtime (through NJA), everything else + * here. Dumps what it answers on a real index like RealIndexParityTest does on the JVM (SEFORIM_INDEX_DIR and * SEFORIM_PARITY_OUT, passed by the iosSimulatorArm64Test task). Skipped without an index. */ class IosRealIndexParityTest { @@ -21,13 +20,10 @@ class IosRealIndexParityTest { fun dumpRealIndex() { val env = NSProcessInfo.processInfo.environment val indexDir = env["SEFORIM_INDEX_DIR"] as String? ?: return - val classPath = (env["SEFORIM_JVM_CLASSPATH"] as String).split(':') - val (jvm, startup) = measureTimedValue { EmbeddedJvm.start(classPath, listOf("-Xmx768m") + (env["SEFORIM_JVM_OPTIONS"] as String?)?.split(' ').orEmpty()) } - println("Java runtime started in $startup") val dictionary = indexDir.substringBeforeLast('/') + "/lexical.db" val engine = LexicalSearchEngine( - index = JvmLuceneIndex(jvm, indexDir), + index = LuceneIndex(indexDir), dictionary = lazy { MagicDictionaryIndex.load(HebrewTextUtils::normalizeHebrew, dictionary) }, ) engine.use { diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index 2e94b877..97fbbd14 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -2,16 +2,13 @@ package io.github.kdroidfilter.seforimlibrary.search import co.touchlab.kermit.Logger import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend -import org.apache.lucene.analysis.Analyzer -import org.apache.lucene.analysis.standard.StandardAnalyzer import java.nio.file.Path private val logger = Logger.withTag("LuceneSearchEngine") /** - * The desktop's [SearchEngine]: the common [LexicalSearchEngine] over the Lucene index at [indexDir], read in process, - * with the lexical dictionary found beside it (or at [dictionaryPath], `-DmagicDict`, `SEFORIM_MAGIC_DICT`). + * The desktop's [SearchEngine]: the common [LexicalSearchEngine] over the Lucene index at [indexDir] (the façade's, in + * this JVM), with the lexical dictionary found beside it (or at [dictionaryPath], `-DmagicDict`, `SEFORIM_MAGIC_DICT`). */ class LuceneSearchEngine private constructor( /** The Lucene index this engine reads, for the searches that share it (the semantic search's vectors). */ @@ -31,9 +28,8 @@ class LuceneSearchEngine private constructor( constructor( indexDir: Path, snippetProvider: SnippetProvider? = null, - analyzer: Analyzer = StandardAnalyzer(), dictionaryPath: Path? = null, - ) : this(LuceneIndexBackend(indexDir, analyzer = analyzer), indexDir, snippetProvider, dictionaryPath) + ) : this(LuceneIndex(indexDir.toString()), indexDir, snippetProvider, dictionaryPath) suspend fun searchAllText( rawQuery: String, diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt deleted file mode 100644 index bae786a5..00000000 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServer.kt +++ /dev/null @@ -1,160 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.SearchFacets -import io.github.kdroidfilter.seforimlibrary.search.index.IndexCursor -import io.github.kdroidfilter.seforimlibrary.search.index.IndexDoc -import io.github.kdroidfilter.seforimlibrary.search.index.IndexPage -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndex -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend -import io.github.kdroidfilter.seforimlibrary.search.index.VectorEncoding -import kotlinx.serialization.KSerializer -import kotlinx.serialization.builtins.ListSerializer -import kotlinx.serialization.builtins.LongArraySerializer -import kotlinx.serialization.builtins.nullable -import kotlinx.serialization.builtins.serializer -import java.io.BufferedInputStream -import java.io.BufferedOutputStream -import java.io.DataInputStream -import java.io.DataOutputStream -import java.io.EOFException -import java.io.FileDescriptor -import java.io.FileOutputStream -import java.io.PrintStream -import java.nio.file.Path -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -/** - * The Java side of the bridge (see `BridgedLuceneIndex`): runs the calls of an app's [LuceneIndex]es on - * [LuceneIndexBackend]s, where Lucene can't run in the app itself: the iOS app calls [call] through JNI, from any - * thread, in the Java runtime it embeds; an Android app runs [main] in a Java process of its own. - */ -object IndexBridgeServer { - /** - * Serves the calls of standard input on standard output, until the input closes: each call is a frame (its id, its - * length, the request), each answer a frame with the same id; calls run concurrently, answered as they complete. - * Anything else printed goes to standard error, which keeps the output for the frames alone. - */ - @JvmStatic - fun main(args: Array) { - val frames = DataOutputStream(BufferedOutputStream(FileOutputStream(FileDescriptor.out))) - System.setOut(PrintStream(FileOutputStream(FileDescriptor.err), true)) - val input = DataInputStream(BufferedInputStream(System.`in`)) - val workers = - Executors.newFixedThreadPool(maxOf(2, Runtime.getRuntime().availableProcessors())) { task -> - Thread(task, "bridge-${threadIds.incrementAndGet()}").apply { isDaemon = true } - } - while (true) { - val id: Int - val request: ByteArray - try { - id = input.readInt() - request = ByteArray(input.readInt()).also { input.readFully(it) } - } catch (_: EOFException) { - break - } - workers.execute { - val response = call(request) - synchronized(frames) { - frames.writeInt(id) - frames.writeInt(response.size) - frames.write(response) - frames.flush() - } - } - } - // The calls already read are answered before the process ends - workers.shutdown() - workers.awaitTermination(1, TimeUnit.MINUTES) - } - - private val threadIds = AtomicInteger() - private val ids = AtomicInteger() - private val indexes = ConcurrentHashMap() - private val cursors = ConcurrentHashMap() - - // A cursor with the index it reads, closed with it. Its calls take turns: a page being read when the index - // closes finishes before its searcher is released - private class OpenCursor( - val index: Int, - private val cursor: IndexCursor, - ) { - private var closed = false - - @Synchronized - fun next(limit: Int): IndexPage { - check(!closed) { "Closed cursor" } - return cursor.next(limit) - } - - @Synchronized - fun close() { - if (closed) return - closed = true - cursor.close() - } - } - - /** Runs the call [request] (a [BridgeRequest] in CBOR) and returns its [BridgeResponse] (CBOR). */ - @JvmStatic - fun call(request: ByteArray): ByteArray { - val response = - try { - BridgeResponse(result = run(BridgeCbor.decodeFromByteArray(BridgeRequest.serializer(), request))) - } catch (e: Throwable) { - BridgeResponse(error = e.stackTraceToString()) - } - return BridgeCbor.encodeToByteArray(BridgeResponse.serializer(), response) - } - - private fun run(request: BridgeRequest): ByteArray? = - when (request) { - is BridgeRequest.Open -> { - val id = ids.incrementAndGet() - indexes[id] = LuceneIndexBackend(Path.of(request.indexDir), cacheQueries = request.cacheQueries) - encode(Int.serializer(), id) - } - is BridgeRequest.Close -> { - // The cursors left open hold the index's reader: closed first - cursors.filterValues { it.index == request.index }.keys.forEach { cursors.remove(it)?.close() } - indexes.remove(request.index)?.close() - null - } - is BridgeRequest.Analyze -> - encode(ListSerializer(String.serializer()), index(request.index).analyze(request.text)) - is BridgeRequest.Search -> - page(index(request.index).search(request.query, request.limit, request.fields, request.sort)) - is BridgeRequest.OpenCursor -> { - val id = ids.incrementAndGet() - cursors[id] = OpenCursor(request.index, index(request.index).openCursor(request.query, request.fields)) - encode(Int.serializer(), id) - } - is BridgeRequest.CursorNext -> - page(cursors[request.cursor]?.next(request.limit) ?: error("Unknown cursor ${request.cursor}")) - is BridgeRequest.CloseCursor -> { - cursors.remove(request.cursor)?.close() - null - } - is BridgeRequest.Facets -> - encode(SearchFacets.serializer(), index(request.index).facets(request.query)) - is BridgeRequest.CollectLongs -> - encode(LongArraySerializer(), index(request.index).collectLongs(request.query, request.field)) - is BridgeRequest.Count -> - encode(Int.serializer(), index(request.index).count(request.query)) - is BridgeRequest.VectorEncodingOf -> - encode(VectorEncoding.serializer().nullable, index(request.index).vectorEncoding(request.field)) - is BridgeRequest.Knn -> - encode(ListSerializer(IndexDoc.serializer()), index(request.index).knn(request.request)) - } - - private fun index(id: Int): LuceneIndex = indexes[id] ?: error("Unknown index $id") - - private fun page(page: IndexPage) = encode(IndexPage.serializer(), page) - - private fun encode( - serializer: KSerializer, - value: T, - ): ByteArray = BridgeCbor.encodeToByteArray(serializer, value) -} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt deleted file mode 100644 index 36f0e087..00000000 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/BridgedLuceneIndexTest.kt +++ /dev/null @@ -1,90 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery -import io.github.kdroidfilter.seforimlibrary.search.index.KnnRequest -import io.github.kdroidfilter.seforimlibrary.search.index.LuceneIndexBackend -import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.document.Document -import org.apache.lucene.document.Field -import org.apache.lucene.document.StoredField -import org.apache.lucene.document.StringField -import org.apache.lucene.document.TextField -import org.apache.lucene.index.IndexWriter -import org.apache.lucene.index.IndexWriterConfig -import org.apache.lucene.store.FSDirectory -import java.nio.file.Files -import java.nio.file.Path -import kotlin.io.path.ExperimentalPathApi -import kotlin.io.path.deleteRecursively -import kotlin.test.AfterTest -import kotlin.test.Test -import kotlin.test.assertContentEquals -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertNull - -/** The bridge's calls end to end, CBOR both ways: the bridged index answers as the in-process one. */ -class BridgedLuceneIndexTest { - private val texts = listOf("בראשית ברא אלהים", "ויאמר אלהים יהי אור", "ויהי ערב ויהי בקר") - - private val indexDir: Path = - Files.createTempDirectory("bridged_index").also { dir -> - IndexWriter(FSDirectory.open(dir), IndexWriterConfig(StandardAnalyzer())).use { writer -> - texts.forEachIndexed { i, text -> - writer.addDocument( - Document().apply { - add(StringField("type", "line", Field.Store.NO)) - add(TextField("text", text, Field.Store.YES)) - add(StoredField("line_id", 100L + i)) - }, - ) - } - } - } - - private val local = LuceneIndexBackend(indexDir) - private val bridged = BridgedLuceneIndex(BridgeTransport { IndexBridgeServer.call(it) }, indexDir.toString()) - - @OptIn(ExperimentalPathApi::class) - @AfterTest - fun cleanUp() { - bridged.close() - local.close() - indexDir.deleteRecursively() - } - - private val elohim = IndexQuery.Term("text", "אלהים") - - @Test - fun `pages carry Hebrew text, ids and scores`() { - val fields = setOf("text", "line_id") - val expected = local.search(elohim, 10, fields) - val actual = bridged.search(elohim, 10, fields) - - assertEquals(expected.totalHits, actual.totalHits) - assertEquals(expected.docs.map { it.strings["text"] }, actual.docs.map { it.strings["text"] }) - assertEquals(expected.docs.map { it.longs["line_id"] }, actual.docs.map { it.longs["line_id"] }) - assertEquals(expected.docs.map { it.score }, actual.docs.map { it.score }) - } - - @Test - fun `cursors, analysis, counts and ids cross the bridge`() { - assertEquals(local.analyze(texts[1]), bridged.analyze(texts[1])) - assertEquals(2, bridged.count(elohim)) - assertContentEquals(local.collectLongs(elohim, "line_id").sorted(), bridged.collectLongs(elohim, "line_id").sorted()) - bridged.openCursor(IndexQuery.MatchAll, setOf("line_id")).use { cursor -> - assertEquals(2, cursor.next(2).docs.size) - assertEquals(1, cursor.next(2).docs.size) - } - } - - @Test - fun `a null result crosses the bridge`() { - assertNull(bridged.vectorEncoding("embedding")) - } - - @Test - fun `a KNN vector with a NaN is refused`() { - assertFailsWith { KnnRequest("embedding", 1, emptySet(), floatVector = floatArrayOf(0f, Float.NaN)) } - } -} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt deleted file mode 100644 index 0981f766..00000000 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/IndexBridgeServerTest.kt +++ /dev/null @@ -1,72 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery -import kotlinx.serialization.builtins.serializer -import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.document.Document -import org.apache.lucene.document.Field -import org.apache.lucene.document.StringField -import org.apache.lucene.index.IndexWriter -import org.apache.lucene.index.IndexWriterConfig -import org.apache.lucene.store.FSDirectory -import java.nio.file.Files -import java.nio.file.Path -import kotlin.io.path.ExperimentalPathApi -import kotlin.io.path.deleteRecursively -import kotlin.test.AfterTest -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertNull -import kotlin.test.assertTrue - -class IndexBridgeServerTest { - private val indexDir: Path = - Files.createTempDirectory("bridge_index").also { dir -> - IndexWriter(FSDirectory.open(dir), IndexWriterConfig(StandardAnalyzer())).use { writer -> - repeat(3) { writer.addDocument(Document().apply { add(StringField("type", "line", Field.Store.YES)) }) } - } - } - - @OptIn(ExperimentalPathApi::class) - @AfterTest - fun cleanUp() { - indexDir.deleteRecursively() - } - - private fun call(request: BridgeRequest): BridgeResponse = - BridgeCbor.decodeFromByteArray( - BridgeResponse.serializer(), - IndexBridgeServer.call(BridgeCbor.encodeToByteArray(BridgeRequest.serializer(), request)), - ) - - private fun callInt(request: BridgeRequest): Int = BridgeCbor.decodeFromByteArray(Int.serializer(), assertNotNull(call(request).result)) - - @Test - fun `a cursor reads its pages until closed`() { - val index = callInt(BridgeRequest.Open(indexDir.toString(), cacheQueries = false)) - val cursor = callInt(BridgeRequest.OpenCursor(index, IndexQuery.MatchAll, emptySet())) - - assertNull(call(BridgeRequest.CursorNext(cursor, 2)).error) - call(BridgeRequest.CloseCursor(cursor)) - assertTrue(call(BridgeRequest.CursorNext(cursor, 2)).error.orEmpty().contains("Unknown cursor")) - call(BridgeRequest.Close(index)) - } - - @Test - fun `closing an index closes the cursors left open on it`() { - val index = callInt(BridgeRequest.Open(indexDir.toString(), cacheQueries = false)) - val other = callInt(BridgeRequest.Open(indexDir.toString(), cacheQueries = false)) - val cursor = callInt(BridgeRequest.OpenCursor(index, IndexQuery.MatchAll, emptySet())) - val otherCursor = callInt(BridgeRequest.OpenCursor(other, IndexQuery.MatchAll, emptySet())) - - call(BridgeRequest.Close(index)) - - assertTrue(call(BridgeRequest.CursorNext(cursor, 2)).error.orEmpty().contains("Unknown cursor")) - // Another index's cursors are left alone - val page = call(BridgeRequest.CursorNext(otherCursor, 5)) - assertNull(page.error) - call(BridgeRequest.Close(other)) - assertEquals(null, call(BridgeRequest.Close(other)).error) - } -} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt deleted file mode 100644 index f4a0e7d3..00000000 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/index/bridge/JavaProcessTransportTest.kt +++ /dev/null @@ -1,74 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search.index.bridge - -import io.github.kdroidfilter.seforimlibrary.search.index.IndexQuery -import org.apache.lucene.analysis.standard.StandardAnalyzer -import org.apache.lucene.document.Document -import org.apache.lucene.document.Field -import org.apache.lucene.document.StringField -import org.apache.lucene.index.IndexWriter -import org.apache.lucene.index.IndexWriterConfig -import org.apache.lucene.store.FSDirectory -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import kotlin.io.path.ExperimentalPathApi -import kotlin.io.path.deleteRecursively -import kotlin.test.AfterTest -import kotlin.test.Test -import kotlin.test.assertEquals - -/** - * The Android path of the bridge, on the desktop: a Java process serving [IndexBridgeServer.main], started by a script - * with the arguments of the Android launcher (`
[Java options...]`). - */ -class JavaProcessTransportTest { - private val dir: Path = Files.createTempDirectory("process_bridge") - private val indexDir: Path = - dir.resolve("index").also { index -> - IndexWriter(FSDirectory.open(index), IndexWriterConfig(StandardAnalyzer())).use { writer -> - repeat(3) { writer.addDocument(Document().apply { add(StringField("type", "line", Field.Store.YES)) }) } - } - } - private val launcher: File = - dir.resolve("launcher.sh").toFile().apply { - writeText("#!/bin/sh\njavaHome=$1; classPath=$2; main=$3; shift 3\nexec \"\$javaHome/bin/java\" -cp \"\$classPath\" \"$@\" \"\$main\"\n") - setExecutable(true) - } - - @OptIn(ExperimentalPathApi::class) - @AfterTest - fun cleanUp() { - dir.deleteRecursively() - } - - private fun transport() = - JavaProcessTransport( - launcher = launcher, - javaHome = File(System.getProperty("java.home")), - classPath = System.getProperty("java.class.path").split(File.pathSeparator).map(::File), - ) - - @Test - fun `the index answers through the Java process`() { - transport().use { transport -> - BridgedLuceneIndex(transport, indexDir.toString()).use { index -> - assertEquals(3, index.count(IndexQuery.MatchAll)) - assertEquals(2, index.search(IndexQuery.MatchAll, limit = 2, fields = setOf("type")).docs.size) - } - } - } - - @Test - fun `a process that died is started again on the next call`() { - transport().use { transport -> - BridgedLuceneIndex(transport, indexDir.toString()).use { index -> - assertEquals(3, index.count(IndexQuery.MatchAll)) - } - // Closing the transport ends its process; the next index starts a new one - transport.close() - BridgedLuceneIndex(transport, indexDir.toString()).use { index -> - assertEquals(3, index.count(IndexQuery.MatchAll)) - } - } - } -} diff --git a/search/src/nativeInterop/cinterop/jni.def b/search/src/nativeInterop/cinterop/jni.def deleted file mode 100644 index 9501d5e5..00000000 --- a/search/src/nativeInterop/cinterop/jni.def +++ /dev/null @@ -1,35 +0,0 @@ -# The JNI of the Java runtime the iOS app embeds (OpenJDK mobile, statically linked into the app: see IOS_RUNTIME.md) -headers = jni.h -package = io.github.kdroidfilter.seforimlibrary.search.jni - ---- - -#include - -// A thread attached by seforim_attach_current_thread is detached when it exits, from this destructor: in plain C, as -// the Kotlin runtime of an exiting thread may already be gone -static pthread_key_t seforim_detach_key; -static int seforim_detach_key_ok = 0; -static pthread_once_t seforim_detach_once = PTHREAD_ONCE_INIT; - -static void seforim_detach_current_thread(void *vm) { - (*(JavaVM *) vm)->DetachCurrentThread((JavaVM *) vm); -} - -static void seforim_create_detach_key(void) { - seforim_detach_key_ok = pthread_key_create(&seforim_detach_key, seforim_detach_current_thread) == 0; -} - -// The JNIEnv of the calling thread, attached to vm (as a daemon) if it isn't yet; NULL when it can't be. *detach_after -// is set when the caller must detach the thread itself once done: no thread-exit destructor could be registered -static JNIEnv *seforim_attach_current_thread(JavaVM *vm, int *detach_after) { - JNIEnv *env = NULL; - *detach_after = 0; - jint status = (*vm)->GetEnv(vm, (void **) &env, JNI_VERSION_21); - if (status == JNI_OK) return env; - if (status != JNI_EDETACHED) return NULL; - if ((*vm)->AttachCurrentThreadAsDaemon(vm, (void **) &env, NULL) != JNI_OK) return NULL; - pthread_once(&seforim_detach_once, seforim_create_detach_key); - if (!seforim_detach_key_ok || pthread_setspecific(seforim_detach_key, vm) != 0) *detach_after = 1; - return env; -} diff --git a/settings.gradle.kts b/settings.gradle.kts index be36cb50..b1a700dd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,9 @@ rootProject.name = "SeforimLibrary" pluginManagement { + // Nucleus JVM Access from its sources beside (-Pnja.local=), before a release has what the search + // needs: its plugins here, its runtimes below + providers.gradleProperty("nja.local").orNull?.let { includeBuild("$it/plugin-build") } repositories { google { content { @@ -69,6 +72,7 @@ dependencyResolutionManagement { include(":core") include(":dao") include(":search") +include(":search-facade") include(":cli") include(":catalog") include(":searchindex") @@ -88,3 +92,6 @@ project(":generator-common").projectDir = file("generator/common") project(":reflinks").projectDir = file("generator/reflinks") includeBuild("SeforimMagicIndexer") + +// Nucleus JVM Access's runtimes from its sources (see pluginManagement) +providers.gradleProperty("nja.local").orNull?.let { includeBuild("$it/runtime") } From cc1198daf3f0112d515817b12e0d3f218c89cefa Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 17 Oct 2026 19:05:56 +0300 Subject: [PATCH 09/14] build: Nucleus JVM Access 0.2.0 from its release The option to build it from its sources (-Pnja.local) goes: 0.2.0 has what the search needs. Signed-off-by: Elie Gambache --- JVM_RUNTIME.md | 5 ----- settings.gradle.kts | 6 ------ 2 files changed, 11 deletions(-) diff --git a/JVM_RUNTIME.md b/JVM_RUNTIME.md index b11b9649..10e68774 100644 --- a/JVM_RUNTIME.md +++ b/JVM_RUNTIME.md @@ -49,11 +49,6 @@ NJA reads them (`nucleusJvm.jdkMobile`); moving to a newer runtime is changing ` - The VM's options are `nucleusJvm.jvmOptions` (a 128 MB heap: about 140 MB of memory on Android, 160 MB on iOS, on real data). -## Building with NJA's sources - -Until a release of NJA has what the search needs (JVM targets, primitive arrays, sealed value hierarchies), -`-Pnja.local=` builds its plugins and runtimes from its sources (settings.gradle.kts). - ## Tests ```bash diff --git a/settings.gradle.kts b/settings.gradle.kts index b1a700dd..ac176c8e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,9 +1,6 @@ rootProject.name = "SeforimLibrary" pluginManagement { - // Nucleus JVM Access from its sources beside (-Pnja.local=), before a release has what the search - // needs: its plugins here, its runtimes below - providers.gradleProperty("nja.local").orNull?.let { includeBuild("$it/plugin-build") } repositories { google { content { @@ -92,6 +89,3 @@ project(":generator-common").projectDir = file("generator/common") project(":reflinks").projectDir = file("generator/reflinks") includeBuild("SeforimMagicIndexer") - -// Nucleus JVM Access's runtimes from its sources (see pluginManagement) -providers.gradleProperty("nja.local").orNull?.let { includeBuild("$it/runtime") } From 33ea06e3e5d161b46e5fa20d6d05b6ace5d0fefc Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 17 Oct 2026 19:31:33 +0300 Subject: [PATCH 10/14] build: jdk-mobile's runtimes downloaded by Nucleus JVM Access 0.3.0 NJA 0.3.0 downloads the runtimes itself (jdk-mobile's latest release, checked against its .sha256, in Gradle's cache) and wires the tasks that read them: search's own download, checksums and staging go. Signed-off-by: Elie Gambache --- JVM_RUNTIME.md | 7 ++--- gradle/libs.versions.toml | 2 +- search/build.gradle.kts | 61 +-------------------------------------- 3 files changed, 5 insertions(+), 65 deletions(-) diff --git a/JVM_RUNTIME.md b/JVM_RUNTIME.md index 10e68774..060dc385 100644 --- a/JVM_RUNTIME.md +++ b/JVM_RUNTIME.md @@ -32,10 +32,9 @@ NJA from the façade's sources) LuceneIndexBackend (inter The mobile JVMs are jdk-mobile's: a stable OpenJDK (27, `openjdk/jdk27u`'s GA releases) with openjdk/mobile's port, built by the CI of [NucleusFramework/openjdk-mobile](https://github.com/NucleusFramework/openjdk-mobile) and published as releases named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.3`: `jdk-27-ga`, the -JDK's build 35, its third release there). `search` downloads the four runtimes (`android-aarch64`, `android-x64`, -`ios-simulator-aarch64`, `ios-device-aarch64`), checks their SHA-256 and stages them in `search/build/jdkMobile`, where -NJA reads them (`nucleusJvm.jdkMobile`); moving to a newer runtime is changing `jdkMobileVersion`, `jdkMobileBuild`, -`jdkMobileRevision` and `jdkMobileSha256`, then running the tests below. +JDK's build 35). NJA downloads the latest one (the runtimes `search` needs: arm64-v8a and x86_64 Android, iOS), +checks them against their `.sha256`, keeps them in Gradle's cache and extracts them into `search/build/nja/jdkMobile`; +`nucleusJvm.jdkMobileRelease` pins one. - **Android**: NJA links HotSpot and the JDK's natives into `libnja.so` (one per ABI: arm64-v8a, x86_64) and puts the runtime (`java.home` per ABI, the façade's class path, the VM's options) in `search`'s assets, extracted once per diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1a8cbac1..e85752e8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ filekitCore = "0.13.0" kotlin = "2.4.20" # Nucleus JVM Access: Lucene on a JVM on every platform (the search's façade, search-facade) -nja = "0.2.0" +nja = "0.3.0" agp = "9.1.1" jvmToolchain = "25" lucene = "10.5.2" diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 12642dd1..11eb41a8 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -1,7 +1,4 @@ import org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeSimulatorTest -import java.net.URI -import java.security.DigestInputStream -import java.security.MessageDigest plugins { alias(libs.plugins.multiplatform) @@ -12,58 +9,6 @@ plugins { group = "io.github.kdroidfilter.seforimlibrary" -// The Java runtimes Lucene runs in on the mobile platforms (see JVM_RUNTIME.md): jdk-mobile's releases, built by the CI -// of NucleusFramework/openjdk-mobile, downloaded and checked: OpenJDK 27 GA (the JDK's build 35) with openjdk/mobile's -// port, its third release there (jdk-mobile-27b35.3: the Android runtimes' static libraries, for NJA's in-process VM) -val jdkMobileVersion = "27" -val jdkMobileBuild = "35" -val jdkMobileRevision = "3" -val jdkMobileRelease = "jdk-mobile-${jdkMobileVersion}b$jdkMobileBuild.$jdkMobileRevision" -val jdkMobileSha256 = - mapOf( - "android-aarch64" to "625d0dcdb36f1e935b06ae5720169e77145500c23987a38e85600a4093ea4396", - "android-x64" to "e6f69c83dd4c04a1822a53815d1a961d510b5b471c96313ee39e7a257148c654", - "ios-simulator-aarch64" to "14043e85df2a019cba5ad55ac19979f4cacbfbd18ada8b2aeb895b52be0d2174", - "ios-device-aarch64" to "e38dc41f3a85b2f0263692030e03228b4d3127d1da07f52a77215b5ce7d962d6", - ) - -// build/jdkMobile/, the layout NJA reads (nucleusJvm.jdkMobile): static/, include/ and lib/ (java.home) -val jdkMobileDir = layout.buildDirectory.dir("jdkMobile") -val stageJdkMobile = - jdkMobileSha256.map { (target, sha256) -> - val asset = "jdk-mobile-$jdkMobileVersion-$target-b$jdkMobileBuild.$jdkMobileRevision.tar.gz" - val archive = layout.buildDirectory.file("jdkMobileDownloads/$asset") - val suffix = target.split("-").joinToString("") { it.replaceFirstChar(Char::uppercase) } - val download = - tasks.register("downloadJdkMobile$suffix") { - description = "Downloads jdk-mobile's runtime for $target ($jdkMobileRelease)." - val url = "https://github.com/NucleusFramework/openjdk-mobile/releases/download/$jdkMobileRelease/$asset" - inputs.property("url", url) - inputs.property("sha256", sha256) - outputs.file(archive) - doLast { - val file = archive.get().asFile.apply { parentFile.mkdirs() } - val digest = MessageDigest.getInstance("SHA-256") - DigestInputStream(URI(url).toURL().openStream(), digest).use { input -> - file.outputStream().use { input.copyTo(it) } - } - val actual = digest.digest().joinToString("") { "%02x".format(it) } - if (actual != sha256) { - file.delete() - error("Unexpected $asset from $url (SHA-256 $actual)") - } - } - } - tasks.register("stageJdkMobile$suffix") { - description = "Extracts jdk-mobile's runtime for $target." - dependsOn(download) - from(tarTree(archive)) - into(jdkMobileDir.map { it.dir(target) }) - // Some of the JDK's files are read-only: the next sync couldn't replace them - filePermissions { user { write = true } } - } - } - kotlin { jvmToolchain(libs.versions.jvmToolchain.get().toInt()) @@ -132,7 +77,7 @@ kotlin { // apps embed (iOS: Zero linked into the app; Android: HotSpot in the app's process), and common code uses its proxies nucleusJvm { facade(project(":search-facade")) - jdkMobile.set(jdkMobileDir) + // The runtimes are jdk-mobile's latest release, which NJA downloads (see JVM_RUNTIME.md) androidAbis.set(setOf("arm64-v8a", "x86_64")) // Lucene maps its files with Panama (madvise): native access, without the warning jvmOptions.addAll("-Xmx128m", "--enable-native-access=ALL-UNNAMED") @@ -140,10 +85,6 @@ nucleusJvm { providers.gradleProperty("luceneQueryCache").orNull?.let { jvmOptions.add("-Dseforim.lucene.queryCache=$it") } } -// NJA reads the runtimes from jdkMobile, and the iOS binaries link them: staged first -tasks.matching { it.name.startsWith("nja") }.configureEach { dependsOn(stageJdkMobile) } -tasks.withType().configureEach { dependsOn(stageJdkMobile) } - tasks.withType().configureEach { device.set(providers.gradleProperty("seforim.iosSimulator").orElse("iPhone 17 Pro")) val indexDir = providers.gradleProperty("indexDir") From d313b0dad5f271cf428193eedf3d195806dc331b Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 17 Oct 2026 21:58:35 +0300 Subject: [PATCH 11/14] fix(search): read the index through NIO on Android Mapped (Lucene's MMapDirectory on HotSpot), the index's pages, the vectors above all, counted in the app's memory (1.4 GB of PSS at a hybrid search's peak on an Android 15 emulator, against 0.45 GB with Lucene on ART), which Android looks at when it kills apps; read through NIO, as the desktop's native image and iOS already do, 0.55 GB. The backend detects Android itself (HotSpot runs in the app's process, with its environment: ANDROID_ROOT). Signed-off-by: Elie Gambache --- .../seforimlibrary/search/lucene/LuceneIndexBackend.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt index 234a6bee..f5d5ed92 100644 --- a/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt +++ b/search-facade/src/main/kotlin/io/github/kdroidfilter/seforimlibrary/search/lucene/LuceneIndexBackend.kt @@ -514,8 +514,9 @@ internal class LuceneIndexBackend( companion object { /** - * The directory of [path]: memory-mapped where Lucene can (HotSpot), read through NIO where its Panama-based - * mapping isn't available (GraalVM native image, the iOS Java runtime). + * The directory of [path]: memory-mapped on the desktop's JVM, read through NIO where Lucene's Panama-based + * mapping isn't available (GraalVM native image, the iOS Java runtime) or not wanted: on Android, the mapped + * index (the vectors above all) would count in the app's memory, which Android looks at when it kills apps. */ fun openDirectory(path: Path): Directory = if (mmapAvailable) FSDirectory.open(path) else NIOFSDirectory(path) @@ -523,6 +524,8 @@ internal class LuceneIndexBackend( private val mmapAvailable: Boolean by lazy { System.getProperty("org.graalvm.nativeimage.imagecode") == null && + // Android: HotSpot runs in the app's process, with its environment + System.getenv("ANDROID_ROOT") == null && System.getProperty("os.name") != "Darwin" && System.getProperty("java.vm.name")?.contains("Zero") != true } From 6f7ee128826ff088b07de4e7a7ccc1af201ef72a Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 00:10:03 +0300 Subject: [PATCH 12/14] feat(search): Lucene compiled into a GraalVM native image on iOS, through Nucleus JVM Access 0.4.0 Zero searched about 8x slower than the image (a first page 1.2 s against 140 ms in the simulator). Lucene's reachability metadata comes with the facade's jar, traced on the JVM tests with -PnativeImageAgent. Android keeps HotSpot. Signed-off-by: Elie Gambache --- JVM_RUNTIME.md | 31 +- gradle/libs.versions.toml | 2 +- search-facade/native-image/caller-filter.json | 7 + .../search-facade/reachability-metadata.json | 327 ++++++++++++++++++ search/build.gradle.kts | 18 +- 5 files changed, 369 insertions(+), 16 deletions(-) create mode 100644 search-facade/native-image/caller-filter.json create mode 100644 search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json diff --git a/JVM_RUNTIME.md b/JVM_RUNTIME.md index 060dc385..1974ef1c 100644 --- a/JVM_RUNTIME.md +++ b/JVM_RUNTIME.md @@ -17,8 +17,8 @@ NJA from the façade's sources) LuceneIndexBackend (inter | | Desktop (JVM) | Android 10+ | iOS | |---|---|---|---| -| The JVM | the app's own | HotSpot (server VM, JIT), in the app's process next to ART | Zero (interpreter: iOS forbids generated code), linked into the app | -| A call | a plain call into NJA's dispatcher, on the calling thread | handed to a HotSpot thread paired with the calling one | JNI, on the calling thread | +| The JVM | the app's own | HotSpot (server VM, JIT), in the app's process next to ART | none: the façade compiled ahead of time into a GraalVM native image, linked into the app (iOS forbids generated code) | +| A call | a plain call into NJA's dispatcher, on the calling thread | handed to a HotSpot thread paired with the calling one | a call into the image's entry point, on the calling thread | - `search-facade` holds the façade: its public API (`LuceneIndex`, `IndexCursor` and the value types: data classes, enums, the sealed `IndexQuery`) is what NJA exports; `LuceneIndexBackend` is internal. @@ -29,24 +29,33 @@ NJA from the façade's sources) LuceneIndexBackend (inter ## The runtimes -The mobile JVMs are jdk-mobile's: a stable OpenJDK (27, `openjdk/jdk27u`'s GA releases) with openjdk/mobile's port, +`nucleusJvm.iosRuntime` is `GRAALVM`: Zero, jdk-mobile's interpreter, searched about 8× slower than the native image +(a first page 1.2 s against 140 ms in the simulator, on real data). Android keeps HotSpot: an Android native image +builds on Linux only, and HotSpot is the fastest once warm. + +- **iOS**: NJA downloads [graalvm-mobile](https://github.com/NucleusFramework/graalvm-mobile)'s latest kit (GraalVM + CE's static libraries, for iOS) and the GraalVM CE release it names, for the build host, and compiles the façade + into `search/build/nja/nativeImage//nja.o` (`njaNativeImage`). The binaries link it with the kit's + libraries (`-export_dynamic`: NJA's client finds the image's entry points at run time): nothing to bundle. Lucene's + reflection and resources are in the façade's jar (`META-INF/native-image/.../reachability-metadata.json`), traced on + the JVM tests with the real index: `./gradlew :search:jvmTest -PindexDir= + -PnativeImageAgent=` (then drop what the tests alone reach). The image reads the index through NIO. + For `search`'s own tests NJA does it all; an app links as Zayit's `searchkit` shows (`assembleIosJavaRuntimeDevice`: + `static/`, the object and the kit's libraries). + +Android's JVM is jdk-mobile's: a stable OpenJDK (27, `openjdk/jdk27u`'s GA releases) with openjdk/mobile's port, built by the CI of [NucleusFramework/openjdk-mobile](https://github.com/NucleusFramework/openjdk-mobile) and published as releases named like the JBR's, `jdk-mobile-b.` (`jdk-mobile-27b35.3`: `jdk-27-ga`, the -JDK's build 35). NJA downloads the latest one (the runtimes `search` needs: arm64-v8a and x86_64 Android, iOS), +JDK's build 35). NJA downloads the latest one (the runtimes `search` needs: arm64-v8a and x86_64 Android), checks them against their `.sha256`, keeps them in Gradle's cache and extracts them into `search/build/nja/jdkMobile`; `nucleusJvm.jdkMobileRelease` pins one. - **Android**: NJA links HotSpot and the JDK's natives into `libnja.so` (one per ABI: arm64-v8a, x86_64) and puts the runtime (`java.home` per ABI, the façade's class path, the VM's options) in `search`'s assets, extracted once per version on the device: an app depending on `search` gets both through the library, with nothing to configure. -- **iOS**: the binaries link jdk-mobile's static libraries (`-force_load`: the runtime finds its natives by name, - `-export_dynamic`) and carry NJA's bundle next to the executable: `lib/` (the VM's `java.home`, which it sets to - `/lib`) and `nja/` (the class path and the VM's options). For `search`'s own tests NJA does both; an - app does them as Zayit's `searchkit` shows (`assembleIosJavaRuntimeDevice`: `static/` to link, `bundle/` to copy into - `App.app`). About 50 MB uncompressed. - **The desktop**: the façade is on the app's class path; nothing else. -- The VM's options are `nucleusJvm.jvmOptions` (a 128 MB heap: about 140 MB of memory on Android, 160 MB on iOS, on - real data). +- The VM's options are `nucleusJvm.jvmOptions` (a 128 MB heap: about 140 MB of memory on Android; the native image + keeps the heap's maximum, about 60 MB of memory on iOS for Lucene alone). ## Tests diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e85752e8..15dbada8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ filekitCore = "0.13.0" kotlin = "2.4.20" # Nucleus JVM Access: Lucene on a JVM on every platform (the search's façade, search-facade) -nja = "0.3.0" +nja = "0.4.0" agp = "9.1.1" jvmToolchain = "25" lucene = "10.5.2" diff --git a/search-facade/native-image/caller-filter.json b/search-facade/native-image/caller-filter.json new file mode 100644 index 00000000..b0bbcf9a --- /dev/null +++ b/search-facade/native-image/caller-filter.json @@ -0,0 +1,7 @@ +{ + "rules": [ + {"excludeClasses": "**"}, + {"includeClasses": "org.apache.lucene.**"}, + {"includeClasses": "io.github.kdroidfilter.seforimlibrary.search.lucene.**"} + ] +} diff --git a/search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json b/search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json new file mode 100644 index 00000000..d0b8b665 --- /dev/null +++ b/search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade/reachability-metadata.json @@ -0,0 +1,327 @@ +{ + "reflection": [ + { + "type": "byte[][]" + }, + { + "type": "com.sun.management.HotSpotDiagnosticMXBean", + "methods": [ + { + "name": "getVMOption", + "parameterTypes": [ + "java.lang.String" + ] + } + ] + }, + { + "type": "com.sun.management.VMOption", + "methods": [ + { + "name": "getValue", + "parameterTypes": [] + } + ] + }, + { + "type": "java.lang.Enum", + "serializable": true + }, + { + "type": "java.lang.Integer" + }, + { + "type": "java.lang.Long" + }, + { + "type": "java.lang.Number" + }, + { + "type": "java.lang.Object" + }, + { + "type": "java.lang.Object[]", + "serializable": true + }, + { + "type": "java.lang.String" + }, + { + "type": "java.lang.management.ManagementFactory", + "methods": [ + { + "name": "getPlatformMXBean", + "parameterTypes": [ + "java.lang.Class" + ] + } + ] + }, + { + "type": "java.util.HashSet", + "serializable": true + }, + { + "type": "java.util.LinkedHashSet", + "serializable": true + }, + { + "type": "java.util.concurrent.ArrayBlockingQueue", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.AbstractOwnableSynchronizer", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.AbstractQueuedSynchronizer", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.ReentrantLock", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.ReentrantLock$NonfairSync", + "serializable": true + }, + { + "type": "java.util.concurrent.locks.ReentrantLock$Sync", + "serializable": true + }, + { + "type": "org.apache.lucene.analysis.LowerCaseFilter" + }, + { + "type": "org.apache.lucene.analysis.StopFilter" + }, + { + "type": "org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.codecs.KnnVectorsFormat" + }, + { + "type": "org.apache.lucene.codecs.hnsw.FlatVectorsFormat" + }, + { + "type": "org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat" + }, + { + "type": "org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat" + }, + { + "type": "org.apache.lucene.index.DocsWithFieldSet" + }, + { + "type": "org.apache.lucene.index.Term" + }, + { + "type": "org.apache.lucene.index.TermStates$PendingTermLookup[]" + }, + { + "type": "org.apache.lucene.index.TermStates[]" + }, + { + "type": "org.apache.lucene.index.Term[]" + }, + { + "type": "org.apache.lucene.internal.hppc.FloatArrayList" + }, + { + "type": "org.apache.lucene.internal.hppc.IntArrayList" + }, + { + "type": "org.apache.lucene.internal.hppc.IntHashSet" + }, + { + "type": "org.apache.lucene.internal.hppc.IntIntHashMap" + }, + { + "type": "org.apache.lucene.internal.hppc.IntObjectHashMap" + }, + { + "type": "org.apache.lucene.internal.hppc.LongArrayList" + }, + { + "type": "org.apache.lucene.internal.hppc.LongHashSet" + }, + { + "type": "org.apache.lucene.internal.hppc.LongObjectHashMap" + }, + { + "type": "org.apache.lucene.search.AutomatonQuery" + }, + { + "type": "org.apache.lucene.search.BoostAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.search.DisiWrapper[]" + }, + { + "type": "org.apache.lucene.search.DocIdSet" + }, + { + "type": "org.apache.lucene.search.LRUQueryCache$CacheAndCount" + }, + { + "type": "org.apache.lucene.search.MaxNonCompetitiveBoostAttributeImpl", + "methods": [ + { + "name": "", + "parameterTypes": [] + } + ] + }, + { + "type": "org.apache.lucene.search.MultiTermQuery" + }, + { + "type": "org.apache.lucene.search.PointInSetQuery" + }, + { + "type": "org.apache.lucene.search.Query" + }, + { + "type": "org.apache.lucene.search.TermStatistics[]" + }, + { + "type": "org.apache.lucene.util.BitDocIdSet" + }, + { + "type": "org.apache.lucene.util.BitSet" + }, + { + "type": "org.apache.lucene.util.ByteBlockPool" + }, + { + "type": "org.apache.lucene.util.BytesRef" + }, + { + "type": "org.apache.lucene.util.BytesRefBlockPool" + }, + { + "type": "org.apache.lucene.util.BytesRefHash" + }, + { + "type": "org.apache.lucene.util.FixedBitSet" + }, + { + "type": "org.apache.lucene.util.FrequencyTrackingRingBuffer" + }, + { + "type": "org.apache.lucene.util.FrequencyTrackingRingBuffer$IntBag" + }, + { + "type": "org.apache.lucene.util.IntArrayDocIdSet" + }, + { + "type": "org.apache.lucene.util.LongValues" + }, + { + "type": "org.apache.lucene.util.PagedBytes" + }, + { + "type": "org.apache.lucene.util.PagedBytes$Reader" + }, + { + "type": "org.apache.lucene.util.RoaringDocIdSet" + }, + { + "type": "org.apache.lucene.util.RoaringDocIdSet$RangeDocIdSet" + }, + { + "type": "org.apache.lucene.util.automaton.CompiledAutomaton" + }, + { + "type": "org.apache.lucene.util.automaton.RunAutomaton" + }, + { + "type": "org.apache.lucene.util.automaton.Transition" + }, + { + "type": "org.apache.lucene.util.packed.DeltaPackedLongValues" + }, + { + "type": "org.apache.lucene.util.packed.DeltaPackedLongValues$Builder" + }, + { + "type": "org.apache.lucene.util.packed.MonotonicLongValues" + }, + { + "type": "org.apache.lucene.util.packed.MonotonicLongValues$Builder" + }, + { + "type": "org.apache.lucene.util.packed.PackedInts$Reader[]" + }, + { + "type": "org.apache.lucene.util.packed.PackedLongValues" + }, + { + "type": "org.apache.lucene.util.packed.PackedLongValues$Builder" + }, + { + "type": "org.gradle.api.internal.tasks.testing.filter.TestFilterSpec", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.junit.JUnitSpec", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.junit.JUnitTestDefinitionProcessorFactory", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.worker.TestWorker", + "serializable": true + }, + { + "type": "org.gradle.api.internal.tasks.testing.worker.TestWorker$State", + "serializable": true + }, + { + "type": "org.junit.runner.Result$SerializedForm", + "serializable": true + } + ], + "resources": [ + { + "glob": "META-INF/services/dev.nucleusframework.nja.server.NjaModule" + }, + { + "glob": "META-INF/services/java.nio.channels.spi.SelectorProvider" + }, + { + "glob": "META-INF/services/java.time.zone.ZoneRulesProvider" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.Codec" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.DocValuesFormat" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat" + }, + { + "glob": "META-INF/services/org.apache.lucene.codecs.PostingsFormat" + } + ] +} diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 11eb41a8..83756b5c 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -1,3 +1,4 @@ +import dev.nucleusframework.nja.plugin.NjaRuntimeType import org.jetbrains.kotlin.gradle.targets.native.tasks.KotlinNativeSimulatorTest plugins { @@ -73,13 +74,15 @@ kotlin { } } -// Lucene, through Nucleus JVM Access: the façade (search-facade) runs on a JVM, the desktop's own or the one the mobile -// apps embed (iOS: Zero linked into the app; Android: HotSpot in the app's process), and common code uses its proxies +// Lucene, through Nucleus JVM Access: the façade (search-facade) runs on the desktop's own JVM, compiled into a GraalVM +// native image on iOS (Zero, the JVM's interpreter, searches 8× slower), on HotSpot in the app's process on Android (an +// Android native image builds on Linux only), and common code uses its proxies nucleusJvm { facade(project(":search-facade")) - // The runtimes are jdk-mobile's latest release, which NJA downloads (see JVM_RUNTIME.md) + // NJA downloads the runtimes: graalvm-mobile's latest release for iOS, jdk-mobile's for Android (see JVM_RUNTIME.md) + iosRuntime.set(NjaRuntimeType.GRAALVM) androidAbis.set(setOf("arm64-v8a", "x86_64")) - // Lucene maps its files with Panama (madvise): native access, without the warning + // Lucene maps its files with Panama (madvise): native access, without the warning (a native image keeps -Xmx only) jvmOptions.addAll("-Xmx128m", "--enable-native-access=ALL-UNNAMED") // Lucene's query cache off: the same search ranks the same every time (the parity tests across platforms) providers.gradleProperty("luceneQueryCache").orNull?.let { jvmOptions.add("-Dseforim.lucene.queryCache=$it") } @@ -103,5 +106,12 @@ tasks.named("jvmTest") { providers.gradleProperty("parityCores").orNull?.let { jvmArgs("-XX:ActiveProcessorCount=$it") } providers.gradleProperty("luceneQueryCache").orNull?.let { systemProperty("seforim.lucene.queryCache", it) } providers.gradleProperty("probeOut").orNull?.let { systemProperty("probeOut", it) } + // The façade's native-image metadata, traced on these tests (with the real index): -PnativeImageAgent= + providers.gradleProperty("nativeImageAgent").orNull?.let { graalvm -> + val out = rootProject.layout.projectDirectory.dir("search-facade/src/main/resources/META-INF/native-image/io.github.kdroidfilter.seforimlibrary/search-facade") + val filter = rootProject.layout.projectDirectory.file("search-facade/native-image/caller-filter.json") + executable("$graalvm/bin/java") + jvmArgs("-agentlib:native-image-agent=config-merge-dir=${out.asFile.path},caller-filter-file=${filter.asFile.path}") + } testLogging.showStandardStreams = true } From 4576d7ee273aadade54a80d5da868c10cc000b30 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 00:45:57 +0300 Subject: [PATCH 13/14] refactor(dao): ConcurrentHashMap's caches on the JVM and Android, cachemap on iOS only Signed-off-by: Elie Gambache --- dao/build.gradle.kts | 3 ++- .../dao/ConcurrentCache.android.kt | 17 +++++++++++++++++ .../seforimlibrary/dao/ConcurrentCache.kt | 14 ++++++++++++++ .../dao/repository/SeforimRepository.kt | 6 +++--- .../seforimlibrary/dao/ConcurrentCache.ios.kt | 17 +++++++++++++++++ .../seforimlibrary/dao/ConcurrentCache.jvm.kt | 17 +++++++++++++++++ 6 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt create mode 100644 dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt create mode 100644 dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt create mode 100644 dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt diff --git a/dao/build.gradle.kts b/dao/build.gradle.kts index 8dda76f1..7bb94c1b 100644 --- a/dao/build.gradle.kts +++ b/dao/build.gradle.kts @@ -31,7 +31,6 @@ kotlin { implementation(libs.kotlinx.datetime) implementation(libs.kermit) implementation(libs.kotlinx.io.core) - implementation(libs.cachemap) } commonTest.dependencies { @@ -54,6 +53,8 @@ kotlin { iosMain.dependencies { implementation(libs.sqlDelight.driver.native) + // The concurrent caches (see ConcurrentCache.ios.kt) + implementation(libs.cachemap) } } diff --git a/dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt b/dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt new file mode 100644 index 00000000..d00c6086 --- /dev/null +++ b/dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt @@ -0,0 +1,17 @@ +package io.github.kdroidfilter.seforimlibrary.dao + +import java.util.concurrent.ConcurrentHashMap + +internal actual fun concurrentCache(): ConcurrentCache = + object : ConcurrentCache { + private val map = ConcurrentHashMap() + + override fun get(key: K): V? = map[key] + + override fun set( + key: K, + value: V, + ) { + map[key] = value + } + } diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt new file mode 100644 index 00000000..9142d94d --- /dev/null +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt @@ -0,0 +1,14 @@ +package io.github.kdroidfilter.seforimlibrary.dao + +/** A cache read and written from several threads at once: concurrent lookups of the same key may both load it. */ +internal interface ConcurrentCache { + operator fun get(key: K): V? + + operator fun set( + key: K, + value: V, + ) +} + +/** On the JVM and Android, ConcurrentHashMap; on iOS (no java.util.concurrent), cachemap's lock-free reads. */ +internal expect fun concurrentCache(): ConcurrentCache diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt index b23ccf45..8396cfa8 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt @@ -22,10 +22,10 @@ import io.github.kdroidfilter.seforimlibrary.core.models.PubPlace import io.github.kdroidfilter.seforimlibrary.core.models.Source import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry import io.github.kdroidfilter.seforimlibrary.core.models.Topic +import io.github.kdroidfilter.seforimlibrary.dao.concurrentCache import io.github.kdroidfilter.seforimlibrary.dao.extensions.toModel import io.github.kdroidfilter.seforimlibrary.db.SeforimDb import io.github.kdroidfilter.seforimlibrary.env.getEnvironmentVariable -import io.github.charlietap.cachemap.cacheMapOf import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO import kotlinx.coroutines.withContext @@ -47,12 +47,12 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L // paying a SQLite prepare + connection round-trip for every breadcrumb/tree walk. // Profiling (see JFR 2026-04-23) showed 70 `sqlite3_prepare` calls in 20 s all // coming from `getCategory`. - private val categoryCache = cacheMapOf() + private val categoryCache = concurrentCache() // book.orderIndex is denormalized onto each link row at insert time (targetBookOrderIndex) // so the commentaries path can sort without JOINing book. Books are immutable once inserted // and exist before any link references them, so a read-through cache is safe during import. - private val bookOrderIndexCache = cacheMapOf() + private val bookOrderIndexCache = concurrentCache() private fun resolveBookOrderIndex(bookId: Long): Long = bookOrderIndexCache[bookId] diff --git a/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt b/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt new file mode 100644 index 00000000..7c95449b --- /dev/null +++ b/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt @@ -0,0 +1,17 @@ +package io.github.kdroidfilter.seforimlibrary.dao + +import io.github.charlietap.cachemap.cacheMapOf + +internal actual fun concurrentCache(): ConcurrentCache = + object : ConcurrentCache { + private val map = cacheMapOf() + + override fun get(key: K): V? = map[key] + + override fun set( + key: K, + value: V, + ) { + map[key] = value + } + } diff --git a/dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt b/dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt new file mode 100644 index 00000000..d00c6086 --- /dev/null +++ b/dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt @@ -0,0 +1,17 @@ +package io.github.kdroidfilter.seforimlibrary.dao + +import java.util.concurrent.ConcurrentHashMap + +internal actual fun concurrentCache(): ConcurrentCache = + object : ConcurrentCache { + private val map = ConcurrentHashMap() + + override fun get(key: K): V? = map[key] + + override fun set( + key: K, + value: V, + ) { + map[key] = value + } + } From 3f563d94ef84793bef2a5547ce5bbc06dd9426e2 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sun, 18 Oct 2026 01:05:29 +0300 Subject: [PATCH 14/14] refactor: the concurrent caches in a module of their own, :cache ConcurrentHashMap on the JVM and Android, cachemap on iOS only, for dao and the apps (Zayit's searchkit). Signed-off-by: Elie Gambache --- cache/build.gradle.kts | 68 +++++++++++++++++++ .../cache}/ConcurrentCache.android.kt | 4 +- .../seforimlibrary/cache}/ConcurrentCache.kt | 6 +- .../cache/ConcurrentCacheTest.kt | 31 +++++++++ .../cache}/ConcurrentCache.ios.kt | 4 +- .../cache}/ConcurrentCache.jvm.kt | 4 +- dao/build.gradle.kts | 3 +- .../dao/repository/SeforimRepository.kt | 2 +- settings.gradle.kts | 1 + 9 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 cache/build.gradle.kts rename {dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao => cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache}/ConcurrentCache.android.kt (70%) rename {dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao => cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache}/ConcurrentCache.kt (63%) create mode 100644 cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt rename {dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao => cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache}/ConcurrentCache.ios.kt (69%) rename {dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao => cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache}/ConcurrentCache.jvm.kt (70%) diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts new file mode 100644 index 00000000..b4003baa --- /dev/null +++ b/cache/build.gradle.kts @@ -0,0 +1,68 @@ + +plugins { + alias(libs.plugins.multiplatform) + alias(libs.plugins.android.library) + alias(libs.plugins.maven.publish) +} + +group = "io.github.kdroidfilter.seforimlibrary" + +/** Caches read and written from several threads at once, on every platform (ConcurrentCache.kt). */ +kotlin { + jvmToolchain(libs.versions.jvmToolchain.get().toInt()) + + android { + namespace = "io.github.kdroidfilter.seforimlibrary.cache" + compileSdk = 35 + minSdk = 21 + } + jvm() + iosArm64() + iosSimulatorArm64() + + sourceSets { + commonTest.dependencies { + implementation(kotlin("test")) + implementation(libs.kotlinx.coroutines.core) + } + + // No java.util.concurrent on iOS: cachemap's map, lock-free reads + iosMain.dependencies { + implementation(libs.cachemap) + } + } + +} + +//Publishing your Kotlin Multiplatform library to Maven Central +//https://www.jetbrains.com/help/kotlin-multiplatform-dev/multiplatform-publish-libraries.html +mavenPublishing { + publishToMavenCentral() + coordinates("io.github.kdroidfilter.seforimlibrary", "cache", "1.0.0") + + pom { + name = "SeforimLibrary" + description = "Kotlin Multiplatform library" + url = "github url" //todo + + licenses { + license { + name = "MIT" + url = "https://opensource.org/licenses/MIT" + } + } + + developers { + developer { + id = "" //todo + name = "" //todo + email = "" //todo + } + } + + scm { + url = "github url" //todo + } + } + if (project.hasProperty("signing.keyId")) signAllPublications() +} diff --git a/dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt b/cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.android.kt similarity index 70% rename from dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt rename to cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.android.kt index d00c6086..1af2f411 100644 --- a/dao/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.android.kt +++ b/cache/src/androidMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.android.kt @@ -1,8 +1,8 @@ -package io.github.kdroidfilter.seforimlibrary.dao +package io.github.kdroidfilter.seforimlibrary.cache import java.util.concurrent.ConcurrentHashMap -internal actual fun concurrentCache(): ConcurrentCache = +actual fun concurrentCache(): ConcurrentCache = object : ConcurrentCache { private val map = ConcurrentHashMap() diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt b/cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.kt similarity index 63% rename from dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt rename to cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.kt index 9142d94d..333ffe59 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.kt +++ b/cache/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.kt @@ -1,7 +1,7 @@ -package io.github.kdroidfilter.seforimlibrary.dao +package io.github.kdroidfilter.seforimlibrary.cache /** A cache read and written from several threads at once: concurrent lookups of the same key may both load it. */ -internal interface ConcurrentCache { +interface ConcurrentCache { operator fun get(key: K): V? operator fun set( @@ -11,4 +11,4 @@ internal interface ConcurrentCache { } /** On the JVM and Android, ConcurrentHashMap; on iOS (no java.util.concurrent), cachemap's lock-free reads. */ -internal expect fun concurrentCache(): ConcurrentCache +expect fun concurrentCache(): ConcurrentCache diff --git a/cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt b/cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt new file mode 100644 index 00000000..bf375002 --- /dev/null +++ b/cache/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCacheTest.kt @@ -0,0 +1,31 @@ +package io.github.kdroidfilter.seforimlibrary.cache + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class ConcurrentCacheTest { + @Test + fun storesAndReplacesValues() { + val cache = concurrentCache() + assertNull(cache["a"]) + cache["a"] = 1 + cache["a"] = 2 + assertEquals(2, cache["a"]) + } + + @Test + fun keepsEveryValueWrittenFromSeveralThreads() = + runBlocking { + val cache = concurrentCache() + (0 until 8) + .map { worker -> + async(Dispatchers.Default) { for (i in 0 until 1_000) cache[worker * 1_000 + i] = i } + }.awaitAll() + for (key in 0 until 8_000) assertEquals(key % 1_000, cache[key]) + } +} diff --git a/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt b/cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.ios.kt similarity index 69% rename from dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt rename to cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.ios.kt index 7c95449b..09694e10 100644 --- a/dao/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.ios.kt +++ b/cache/src/iosMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.ios.kt @@ -1,8 +1,8 @@ -package io.github.kdroidfilter.seforimlibrary.dao +package io.github.kdroidfilter.seforimlibrary.cache import io.github.charlietap.cachemap.cacheMapOf -internal actual fun concurrentCache(): ConcurrentCache = +actual fun concurrentCache(): ConcurrentCache = object : ConcurrentCache { private val map = cacheMapOf() diff --git a/dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt b/cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.jvm.kt similarity index 70% rename from dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt rename to cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.jvm.kt index d00c6086..1af2f411 100644 --- a/dao/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/ConcurrentCache.jvm.kt +++ b/cache/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/cache/ConcurrentCache.jvm.kt @@ -1,8 +1,8 @@ -package io.github.kdroidfilter.seforimlibrary.dao +package io.github.kdroidfilter.seforimlibrary.cache import java.util.concurrent.ConcurrentHashMap -internal actual fun concurrentCache(): ConcurrentCache = +actual fun concurrentCache(): ConcurrentCache = object : ConcurrentCache { private val map = ConcurrentHashMap() diff --git a/dao/build.gradle.kts b/dao/build.gradle.kts index 7bb94c1b..2a5b5cb4 100644 --- a/dao/build.gradle.kts +++ b/dao/build.gradle.kts @@ -25,6 +25,7 @@ kotlin { sourceSets { commonMain.dependencies { api(project(":core")) + implementation(project(":cache")) implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.serialization.json) implementation(libs.kotlinx.serialization.protobuf) @@ -53,8 +54,6 @@ kotlin { iosMain.dependencies { implementation(libs.sqlDelight.driver.native) - // The concurrent caches (see ConcurrentCache.ios.kt) - implementation(libs.cachemap) } } diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt index 8396cfa8..e02f7ba2 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt @@ -7,6 +7,7 @@ import app.cash.sqldelight.db.SqlCursor import app.cash.sqldelight.db.SqlDriver import co.touchlab.kermit.Logger import co.touchlab.kermit.Severity +import io.github.kdroidfilter.seforimlibrary.cache.concurrentCache import io.github.kdroidfilter.seforimlibrary.core.models.AltTocEntry import io.github.kdroidfilter.seforimlibrary.core.models.AltTocStructure import io.github.kdroidfilter.seforimlibrary.core.models.Author @@ -22,7 +23,6 @@ import io.github.kdroidfilter.seforimlibrary.core.models.PubPlace import io.github.kdroidfilter.seforimlibrary.core.models.Source import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry import io.github.kdroidfilter.seforimlibrary.core.models.Topic -import io.github.kdroidfilter.seforimlibrary.dao.concurrentCache import io.github.kdroidfilter.seforimlibrary.dao.extensions.toModel import io.github.kdroidfilter.seforimlibrary.db.SeforimDb import io.github.kdroidfilter.seforimlibrary.env.getEnvironmentVariable diff --git a/settings.gradle.kts b/settings.gradle.kts index ac176c8e..5af0e0fb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -67,6 +67,7 @@ dependencyResolutionManagement { } } include(":core") +include(":cache") include(":dao") include(":search") include(":search-facade")