fix(generator): stop losing links and misfiling categories silently

dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt

@@ -2071,6 +2071,22 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L
             val connectionTypeId = getOrCreateConnectionType(link.connectionType.name)
             logger.d{"Using connection type ID: $connectionTypeId for type: ${link.connectionType.name}"}
 
+            // A caller-chosen id (build allocator) keeps later allocations from colliding with this row.
+            if (link.id > 0) {
+                database.linkQueriesQueries.insertWithId(
+                    id = link.id,
+                    sourceBookId = link.sourceBookId,
+                    targetBookId = link.targetBookId,
+                    sourceLineId = link.sourceLineId,
+                    targetLineId = link.targetLineId,
+                    targetLineIndex = link.targetLineIndex.toLong(),
+                    targetBookOrderIndex = resolveBookOrderIndex(link.targetBookId),
+                    connectionTypeId = connectionTypeId,
+                    isDeclaredBase = if (link.isDeclaredBase) 1L else 0L,
+                )
+                checkLinkIdInserted(link.id, link.sourceLineId, link.targetLineId, connectionTypeId)
+                return@withContext link.id
+            }
             database.linkQueriesQueries.insert(
                 sourceBookId = link.sourceBookId,
                 targetBookId = link.targetBookId,
@@ -2149,6 +2165,7 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L
                         connectionTypeId = connectionTypeId,
                         isDeclaredBase = declaredFlag,
                     )
+                    checkLinkIdInserted(link.id, link.sourceLineId, link.targetLineId, connectionTypeId)
                 } else {
                     database.linkQueriesQueries.insert(
                         sourceBookId = link.sourceBookId,
@@ -2854,6 +2871,24 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L
             connectionTypeId = connectionTypeId,
             isDeclaredBase = if (isDeclaredBase) 1L else 0L,
         )
+        checkLinkIdInserted(id, sourceLineId, targetLineId, connectionTypeId)
+    }
+
+    /**
+     * `insertWithId` is INSERT OR IGNORE so re-inserting the same link is a no-op. An id already held by a
+     * different link means two id sources disagree: fail instead of silently dropping the new link.
+     */
+    private fun checkLinkIdInserted(id: Long, sourceLineId: Long, targetLineId: Long, connectionTypeId: Long) {
+        if (database.linkQueriesQueries.changes().executeAsOne() > 0L) return
+        val existing = database.linkQueriesQueries.selectLineKeyById(id).executeAsOneOrNull() ?: return
+        check(
+            existing.sourceLineId == sourceLineId &&
+                existing.targetLineId == targetLineId &&
+                existing.connectionTypeId == connectionTypeId,
+        ) {
+            "Link id $id is already used by link ${existing.sourceLineId}->${existing.targetLineId} " +
+                "(type ${existing.connectionTypeId}); cannot insert $sourceLineId->$targetLineId (type $connectionTypeId)"
+        }
     }
 
     // ─── schema_meta accessors ────────────────────────────────────────────────

dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/LinkQueries.sq

@@ -144,6 +144,12 @@ DELETE FROM link WHERE sourceBookId = ? OR targetBookId = ?;
 lastInsertRowId:
 SELECT last_insert_rowid();
 
+changes:
+SELECT changes();
+
+selectLineKeyById:
+SELECT sourceLineId, targetLineId, connectionTypeId FROM link WHERE id = ?;
+
 -- Count links by source book
 countLinksBySourceBook:
 SELECT COUNT(*) FROM link WHERE sourceBookId = ?;

dao/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepositoryIntegrationTest.kt

@@ -12,6 +12,7 @@ import kotlin.test.AfterTest
 import kotlin.test.BeforeTest
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
 import kotlin.test.assertNotEquals
 import kotlin.test.assertNotNull
 import kotlin.test.assertNull
@@ -526,6 +527,22 @@ class SeforimRepositoryIntegrationTest {
         )
     }
 
+    @Test
+    fun `insertLink fails when its id is held by another link`() = runBlocking {
+        val sourceId = repository.insertSource("Test")
+        val catId = repository.insertCategory(Category(parentId = null, title = "C", level = 0, order = 1))
+        val bookId = repository.insertBook(Book(categoryId = catId, sourceId = sourceId, title = "B", order = 1f))
+        val l = (0 until 3).map { repository.insertLine(Line(bookId = bookId, lineIndex = it, content = "L$it")) }
+        fun link(src: Long, tgt: Long) = Link(
+            id = 42, sourceBookId = bookId, targetBookId = bookId, sourceLineId = src, targetLineId = tgt,
+            targetLineIndex = 0, connectionType = ConnectionType.COMMENTARY,
+        )
+        repository.insertLinksBatch(listOf(link(l[0], l[1])))
+        repository.insertLinksBatch(listOf(link(l[0], l[1]))) // same link again: no-op
+        assertFailsWith<IllegalStateException> { repository.insertLinksBatch(listOf(link(l[0], l[2]))) }
+        Unit
+    }
+
     // ==================== Virtual SOURCE view tests ====================
 
     // Validates the single-direction storage + virtual SOURCE view contract:

generator/otzariasqlite/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/Generator.kt

@@ -178,19 +178,26 @@ class DatabaseGenerator(
         return candidates.firstOrNull { comparableLabel(it.title) == targetKey }
     }
 
+    private val categoryPathById = mutableMapOf<Long, String>()
+
+    /** Titles from the root down to [id], joined by '/', as the category's allocator key. */
+    private suspend fun categoryPath(id: Long): String = categoryPathById.getOrPut(id) {
+        val category = repository.getCategory(id) ?: error("Category $id not found")
+        category.parentId?.let { "${categoryPath(it)}/${category.title}" } ?: category.title
+    }
+
     private suspend fun ensureCategoryHierarchy(rawTitle: String, parentId: Long?, startLevel: Int): CategoryPlacement {
         val normalizedSegments = normalizeCategorySegments(rawTitle)
         var currentParent = parentId
         var currentLevel = startLevel
         var lastId: Long? = null
-        val pathSoFar = StringBuilder()
 
         for (title in normalizedSegments) {
-            if (pathSoFar.isNotEmpty()) pathSoFar.append('/')
-            pathSoFar.append(title)
             val existing = findExistingCategory(currentParent, title)
             val categoryId = existing?.id ?: bindings.upsertCategory(
-                canonicalPath = pathSoFar.toString(),
+                // Full path from the root: a bare folder name ("סדר מועד") is shared by many folders,
+                // and one id per name filed all of them under the first one created.
+                canonicalPath = currentParent?.let { "${categoryPath(it)}/$title" } ?: title,
                 parentId = currentParent,
                 title = title,
                 level = currentLevel,
@@ -1478,19 +1485,23 @@ class DatabaseGenerator(
                         continue
                     }
 
+                    val connectionType = ConnectionType.fromString(linkData.connectionType)
                     val link = Link(
+                        id = allocator.linkId(sourceLineId, targetLineId, bindings.upsertConnectionType(connectionType.name)),
                         sourceBookId = sourceBook.id,
                         targetBookId = targetBook.id,
                         sourceLineId = sourceLineId,
                         targetLineId = targetLineId,
                         targetLineIndex = targetLineIndex,
-                        connectionType = ConnectionType.fromString(linkData.connectionType)
+                        connectionType = connectionType
                     )
 
                     logger.d { "Inserting link from book ${sourceBook.id} to book ${targetBook.id}" }
                     val linkId = repository.insertLink(link)
                     logger.d { "Link inserted with ID: $linkId" }
                     processed++
+                } catch (e: IllegalStateException) {
+                    throw e // link id collision: data would be lost, fail the build
                 } catch (e: Exception) {
                     // Changed from error to debug level to reduce unnecessary error logs
                     logger.d(e) { "Error processing link: ${linkData.heRef_2}" }
@@ -1553,16 +1564,27 @@ class DatabaseGenerator(
                 // Skip links where source or target is a heading line
                 if (sourceLineId in headingLineIds || targetLineId in headingLineIds) continue
 
-                val link = Link(
+                val link = canonicalOtzariaLink(
                     sourceBookId = sourceBook.id,
-                    targetBookId = targetBook.id,
                     sourceLineId = sourceLineId,
+                    sourceLineIndex = sourceLineIndex,
+                    targetBookId = targetBook.id,
                     targetLineId = targetLineId,
                     targetLineIndex = targetLineIndex,
-                    connectionType = ConnectionType.fromString(linkData.connectionType)
+                    connectionType = ConnectionType.fromString(linkData.connectionType),
+                )
+                repository.insertLink(
+                    link.copy(
+                        id = allocator.linkId(
+                            link.sourceLineId,
+                            link.targetLineId,
+                            bindings.upsertConnectionType(link.connectionType.name),
+                        ),
+                    ),
                 )
-                repository.insertLink(link)
                 processed++
+            } catch (e: IllegalStateException) {
+                throw e // link id collision: data would be lost, fail the build
             } catch (_: Exception) {
                 // Skip malformed entries but continue
             }

generator/otzariasqlite/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/OtzariaLinkOrientation.kt

@@ -0,0 +1,44 @@
+package io.github.kdroidfilter.seforimlibrary.otzariasqlite
+
+import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType
+import io.github.kdroidfilter.seforimlibrary.core.models.Link
+
+/**
+ * Builds the stored form of an Otzaria link read from `<source book>_links.json`.
+ *
+ * Otzaria often lists a commentary's links only in the commentary's own file, typed
+ * `source` ("the target is my source"), e.g. קרן אורה על זבחים → זבחים, with nothing
+ * in זבחים_links.json. Stored links are always base → dependant (see [Link]), so a
+ * `source` link is flipped into a COMMENTARY link from the base line. Stored as is,
+ * neither the commentary panel nor the virtual SOURCE view reads it.
+ *
+ * The returned link has no id yet.
+ */
+internal fun canonicalOtzariaLink(
+    sourceBookId: Long,
+    sourceLineId: Long,
+    sourceLineIndex: Int,
+    targetBookId: Long,
+    targetLineId: Long,
+    targetLineIndex: Int,
+    connectionType: ConnectionType,
+): Link =
+    if (connectionType == ConnectionType.SOURCE) {
+        Link(
+            sourceBookId = targetBookId,
+            targetBookId = sourceBookId,
+            sourceLineId = targetLineId,
+            targetLineId = sourceLineId,
+            targetLineIndex = sourceLineIndex,
+            connectionType = ConnectionType.COMMENTARY,
+        )
+    } else {
+        Link(
+            sourceBookId = sourceBookId,
+            targetBookId = targetBookId,
+            sourceLineId = sourceLineId,
+            targetLineId = targetLineId,
+            targetLineIndex = targetLineIndex,
+            connectionType = connectionType,
+        )
+    }

generator/otzariasqlite/src/commonTest/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/OtzariaLinkOrientationTest.kt

@@ -0,0 +1,23 @@
+package io.github.kdroidfilter.seforimlibrary.otzariasqlite
+
+import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType
+import io.github.kdroidfilter.seforimlibrary.core.models.Link
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class OtzariaLinkOrientationTest {
+    @Test
+    fun sourceLinkIsFlippedIntoACommentaryFromTheBase() {
+        // קרן אורה על זבחים (book 2, line 21) → זבחים (book 1, line 7), typed "source"
+        val link = canonicalOtzariaLink(2, 200, 21, 1, 100, 7, ConnectionType.SOURCE)
+
+        assertEquals(Link(sourceBookId = 1, targetBookId = 2, sourceLineId = 100, targetLineId = 200, targetLineIndex = 21, connectionType = ConnectionType.COMMENTARY), link)
+    }
+
+    @Test
+    fun otherLinksKeepTheirOrientation() {
+        val link = canonicalOtzariaLink(1, 100, 7, 2, 200, 21, ConnectionType.COMMENTARY)
+
+        assertEquals(Link(sourceBookId = 1, targetBookId = 2, sourceLineId = 100, targetLineId = 200, targetLineIndex = 21, connectionType = ConnectionType.COMMENTARY), link)
+    }
+}

generator/otzariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/GenerateHavroutaLinks.kt

@@ -92,7 +92,7 @@ fun main(args: Array<String>) = runBlocking {
                     "generated_at" to java.time.Instant.now().toString(),
                 ),
             )
-        }.onFailure { logger.w(it) { "Failed to write build_state to $buildStatePath" } }
+        }.getOrElse { throw IllegalStateException("Failed to write build_state to $buildStatePath", it) }
         Unit
     } catch (e: Exception) {
         logger.e(e) { "Error generating Havrouta links" }

generator/otzariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/GenerateLines.kt

@@ -116,7 +116,8 @@ fun main(args: Array<String>) = runBlocking {
                     repository.executeRawQuery("PRAGMA foreign_keys=ON")
                     logger.i { "Seeding completed. Imported ${tables.size} tables." }
                 }.onFailure { e ->
-                    logger.e(e) { "Failed to seed in-memory DB from $baseDbPath; continuing with empty DB." }
+                    logger.e(e) { "Failed to seed in-memory DB from $baseDbPath" }
+                    throw e // the persist step would otherwise overwrite the DB with a partial copy
                 }
             } else {
                 logger.w { "appendExistingDb enabled but base DB not found at $baseDbPath; starting from empty in-memory DB" }
@@ -176,7 +177,7 @@ fun main(args: Array<String>) = runBlocking {
                     "generated_at" to java.time.Instant.now().toString(),
                 ),
             )
-        }.onFailure { logger.w(it) { "Failed to write build_state to $buildStatePath" } }
+        }.getOrElse { throw IllegalStateException("Failed to write build_state to $buildStatePath", it) }
         Unit
         logger.i { "Phase 1 completed successfully. DB at ${if (useMemoryDb) persistDbPath else dbPath}" }
     } catch (e: Exception) {

generator/otzariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/otzariasqlite/GenerateLinks.kt

@@ -78,7 +78,8 @@ fun main(args: Array<String>) = runBlocking {
                     repository.executeRawQuery("PRAGMA foreign_keys=ON")
                     logger.i { "Seeding completed. Imported ${'$'}{tables.size} tables." }
                 }.onFailure { e ->
-                    logger.e(e) { "Failed to seed in-memory DB from $baseDb. Links may not be processed." }
+                    logger.e(e) { "Failed to seed in-memory DB from $baseDb" }
+                    throw e // the persist step would otherwise overwrite the DB with a partial copy
                 }
             } else {
                 logger.w { "Base DB not found at $baseDb; running with empty in-memory DB" }
@@ -108,7 +109,7 @@ fun main(args: Array<String>) = runBlocking {
                     "generated_at" to java.time.Instant.now().toString(),
                 ),
             )
-        }.onFailure { logger.w(it) { "Failed to write build_state to $buildStatePath" } }
+        }.getOrElse { throw IllegalStateException("Failed to write build_state to $buildStatePath", it) }
         Unit
         if (useMemoryDb) {
             // Persist in-memory DB to disk using VACUUM INTO (target must not exist)

generator/searchindex/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/searchindex/BuildLuceneIndex.kt

@@ -99,7 +99,8 @@ fun main() = runBlocking {
                 repo.executeRawQuery("PRAGMA foreign_keys=ON")
                 logger.i { "[IndexDefault] Seed complete: ${'$'}{tables.size} tables copied" }
             }.onFailure { e ->
-                logger.e(e) { "[IndexDefault] Failed to seed in-memory DB; falling back to disk reads" }
+                logger.e(e) { "[IndexDefault] Failed to seed in-memory DB" }
+                throw e // the index would be built from an empty or partial DB
             }
         } else {
             logger.w { "[IndexDefault] Base DB not found at $baseDb; cannot seed in-memory DB" }

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/GenerateSefariaSqlite.kt

@@ -128,7 +128,7 @@ fun main(args: Array<String>) = runBlocking {
                     "build_version" to buildVersion.toString(),
                 ),
             )
-        }.onFailure { logger.w(it) { "Failed to write build_state to $buildStatePath" } }
+        }.getOrElse { throw IllegalStateException("Failed to write build_state to $buildStatePath", it) }
         Unit
 
         logger.i { "Sefaria -> SQLite completed. DB at ${if (useMemoryDb) persistDbPath else dbPath}" }

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/RenameCategoriesPostProcess.kt

@@ -56,7 +56,9 @@ fun main(args: Array<String>) {
         "ראשונים על התלמוד" to "ראשונים",
         "אחרונים על התלמוד" to "אחרונים",
         "ראשונים על המשנה" to "ראשונים",
-        "אחרונים על המשנה" to "אחרונים"
+        "אחרונים על המשנה" to "אחרונים",
+        // Honorific as in the references; also merges Otzaria's "כתבי הארי ורבי חיים ויטאל" folder into it
+        "כתבי האר״י וחיים ויטאל" to "כתבי האר״י ורבי חיים ויטאל"
     )
 
     try {