From c440c75f8c50ff8f0ddc2a8b6bf485b88f344949 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 09:19:55 +0800 Subject: [PATCH 01/17] test(executor): add characterization tests for local executor sync Pin current behaviour of LocalExecutorService before refactoring: streaming/legacy paths, pre-count, three cancel routes, NULL handling and column projection. Backed by a network-free FakePluginService so the suite runs without a real database. Two assertions intentionally capture known quirks (JDBC-abort cancel reporting count=0, and STOPPED state after all rows committed) to be fixed in a later phase. --- .../LocalExecutorCharacterizationTest.kt | 372 ++++++++++++++++++ .../test/local/support/FakePluginService.kt | 184 +++++++++ 2 files changed, 556 insertions(+) create mode 100644 test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt create mode 100644 test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt new file mode 100644 index 0000000000..650420d74c --- /dev/null +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt @@ -0,0 +1,372 @@ +package io.edurt.datacap.test.local + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import io.edurt.datacap.executor.common.RunState +import io.edurt.datacap.executor.configure.ExecutorConfigure +import io.edurt.datacap.executor.configure.ExecutorProgressListener +import io.edurt.datacap.executor.configure.ExecutorRequest +import io.edurt.datacap.executor.configure.OriginColumn +import io.edurt.datacap.executor.local.LocalExecutorService +import io.edurt.datacap.spi.model.Configure +import io.edurt.datacap.spi.model.Response +import io.edurt.datacap.test.local.support.FakePluginService +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.CopyOnWriteArrayList + +/** + * LocalExecutorService 的“特性测试”（characterization test）： + * 只依赖内存 fake，不连任何数据库，用来在重构前锁定当前可观测行为。 + * + * 注意：这里断言的是【当前实际行为】，其中个别断言（见 TODO(phase2)）刻画的是已知的、 + * 计划在 Phase 2 修正的可疑行为。重构后如果这些断言需要修改，说明改动是有意为之、可见的。 + */ +class LocalExecutorCharacterizationTest +{ + private val mapper = ObjectMapper() + + // --------------------------------------------------------------------- + // Streaming 双端流式路径 + // --------------------------------------------------------------------- + + @Test + fun streamingCopiesProjectsAndPreservesNull() + { + // 源端返回 3 列，但只映射其中两列，并调换顺序，验证投影 + 丢列 + NULL 保留 + val source = FakePluginService( + streaming = true, + headers = listOf("id", "name", "age"), + streamRows = listOf( + listOf(1, "alice", 30), + listOf(2, null, 41), + listOf(3, "carol", 25) + ) + ) + val sink = FakePluginService(streaming = true) + + // 目标列顺序 = [full_name(<-name), uid(<-id)]，故意与源 headers 顺序不同 + val request = buildRequest( + source, sink, + originColumns = linkedSetOf( + OriginColumn("full_name", "name"), + OriginColumn("uid", "id") + ) + ) + + val response = LocalExecutorService().start(request) + + assertTrue(response.successful) + assertEquals(RunState.SUCCESS, response.state) + assertEquals(3, response.count) + assertEquals( + listOf( + listOf("alice", 1), + listOf(null, 2), + listOf("carol", 3) + ), + sink.committedRows.toList() + ) + } + + @Test + fun streamingPreCountPopulatesProgressTotal() + { + val source = FakePluginService( + streaming = true, + headers = listOf("id"), + streamRows = listOf(listOf(1), listOf(2)), + // preCount 会对包了 datacap_precount_t 的查询调用 execute() + executeHandler = { sql -> + if (sql.contains("datacap_precount_t")) countResponse(42) + else Response.builder().isSuccessful(true).columns(emptyList()).build() + } + ) + val sink = FakePluginService(streaming = true) + + val progress = CopyOnWriteArrayList>() + val request = buildRequest( + source, sink, + originColumns = linkedSetOf(OriginColumn("id", "id")) + ).apply { + preCount = true + progressListener = ExecutorProgressListener { processed, total -> progress.add(processed to total) } + } + + val response = LocalExecutorService().start(request) + + assertTrue(response.successful) + assertEquals(2, response.count) + // 第一条进度应是 pre-count 后上报的 (0, 42)，最后一条应带上相同 total=42 + assertEquals(0L to 42L, progress.first()) + assertEquals(42L, progress.last().second) + } + + @Test + fun streamingStopMidStreamViaCancelledFlag() + { + val service = LocalExecutorService() + val source = FakePluginService( + streaming = true, + headers = listOf("id"), + streamRows = (1..5).map { listOf(it) } + ) + val sink = FakePluginService(streaming = true) + val request = buildRequest( + source, sink, + taskName = "stop-flag", + originColumns = linkedSetOf(OriginColumn("id", "id")) + ) + + // 在回调第 3 行（index=2）之前请求停止：第 2 行已写入，第 3 行 onRow 入口检测到 cancelled 抛出 + source.beforeRow = { i -> + if (i == 2) + { + val stopResponse = service.stop(stopRequest("stop-flag")) + assertEquals(RunState.STOPPED, stopResponse.state) + } + } + + val response = service.start(request) + + assertFalse(response.successful) + assertEquals(RunState.STOPPED, response.state) + assertEquals(2, response.count) + assertEquals(2, sink.committedRows.size) + } + + @Test + fun streamingStopDetectedAfterLoop() + { + // 模拟“驱动在 cancel 后让 rs.next() 直接返回 false（不抛异常）”： + // 所有行都回调完了才 stop，靠循环后的二次 cancelled 检查兜底 + val service = LocalExecutorService() + val source = FakePluginService( + streaming = true, + headers = listOf("id"), + streamRows = (1..3).map { listOf(it) } + ) + val sink = FakePluginService(streaming = true) + val request = buildRequest( + source, sink, + taskName = "stop-afterloop", + originColumns = linkedSetOf(OriginColumn("id", "id")) + ) + source.afterRows = { service.stop(stopRequest("stop-afterloop")) } + + val response = service.start(request) + + assertFalse(response.successful) + assertEquals(RunState.STOPPED, response.state) + // 全部 3 行其实都已提交，但因为收到停止请求，最终状态仍被判为 STOPPED + assertEquals(3, response.count) + assertEquals(3, sink.committedRows.size) + } + + @Test + fun streamingDriverAbortAfterCancelIsReportedAsStopped() + { + // 第三条 cancel 检测路径：cancel 后底层 fetch 抛 SQLException，走 catch(Exception)+cancelled 逆判定 + val service = LocalExecutorService() + val source = FakePluginService( + streaming = true, + headers = listOf("id"), + streamRows = (1..5).map { listOf(it) } + ) + val sink = FakePluginService(streaming = true) + val request = buildRequest( + source, sink, + taskName = "stop-abort", + originColumns = linkedSetOf(OriginColumn("id", "id")) + ) + source.beforeRow = { i -> if (i == 2) service.stop(stopRequest("stop-abort")) } + source.throwAtRow = 2 + + val response = service.start(request) + + assertFalse(response.successful) + assertEquals(RunState.STOPPED, response.state) + // 前 2 行确实已落库 + assertEquals(2, sink.committedRows.size) + // TODO(phase2): 当前实现用 rowsAtStop 上报，而 rowsAtStop 每 1000 行才更新一次， + // 所以小数据量下 count=0（丢失了已处理行数）。这是计划在 Phase 2 修正的已知问题。 + assertEquals(0, response.count) + } + + // --------------------------------------------------------------------- + // Legacy 回退路径（源或汇任一不支持流式） + // --------------------------------------------------------------------- + + @Test + fun legacyWithStreamingSinkUsesBatchWriter() + { + // 源端不支持流式 -> execute() 全量返回 ObjectNode；汇端支持流式 -> 走 BatchWriter + val rows = listOf( + objectNode("id" to 1, "name" to "alice"), + objectNode("id" to 2, "name" to null) + ) + val source = FakePluginService( + streaming = false, + executeHandler = { rowsResponse(rows) } + ) + val sink = FakePluginService(streaming = true) + val request = buildRequest( + source, sink, + originColumns = linkedSetOf( + OriginColumn("full_name", "name"), + OriginColumn("uid", "id") + ) + ) + + val response = LocalExecutorService().start(request) + + assertTrue(response.successful) + assertEquals(RunState.SUCCESS, response.state) + assertEquals(2, response.count) + assertEquals( + listOf( + listOf("alice", 1), + listOf(null, 2) + ), + sink.committedRows.toList() + ) + } + + @Test + fun legacyBothNonStreamingBuildsInsertSql() + { + // 双端都不支持流式 -> 拼 INSERT 字符串，验证列名反引号、单引号转义、NULL、数字不加引号 + val rows = listOf( + objectNode("id" to 1, "name" to "O'Brien"), + objectNode("id" to 2, "name" to null) + ) + val source = FakePluginService( + streaming = false, + executeHandler = { rowsResponse(rows) } + ) + val sink = FakePluginService( + streaming = false, + executeHandler = { Response.builder().isSuccessful(true).columns(emptyList()).build() } + ) + val request = buildRequest( + source, sink, + originColumns = linkedSetOf( + OriginColumn("uid", "id"), + OriginColumn("full_name", "name") + ) + ) + + val response = LocalExecutorService().start(request) + + assertTrue(response.successful) + assertEquals(2, response.count) + val sql = sink.executedSql.joinToString("\n") + assertTrue("should build INSERT: $sql", sql.contains("INSERT INTO `target_db`.`target_tbl`")) + assertTrue("should quote columns: $sql", sql.contains("`uid`") && sql.contains("`full_name`")) + assertTrue("should escape single quote: $sql", sql.contains("'O''Brien'")) + assertTrue("should emit NULL literal: $sql", sql.contains("NULL")) + } + + // --------------------------------------------------------------------- + // 失败路径 + // --------------------------------------------------------------------- + + @Test + fun sourceFailureIsReportedAsFailure() + { + val source = FakePluginService( + streaming = false, + executeHandler = { + Response.builder().isSuccessful(false).message("boom").build() + } + ) + val sink = FakePluginService(streaming = true) + val request = buildRequest( + source, sink, + originColumns = linkedSetOf(OriginColumn("id", "id")) + ) + + val response = LocalExecutorService().start(request) + + assertFalse(response.successful) + assertEquals(RunState.FAILURE, response.state) + assertNotNull(response.message) + } + + // --------------------------------------------------------------------- + // helpers + // --------------------------------------------------------------------- + + private fun buildRequest( + source: FakePluginService, + sink: FakePluginService, + taskName: String = "", + originColumns: LinkedHashSet + ): ExecutorRequest + { + val input = ExecutorConfigure( + type = "TestInput", + configure = null, + supportOptions = emptySet() + ).apply { + plugin = source + query = "SELECT * FROM source" + database = "target_db" + table = "target_tbl" + originConfigure = Configure() + this.originColumns = originColumns + } + val output = ExecutorConfigure( + type = "TestOutput", + configure = null, + supportOptions = emptySet() + ).apply { + plugin = sink + originConfigure = Configure() + } + // workHome=null -> 不落任务日志文件；taskName 非空才可被 stop() 定位 + return ExecutorRequest(null, input, output).apply { + this.taskName = taskName + this.userName = "tester" + } + } + + private fun stopRequest(taskName: String): ExecutorRequest + { + val placeholder = ExecutorConfigure(null) + return ExecutorRequest(taskName, "", placeholder, placeholder) + } + + private fun objectNode(vararg pairs: Pair): ObjectNode + { + val node = mapper.createObjectNode() + for ((k, v) in pairs) + { + when (v) + { + null -> node.putNull(k) + is Int -> node.put(k, v) + is Long -> node.put(k, v) + is Boolean -> node.put(k, v) + is Double -> node.put(k, v) + else -> node.put(k, v.toString()) + } + } + return node + } + + private fun rowsResponse(rows: List): Response = + Response.builder() + .isSuccessful(true) + .columns(rows.toList()) + .build() + + private fun countResponse(total: Long): Response = + Response.builder() + .isSuccessful(true) + .columns(listOf(mapper.createObjectNode().put("total", total))) + .build() +} diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt new file mode 100644 index 0000000000..7ec8d31309 --- /dev/null +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt @@ -0,0 +1,184 @@ +package io.edurt.datacap.test.local.support + +import io.edurt.datacap.spi.PluginService +import io.edurt.datacap.spi.PluginType +import io.edurt.datacap.spi.adapter.BatchWriter +import io.edurt.datacap.spi.adapter.RowCallback +import io.edurt.datacap.spi.model.Configure +import io.edurt.datacap.spi.model.Response +import java.lang.reflect.InvocationHandler +import java.lang.reflect.Proxy +import java.sql.Statement +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicInteger + +/** + * 纯内存的 PluginService 假实现，专门用于 LocalExecutorService 的特性测试（characterization test）。 + * 不连接任何真实数据库，把流式读、批量写、execute()、以及 JDBC Statement.cancel() 全部做成可控 / 可观测。 + * + * A network-free fake PluginService used to pin down the current behaviour of LocalExecutorService. + * Streaming reads, batch writes, execute() and JDBC Statement.cancel() are all controllable and observable. + * + * 作为“源端”（source）时使用： + * - streaming=true -> executeStream() 会把 [streamRows] 按 [headers] 顺序逐行回调 + * - streaming=false -> 走 execute()，由 [executeHandler] 决定返回什么（用于 legacy 全量读 & pre-count） + * + * 作为“汇端”（sink）时使用： + * - streaming=true -> openBatchWriter() 返回 [CapturingBatchWriter]，把落库行收集到 [committedRows] + * - streaming=false -> 走 execute()，[executedSql] 收集所有拼出来的 INSERT 语句 + */ +class FakePluginService( + private val streaming: Boolean = true, + private val headers: List = emptyList(), + private val streamRows: List> = emptyList(), + private val types: List = headers.map { "VARCHAR" }, + private val executeHandler: ((String) -> Response)? = null +) : PluginService +{ + /** 汇端：真正“落库”的行，按写入顺序（已投影为目标列顺序） */ + val committedRows: CopyOnWriteArrayList> = CopyOnWriteArrayList() + + /** 汇端：本 fake 打开过的所有 batch writer，便于断言 flush / writtenCount */ + val batchWriters: CopyOnWriteArrayList = CopyOnWriteArrayList() + + /** 汇端：execute() 收到的所有 SQL（legacy 双端非流式路径的 INSERT 字符串） */ + val executedSql: CopyOnWriteArrayList = CopyOnWriteArrayList() + + /** 源端：底层 Statement.cancel() 被调用的次数 */ + val cancelCount: AtomicInteger = AtomicInteger(0) + + /** 源端：在回调第 i 行之前触发的钩子，测试可在这里调用 service.stop() 模拟“流式中途停止” */ + @Volatile + var beforeRow: ((Int) -> Unit)? = null + + /** 源端：所有行回调完成之后触发的钩子，用于模拟“驱动在 cancel 后让 rs.next() 直接返回 false” */ + @Volatile + var afterRows: (() -> Unit)? = null + + /** 源端：在即将回调第 [throwAtRow] 行时抛异常，模拟“驱动 cancel 后 fetch 抛 SQLException” */ + @Volatile + var throwAtRow: Int = -1 + + override fun supportsStreaming(): Boolean = streaming + + override fun type(): PluginType = if (streaming) PluginType.JDBC else PluginType.HTTP + + override fun executeStream(configure: Configure, content: String, fetchSize: Int, callback: RowCallback) + { + callback.onStatement(newFakeStatement()) + callback.onSchema(headers, types) + for (i in streamRows.indices) + { + beforeRow?.invoke(i) + if (throwAtRow == i) + { + throw RuntimeException("Simulated driver abort after cancel") + } + callback.onRow(streamRows[i]) + } + afterRows?.invoke() + } + + override fun openBatchWriter( + configure: Configure, + database: String, + table: String, + columns: List, + batchSize: Int + ): BatchWriter + { + val writer = CapturingBatchWriter(columns, batchSize, committedRows) + batchWriters.add(writer) + return writer + } + + override fun execute(configure: Configure, content: String): Response + { + executedSql.add(content) + return executeHandler?.invoke(content) + ?: Response.builder() + .isSuccessful(true) + .columns(emptyList()) + .build() + } + + /** + * 用动态代理造一个只实现了 isClosed()/cancel() 的假 Statement， + * 其余方法返回各自类型的零值，够 LocalExecutorService.stop() 使用。 + */ + private fun newFakeStatement(): Statement + { + val handler = InvocationHandler { _, method, _ -> + when (method.name) + { + "isClosed" -> false + "cancel" -> + { + cancelCount.incrementAndGet() + null + } + + else -> when (method.returnType) + { + java.lang.Boolean.TYPE -> false + java.lang.Integer.TYPE -> 0 + java.lang.Long.TYPE -> 0L + else -> null + } + } + } + return Proxy.newProxyInstance( + javaClass.classLoader, + arrayOf(Statement::class.java), + handler + ) as Statement + } +} + +/** + * 内存版 BatchWriter：模拟 JdbcBatchWriter 的“攒够 batchSize 才 flush、close 时 flush 剩余”语义， + * writtenCount() 只统计已 flush（已提交）的行数，与真实实现保持一致。 + */ +class CapturingBatchWriter( + private val columns: List, + private val batchSize: Int, + private val sink: MutableList> +) : BatchWriter +{ + private val buffer: MutableList> = ArrayList() + private var committed: Long = 0L + + /** 累计调用 flush() 的次数，便于断言 batch 切分行为 */ + var flushCount: Int = 0 + private set + + override fun addRow(row: List<*>) + { + require(row.size == columns.size) { + "Row size ${row.size} does not match column count ${columns.size}" + } + buffer.add(ArrayList(row)) + if (buffer.size >= batchSize) + { + flush() + } + } + + override fun writtenCount(): Long = committed + + override fun close() + { + if (buffer.isNotEmpty()) + { + flush() + } + } + + private fun flush() + { + sink.addAll(buffer) + committed += buffer.size + buffer.clear() + flushCount++ + } +} From ab76f136dfcd482fe4ea8ff48a67dd9f026b0afb Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 09:20:06 +0800 Subject: [PATCH 02/17] refactor(executor): split LocalExecutorService into sync strategies Extract the 576-line class into focused units without behaviour change: - SyncStrategy + SyncContext with StreamingSyncStrategy / LegacySyncStrategy - TaskRegistry (object) for the running-task map and cancel executor, kept a singleton so stop() still works across service instances - TaskHandle / TaskCancelledException and ValueCodec for value/SQL codecs start() now only resolves inputs, runs pre-count and picks a strategy; stop() delegates to TaskRegistry. Characterization suite stays green and spotbugs reports no findings. --- .../executor/local/LegacySyncStrategy.kt | 145 +++++++ .../executor/local/LocalExecutorService.kt | 385 ++---------------- .../executor/local/StreamingSyncStrategy.kt | 101 +++++ .../datacap/executor/local/SyncStrategy.kt | 50 +++ .../datacap/executor/local/TaskHandle.kt | 33 ++ .../datacap/executor/local/TaskRegistry.kt | 46 +++ .../datacap/executor/local/ValueCodec.kt | Bin 0 -> 2462 bytes 7 files changed, 407 insertions(+), 353 deletions(-) create mode 100644 executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt create mode 100644 executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt create mode 100644 executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt create mode 100644 executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt create mode 100644 executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt create mode 100644 executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/ValueCodec.kt diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt new file mode 100644 index 0000000000..85c6cbd1a0 --- /dev/null +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt @@ -0,0 +1,145 @@ +package io.edurt.datacap.executor.local + +import com.fasterxml.jackson.databind.node.ObjectNode +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.edurt.datacap.common.sql.SqlBuilder +import io.edurt.datacap.common.sql.configure.SqlBody +import io.edurt.datacap.common.sql.configure.SqlColumn +import io.edurt.datacap.common.sql.configure.SqlType +import io.edurt.datacap.spi.PluginService +import io.edurt.datacap.spi.model.Configure + +/** + * 回退路径：源或汇任一不支持流式（如 HTTP / Native 插件）。仍然使用旧的全量读取， + * 但目标端按 batch 切片提交，避免一次性拼接巨大 SQL 字符串；同时修复 NULL、类型、转义问题。 + * + * 内部再按“目标端是否支持流式”分两条子路：BatchWriter / 拼 INSERT 字符串。 + * 逻辑从 LocalExecutorService.runLegacy 原样迁移，行为不变。 + */ +@SuppressFBWarnings(value = ["BC_BAD_CAST_TO_ABSTRACT_COLLECTION", "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"]) +internal class LegacySyncStrategy : SyncStrategy +{ + override fun sync(context: SyncContext): Long + { + val inputPlugin = context.inputPlugin + val inputConfigure = context.inputConfigure + val query = context.query + val outputPlugin = context.outputPlugin + val outputConfigure = context.outputConfigure + val database = context.database + val table = context.table + val originColumns = context.originColumns + val batchSize = context.batchSize + val taskLog = context.taskLog + val totalCount = context.totalCount + val progressListener = context.progressListener + val handle = context.handle + + taskLog.info("Legacy sync start: target=`{}`.`{}` batchSize={}", database, table, batchSize) + val startNanos = System.nanoTime() + val inputResult = inputPlugin.execute(inputConfigure, query) + if (inputResult.isSuccessful != true) + { + throw RuntimeException(inputResult.message ?: "Input plugin failed") + } + val rows = inputResult.columns ?: return 0L + taskLog.info("Legacy sync: source materialized {} rows", rows.size) + // 全量路径已经能拿到精确总数，覆盖 pre-count 的估算 + val effectiveTotal = if (totalCount >= 0) totalCount else rows.size.toLong() + + // 目标端能流式：用 BatchWriter 安全 + 节省内存 + if (outputPlugin.supportsStreaming()) + { + val targetColumns = originColumns.map { it.name } + val sourceKeys = originColumns.map { it.original } + var written = 0L + val writer = outputPlugin.openBatchWriter( + outputConfigure, database, table, targetColumns, batchSize + ) + writer.use { writer -> + for (item in rows) + { + if (handle.cancelled.get()) + { + throw TaskCancelledException(written) + } + val node = item as? ObjectNode ?: continue + val projected = ArrayList(sourceKeys.size) + for (key in sourceKeys) + { + projected.add(ValueCodec.jsonNodeToJdbcValue(node.get(key))) + } + writer.addRow(projected) + written ++ + if (written % PROGRESS_INTERVAL == 0L) + { + taskLog.info("Legacy sync progress: written={} committed={}", written, writer.writtenCount()) + progressListener?.onProgress(writer.writtenCount(), effectiveTotal) + } + } + } + val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 + taskLog.info("Legacy sync done: rows={} committed={} elapsed={}s", written, writer.writtenCount(), "%.1f".format(totalSeconds)) + return written + } + + // 双端都不支持流式：用 INSERT 字符串，但按 batch 提交，不再拼一个巨大字符串 + var written = 0L + val batch = ArrayList(batchSize) + for (item in rows) + { + if (handle.cancelled.get()) + { + throw TaskCancelledException(written) + } + val node = item as? ObjectNode ?: continue + val sqlColumns = ArrayList(originColumns.size) + for (col in originColumns) + { + sqlColumns.add( + SqlColumn.builder() + .column("`${col.name}`") + .value(ValueCodec.formatSqlLiteral(node.get(col.original))) + .build() + ) + } + val body = SqlBody.builder() + .type(SqlType.INSERT) + .database(database) + .table(table) + .columns(sqlColumns) + .build() + batch.add(SqlBuilder(body).sql) + if (batch.size >= batchSize) + { + flushLegacyBatch(outputPlugin, outputConfigure, batch) + written += batch.size + batch.clear() + if (written % PROGRESS_INTERVAL == 0L) + { + taskLog.info("Legacy sync progress (sql batch): written={}", written) + progressListener?.onProgress(written, effectiveTotal) + } + } + } + if (batch.isNotEmpty()) + { + flushLegacyBatch(outputPlugin, outputConfigure, batch) + written += batch.size + batch.clear() + } + val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 + taskLog.info("Legacy sync done: rows={} elapsed={}s", written, "%.1f".format(totalSeconds)) + return written + } + + private fun flushLegacyBatch(plugin: PluginService, configure: Configure, batch: List) + { + val joined = batch.joinToString("\n") + val result = plugin.execute(configure, joined) + if (result.isSuccessful != true) + { + throw RuntimeException(result.message ?: "Output plugin failed") + } + } +} diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt index 135bc5da7a..4565555587 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt @@ -1,49 +1,29 @@ package io.edurt.datacap.executor.local -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.ObjectNode import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.edurt.datacap.common.sql.SqlBuilder -import io.edurt.datacap.common.sql.configure.SqlBody -import io.edurt.datacap.common.sql.configure.SqlColumn -import io.edurt.datacap.common.sql.configure.SqlType import io.edurt.datacap.executor.ExecutorService import io.edurt.datacap.executor.common.RunState -import io.edurt.datacap.executor.configure.ExecutorProgressListener import io.edurt.datacap.executor.configure.ExecutorRequest import io.edurt.datacap.executor.configure.ExecutorResponse -import io.edurt.datacap.executor.configure.OriginColumn import io.edurt.datacap.lib.logger.LoggerExecutor import io.edurt.datacap.lib.logger.logback.LogbackExecutor import io.edurt.datacap.spi.PluginService -import io.edurt.datacap.spi.adapter.BatchWriter -import io.edurt.datacap.spi.adapter.RowCallback import io.edurt.datacap.spi.model.Configure import org.slf4j.Logger import org.slf4j.LoggerFactory -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicBoolean -@SuppressFBWarnings(value = ["BC_BAD_CAST_TO_ABSTRACT_COLLECTION", "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", "MS_MUTABLE_COLLECTION_PKGPROTECT"]) +/** + * 本地执行器：在 DataCap 进程内把“源查询结果”搬运到“目标表”，支持流式与回退两种同步策略、 + * 以及按 taskName 的运行时停止。 + * + * 编排职责保留在这里；具体搬运逻辑委托给 [SyncStrategy]（[StreamingSyncStrategy] / [LegacySyncStrategy]）， + * 活跃任务与取消线程池委托给 [TaskRegistry]，值/字面量转换委托给 [ValueCodec]。 + */ +@SuppressFBWarnings(value = ["RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", "BC_BAD_CAST_TO_ABSTRACT_COLLECTION"]) class LocalExecutorService : ExecutorService { private val log = LoggerFactory.getLogger(LocalExecutorService::class.java) - /** - * 已注册的活跃任务句柄。 - * - cancelled: 调度循环每条行检查 - * - sourceStatement: stop() 会调 cancel() 立即终止源端 JDBC 查询，不必等下一行 - * - taskLog: 让 stop() 把"用户停止"事件写到任务专属日志里 - * - rowsAtStop: 记录被停止时已处理的行数，供 history 落库 - */ - private class TaskHandle - { - val cancelled: AtomicBoolean = AtomicBoolean(false) - @Volatile var sourceStatement: java.sql.Statement? = null - @Volatile var taskLog: Logger? = null - @Volatile var rowsAtStop: Long = 0L - } - override fun start(request: ExecutorRequest): ExecutorResponse { val response = ExecutorResponse() @@ -53,7 +33,7 @@ class LocalExecutorService : ExecutorService handle.taskLog = taskLog if (request.taskName.isNotBlank()) { - runningTasks[request.taskName] = handle + TaskRegistry.register(request.taskName, handle) } try { @@ -75,8 +55,6 @@ class LocalExecutorService : ExecutorService { throw IllegalArgumentException("Input column mapping is empty") } - val targetColumns = originColumns.map { it.name } - val sourceKeys = originColumns.map { it.original } val fetchSize = if (request.fetchSize > 0) request.fetchSize else DEFAULT_FETCH_SIZE val batchSize = if (request.batchSize > 0) request.batchSize else DEFAULT_BATCH_SIZE @@ -100,24 +78,26 @@ class LocalExecutorService : ExecutorService progressListener?.onProgress(0L, totalCount) } - val written = if (inputPlugin.supportsStreaming() && outputPlugin.supportsStreaming()) - { - runStreaming( - inputPlugin, inputConfigure, query, fetchSize, - outputPlugin, outputConfigure, database, table, - targetColumns, sourceKeys, batchSize, taskLog, - totalCount, progressListener, handle - ) - } - else - { - runLegacy( - inputPlugin, inputConfigure, query, - outputPlugin, outputConfigure, database, table, - originColumns, batchSize, taskLog, - totalCount, progressListener, handle - ) - } + val context = SyncContext( + inputPlugin, inputConfigure, query, fetchSize, + outputPlugin, outputConfigure, database, table, + originColumns, + originColumns.map { it.name }, + originColumns.map { it.original }, + batchSize, taskLog, totalCount, progressListener, handle + ) + + val strategy: SyncStrategy = + if (inputPlugin.supportsStreaming() && outputPlugin.supportsStreaming()) + { + StreamingSyncStrategy() + } + else + { + LegacySyncStrategy() + } + + val written = strategy.sync(context) response.count = if (written > Int.MAX_VALUE.toLong()) Int.MAX_VALUE else written.toInt() response.successful = true @@ -158,16 +138,13 @@ class LocalExecutorService : ExecutorService { if (request.taskName.isNotBlank()) { - runningTasks.remove(request.taskName, handle) + TaskRegistry.unregister(request.taskName, handle) } loggerExecutor?.destroy() } return response } - /** 取消传播专用异常，携带已处理行数便于上游记录 */ - private class TaskCancelledException(val processed: Long) : RuntimeException("Task cancelled by user") - private fun newTaskLogger(request: ExecutorRequest): LoggerExecutor<*>? { val workHome = request.workHome @@ -199,7 +176,7 @@ class LocalExecutorService : ExecutorService { return ExecutorResponse(false, false, RunState.FAILURE, "taskName is required") } - val handle = runningTasks[taskName] + val handle = TaskRegistry.find(taskName) if (handle == null) { return ExecutorResponse(false, false, RunState.FAILURE, "Task [ $taskName ] is not running on this node") @@ -213,7 +190,7 @@ class LocalExecutorService : ExecutorService val stmt = handle.sourceStatement if (stmt != null) { - cancelExecutor.submit { + TaskRegistry.submitCancel { try { if (!stmt.isClosed) @@ -270,307 +247,9 @@ class LocalExecutorService : ExecutorService } } - /** - * 流式路径：源端 fetchSize 拉取，目标端 PreparedStatement 批量写。 - * 全程不在 JVM 内物化整个结果集。 - */ - private fun runStreaming( - inputPlugin: PluginService, - inputConfigure: Configure, - query: String, - fetchSize: Int, - outputPlugin: PluginService, - outputConfigure: Configure, - database: String, - table: String, - targetColumns: List, - sourceKeys: List, - batchSize: Int, - taskLog: Logger, - totalCount: Long, - progressListener: ExecutorProgressListener?, - handle: TaskHandle - ): Long - { - taskLog.info( - "Streaming sync start: target=`{}`.`{}` columns={} fetchSize={} batchSize={} total={}", - database, table, targetColumns.size, fetchSize, batchSize, totalCount - ) - val startNanos = System.nanoTime() - var written = 0L - val writer: BatchWriter = outputPlugin.openBatchWriter( - outputConfigure, database, table, targetColumns, batchSize - ) - writer.use { writer -> - val indexByHeader = HashMap() - inputPlugin.executeStream(inputConfigure, query, fetchSize, object : RowCallback - { - override fun onSchema(headers: List, types: List) - { - indexByHeader.clear() - headers.forEachIndexed { i, h -> indexByHeader[h.lowercase()] = i } - taskLog.info("Streaming sync: source returned headers={}", headers) - } - - override fun onStatement(statement: java.sql.Statement) - { - handle.sourceStatement = statement - } - - override fun onRow(row: List) - { - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } - val projected = ArrayList(sourceKeys.size) - for (key in sourceKeys) - { - val idx = indexByHeader[key.lowercase()] - ?: throw IllegalStateException("Source column '$key' not found in query result") - projected.add(row[idx]) - } - writer.addRow(projected) - written ++ - if (written % PROGRESS_INTERVAL == 0L) - { - val seconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - val rps = if (seconds > 0) (written / seconds).toLong() else 0L - taskLog.info( - "Streaming sync progress: read={} committed={} elapsed={}s rps={}", - written, writer.writtenCount(), "%.1f".format(seconds), rps - ) - progressListener?.onProgress(writer.writtenCount(), totalCount) - // 顺手记录最近一次进度行数，给 stop 后 catch 取整时使用 - handle.rowsAtStop = writer.writtenCount() - } - } - }) - } - // 某些驱动在 Statement.cancel() 后直接让 rs.next() 返回 false（不抛异常），需要在循环结束后再判一次 - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } - val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - taskLog.info( - "Streaming sync done: rows={} committed={} elapsed={}s", - written, writer.writtenCount(), "%.1f".format(totalSeconds) - ) - return written - } - - /** - * 回退路径：源或汇任一不支持流式（如 HTTP / Native 插件）。仍然使用旧的全量读取， - * 但目标端按 batch 切片提交，避免一次性拼接巨大 SQL 字符串；同时修复 NULL、类型、转义问题。 - */ - private fun runLegacy( - inputPlugin: PluginService, - inputConfigure: Configure, - query: String, - outputPlugin: PluginService, - outputConfigure: Configure, - database: String, - table: String, - originColumns: List, - batchSize: Int, - taskLog: Logger, - totalCount: Long, - progressListener: ExecutorProgressListener?, - handle: TaskHandle - ): Long - { - taskLog.info("Legacy sync start: target=`{}`.`{}` batchSize={}", database, table, batchSize) - val startNanos = System.nanoTime() - val inputResult = inputPlugin.execute(inputConfigure, query) - if (inputResult.isSuccessful != true) - { - throw RuntimeException(inputResult.message ?: "Input plugin failed") - } - val rows = inputResult.columns ?: return 0L - taskLog.info("Legacy sync: source materialized {} rows", rows.size) - // 全量路径已经能拿到精确总数，覆盖 pre-count 的估算 - val effectiveTotal = if (totalCount >= 0) totalCount else rows.size.toLong() - - // 目标端能流式：用 BatchWriter 安全 + 节省内存 - if (outputPlugin.supportsStreaming()) - { - val targetColumns = originColumns.map { it.name } - val sourceKeys = originColumns.map { it.original } - var written = 0L - val writer = outputPlugin.openBatchWriter( - outputConfigure, database, table, targetColumns, batchSize - ) - writer.use { writer -> - for (item in rows) - { - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } - val node = item as? ObjectNode ?: continue - val projected = ArrayList(sourceKeys.size) - for (key in sourceKeys) - { - projected.add(jsonNodeToJdbcValue(node.get(key))) - } - writer.addRow(projected) - written ++ - if (written % PROGRESS_INTERVAL == 0L) - { - taskLog.info("Legacy sync progress: written={} committed={}", written, writer.writtenCount()) - progressListener?.onProgress(writer.writtenCount(), effectiveTotal) - } - } - } - val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - taskLog.info("Legacy sync done: rows={} committed={} elapsed={}s", written, writer.writtenCount(), "%.1f".format(totalSeconds)) - return written - } - - // 双端都不支持流式：用 INSERT 字符串，但按 batch 提交，不再拼一个巨大字符串 - var written = 0L - val batch = ArrayList(batchSize) - for (item in rows) - { - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } - val node = item as? ObjectNode ?: continue - val sqlColumns = ArrayList(originColumns.size) - for (col in originColumns) - { - sqlColumns.add( - SqlColumn.builder() - .column("`${col.name}`") - .value(formatSqlLiteral(node.get(col.original))) - .build() - ) - } - val body = SqlBody.builder() - .type(SqlType.INSERT) - .database(database) - .table(table) - .columns(sqlColumns) - .build() - batch.add(SqlBuilder(body).sql) - if (batch.size >= batchSize) - { - flushLegacyBatch(outputPlugin, outputConfigure, batch) - written += batch.size - batch.clear() - if (written % PROGRESS_INTERVAL == 0L) - { - taskLog.info("Legacy sync progress (sql batch): written={}", written) - progressListener?.onProgress(written, effectiveTotal) - } - } - } - if (batch.isNotEmpty()) - { - flushLegacyBatch(outputPlugin, outputConfigure, batch) - written += batch.size - batch.clear() - } - val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - taskLog.info("Legacy sync done: rows={} elapsed={}s", written, "%.1f".format(totalSeconds)) - return written - } - - private fun flushLegacyBatch(plugin: PluginService, configure: Configure, batch: List) - { - val joined = batch.joinToString("\n") - val result = plugin.execute(configure, joined) - if (result.isSuccessful != true) - { - throw RuntimeException(result.message ?: "Output plugin failed") - } - } - - /** - * 把 JsonNode 转成 JDBC 可识别的真实类型；保留 NULL 语义。 - */ - private fun jsonNodeToJdbcValue(node: JsonNode?): Any? - { - if (node == null || node.isNull) return null - return when - { - node.isBoolean -> node.asBoolean() - node.isInt -> node.asInt() - node.isLong -> node.asLong() - node.isBigInteger -> node.bigIntegerValue() - node.isFloat || node.isDouble -> node.asDouble() - node.isBigDecimal -> node.decimalValue() - node.isBinary -> - { - try - { - node.binaryValue() - } - catch (e: Exception) - { - node.asText() - } - } - - else -> node.asText() - } - } - - /** - * 旧路径下需要拼 SQL 字符串：按类型生成字面量，正确处理 NULL / 数字 / 布尔 / 字符串转义。 - * 仅在双端均不支持流式（HTTP / Native 输出插件）时使用。 - */ - private fun formatSqlLiteral(node: JsonNode?): String - { - if (node == null || node.isNull) return "NULL" - return when - { - node.isBoolean -> if (node.asBoolean()) "TRUE" else "FALSE" - node.isIntegralNumber -> node.asLong().toString() - node.isFloatingPointNumber || node.isBigDecimal -> node.asText() - else -> "'${escapeSqlString(node.asText())}'" - } - } - - private fun escapeSqlString(s: String): String - { - if (s.isEmpty()) return s - val sb = StringBuilder(s.length + 4) - for (c in s) - { - when (c) - { - '\'' -> sb.append("''") - '\\' -> sb.append("\\\\") - '\u0000' -> - { - // SQL 不允许 NUL 字符，跳过以避免协议层错误 - } - - else -> sb.append(c) - } - } - return sb.toString() - } - companion object { private const val DEFAULT_FETCH_SIZE = 1000 private const val DEFAULT_BATCH_SIZE = 1000 - private const val PROGRESS_INTERVAL = 1_000L - - // 进程内活跃任务表。多个并发 sync 共享同一 LocalExecutorService 实例， - // 用 taskName 唯一索引；stop() 通过 taskName 查到 handle 后设置取消标志位 - private val runningTasks: ConcurrentHashMap = ConcurrentHashMap() - - // Statement.cancel() 可能会阻塞（驱动会新开连接发 KILL），放到独立线程跑避免拖住调用方 - private val cancelExecutor: java.util.concurrent.ExecutorService = - java.util.concurrent.Executors.newCachedThreadPool { r -> - val t = Thread(r, "local-executor-cancel") - t.isDaemon = true - t - } } } diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt new file mode 100644 index 0000000000..e2d016aa08 --- /dev/null +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt @@ -0,0 +1,101 @@ +package io.edurt.datacap.executor.local + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.edurt.datacap.spi.adapter.BatchWriter +import io.edurt.datacap.spi.adapter.RowCallback + +/** + * 流式路径：源端 fetchSize 拉取，目标端 PreparedStatement 批量写。 + * 全程不在 JVM 内物化整个结果集。 + * + * 仅当源与汇【都】支持流式时选用。逻辑从 LocalExecutorService.runStreaming 原样迁移，行为不变。 + */ +@SuppressFBWarnings(value = ["RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"]) +internal class StreamingSyncStrategy : SyncStrategy +{ + override fun sync(context: SyncContext): Long + { + val inputPlugin = context.inputPlugin + val inputConfigure = context.inputConfigure + val query = context.query + val fetchSize = context.fetchSize + val outputPlugin = context.outputPlugin + val outputConfigure = context.outputConfigure + val database = context.database + val table = context.table + val targetColumns = context.targetColumns + val sourceKeys = context.sourceKeys + val batchSize = context.batchSize + val taskLog = context.taskLog + val totalCount = context.totalCount + val progressListener = context.progressListener + val handle = context.handle + + taskLog.info( + "Streaming sync start: target=`{}`.`{}` columns={} fetchSize={} batchSize={} total={}", + database, table, targetColumns.size, fetchSize, batchSize, totalCount + ) + val startNanos = System.nanoTime() + var written = 0L + val writer: BatchWriter = outputPlugin.openBatchWriter( + outputConfigure, database, table, targetColumns, batchSize + ) + writer.use { writer -> + val indexByHeader = HashMap() + inputPlugin.executeStream(inputConfigure, query, fetchSize, object : RowCallback + { + override fun onSchema(headers: List, types: List) + { + indexByHeader.clear() + headers.forEachIndexed { i, h -> indexByHeader[h.lowercase()] = i } + taskLog.info("Streaming sync: source returned headers={}", headers) + } + + override fun onStatement(statement: java.sql.Statement) + { + handle.sourceStatement = statement + } + + override fun onRow(row: List) + { + if (handle.cancelled.get()) + { + throw TaskCancelledException(written) + } + val projected = ArrayList(sourceKeys.size) + for (key in sourceKeys) + { + val idx = indexByHeader[key.lowercase()] + ?: throw IllegalStateException("Source column '$key' not found in query result") + projected.add(row[idx]) + } + writer.addRow(projected) + written ++ + if (written % PROGRESS_INTERVAL == 0L) + { + val seconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 + val rps = if (seconds > 0) (written / seconds).toLong() else 0L + taskLog.info( + "Streaming sync progress: read={} committed={} elapsed={}s rps={}", + written, writer.writtenCount(), "%.1f".format(seconds), rps + ) + progressListener?.onProgress(writer.writtenCount(), totalCount) + // 顺手记录最近一次进度行数，给 stop 后 catch 取整时使用 + handle.rowsAtStop = writer.writtenCount() + } + } + }) + } + // 某些驱动在 Statement.cancel() 后直接让 rs.next() 返回 false（不抛异常），需要在循环结束后再判一次 + if (handle.cancelled.get()) + { + throw TaskCancelledException(written) + } + val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 + taskLog.info( + "Streaming sync done: rows={} committed={} elapsed={}s", + written, writer.writtenCount(), "%.1f".format(totalSeconds) + ) + return written + } +} diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt new file mode 100644 index 0000000000..13e23d0c42 --- /dev/null +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt @@ -0,0 +1,50 @@ +package io.edurt.datacap.executor.local + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.edurt.datacap.executor.configure.ExecutorProgressListener +import io.edurt.datacap.executor.configure.OriginColumn +import io.edurt.datacap.spi.PluginService +import io.edurt.datacap.spi.model.Configure +import org.slf4j.Logger + +/** 每处理多少行上报一次进度 / 打一次日志。行为与重构前保持一致。 */ +internal const val PROGRESS_INTERVAL = 1_000L + +/** + * 一次同步任务所需的全部上下文。由 LocalExecutorService.start() 组装，交给具体 [SyncStrategy]。 + * 把原先 runStreaming / runLegacy 的一长串形参收敛到这里，行为不变，只是不再逐个透传。 + * + * targetColumns / sourceKeys 是从 [originColumns] 预先派生的（name / original）， + * 避免各策略重复计算。 + */ +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"]) +internal class SyncContext( + val inputPlugin: PluginService, + val inputConfigure: Configure, + val query: String, + val fetchSize: Int, + val outputPlugin: PluginService, + val outputConfigure: Configure, + val database: String, + val table: String, + val originColumns: List, + val targetColumns: List, + val sourceKeys: List, + val batchSize: Int, + val taskLog: Logger, + val totalCount: Long, + val progressListener: ExecutorProgressListener?, + val handle: TaskHandle +) + +/** + * 同步策略：把“源 -> 汇”的一次搬运抽象出来，返回已处理（读入）行数。 + * 取消时应抛出 [TaskCancelledException]（携带已处理行数），由 start() 统一转成 STOPPED。 + * + * A single "source -> sink" copy. Returns the number of rows read. On cancellation it throws + * [TaskCancelledException] carrying the processed count, which start() maps to STOPPED. + */ +internal interface SyncStrategy +{ + fun sync(context: SyncContext): Long +} diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt new file mode 100644 index 0000000000..7eac9a55dc --- /dev/null +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt @@ -0,0 +1,33 @@ +package io.edurt.datacap.executor.local + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.slf4j.Logger +import java.util.concurrent.atomic.AtomicBoolean + +/** + * 已注册的活跃任务句柄。 + * - cancelled: 调度循环每条行检查 + * - sourceStatement: stop() 会调 cancel() 立即终止源端 JDBC 查询，不必等下一行 + * - taskLog: 让 stop() 把"用户停止"事件写到任务专属日志里 + * - rowsAtStop: 记录被停止时已处理的行数，供 history 落库 + * + * An in-flight task handle shared between start() and stop(). Extracted verbatim from the former + * inner class in LocalExecutorService; behaviour is unchanged. + */ +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"]) +internal class TaskHandle +{ + val cancelled: AtomicBoolean = AtomicBoolean(false) + + @Volatile + var sourceStatement: java.sql.Statement? = null + + @Volatile + var taskLog: Logger? = null + + @Volatile + var rowsAtStop: Long = 0L +} + +/** 取消传播专用异常，携带已处理行数便于上游记录 */ +internal class TaskCancelledException(val processed: Long) : RuntimeException("Task cancelled by user") diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt new file mode 100644 index 0000000000..c8de1ee8e0 --- /dev/null +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt @@ -0,0 +1,46 @@ +package io.edurt.datacap.executor.local + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executors + +/** + * 进程内活跃任务表 + 取消线程池。 + * + * 多个并发 sync 共享同一实例（Kotlin object 单例），用 taskName 唯一索引； + * stop() 通过 taskName 查到 handle 后设置取消标志位。 + * 之前这些是 LocalExecutorService 的 companion static 成员——为了让不同 LocalExecutorService + * 实例之间的 stop() 仍能互相找到任务，这里必须保持“单例、跨实例共享”的语义，故用 object。 + * + * Statement.cancel() 可能会阻塞（驱动会新开连接发 KILL），放到独立线程跑避免拖住调用方。 + */ +@SuppressFBWarnings(value = ["RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"]) +internal object TaskRegistry +{ + private val runningTasks: ConcurrentHashMap = ConcurrentHashMap() + + private val cancelExecutor: java.util.concurrent.ExecutorService = + Executors.newCachedThreadPool { r -> + val t = Thread(r, "local-executor-cancel") + t.isDaemon = true + t + } + + fun register(taskName: String, handle: TaskHandle) + { + runningTasks[taskName] = handle + } + + /** 仅当当前登记的仍是同一个 handle 时才移除，避免误删同名的后续任务 */ + fun unregister(taskName: String, handle: TaskHandle) + { + runningTasks.remove(taskName, handle) + } + + fun find(taskName: String): TaskHandle? = runningTasks[taskName] + + fun submitCancel(block: () -> Unit) + { + cancelExecutor.submit(block) + } +} diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/ValueCodec.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/ValueCodec.kt new file mode 100644 index 0000000000000000000000000000000000000000..7812dd3c1a491b3cb98b8cb00fc26ae560cefc7f GIT binary patch literal 2462 zcmbVOT~FIq7~XY$#ZfdV3EP}q>|T0Q4Wr0YD){D z1Gcg2x#}exT)ctYkhWJ{;WBq6m5iA@iQ=#}sTr9R81SGIAQASSdYF!pneJP=T52xR zQJJW;9o()IUYCAIJDIA#J0~^%X2GnlHFxHnrzLBx(NZY;#Z$BX=VfEtT3_*~WN+>P z-2(9frLhiwvwuC}`wRAi`yRiN2=oYCK2mW$Xyc|}&WWd)B@`2JV3U#Wi+s|u! zcgd`mt<9RXUb0@7c%yRJSpH&c>=tAg0X3Q?$a(V$oNKzHAHR}^G6 zl`pW_cp^|*8eGdXNC?fK0Y84QsHjoRR=Gj5Bbio(Pyj*dPj`*v4&CQ2StrkGzH2^J0GJLE=)1P*W jr*OGCV3+g=r`95b Date: Fri, 4 Sep 2026 09:30:14 +0800 Subject: [PATCH 03/17] fix(executor): report committed row count when a sync is stopped Unify cancellation behind a single CancellationToken and a runCancelable writer wrapper, collapsing the three former detection points (in-loop throw, post-loop flag, driver-abort exception) into one path that always surfaces TaskCancelledException with the committed count. Previously a stop that aborted the JDBC fetch reported count=0 because it read the stale rowsAtStop counter (updated only every 1000 rows); it now reports the rows actually committed to the target. Row counts are unified on writtenCount() across success and cancellation. Also precompute source-column indexes at onSchema instead of a per-row map lookup in the streaming path. --- .../executor/local/LegacySyncStrategy.kt | 41 ++++++------- .../executor/local/LocalExecutorService.kt | 13 +++-- .../executor/local/StreamingSyncStrategy.kt | 58 +++++++++---------- .../datacap/executor/local/SyncStrategy.kt | 40 +++++++++++++ .../datacap/executor/local/TaskHandle.kt | 42 +++++++++++--- .../LocalExecutorCharacterizationTest.kt | 11 ++-- 6 files changed, 132 insertions(+), 73 deletions(-) diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt index 85c6cbd1a0..1fabc1b2b7 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt @@ -14,7 +14,8 @@ import io.edurt.datacap.spi.model.Configure * 但目标端按 batch 切片提交，避免一次性拼接巨大 SQL 字符串；同时修复 NULL、类型、转义问题。 * * 内部再按“目标端是否支持流式”分两条子路：BatchWriter / 拼 INSERT 字符串。 - * 逻辑从 LocalExecutorService.runLegacy 原样迁移，行为不变。 + * 取消处理与流式路径一致：BatchWriter 子路由 [runCancelable] 收敛为“已提交行数”； + * SQL 子路的 written 本身就是已 flush（已提交）行数。 */ @SuppressFBWarnings(value = ["BC_BAD_CAST_TO_ABSTRACT_COLLECTION", "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"]) internal class LegacySyncStrategy : SyncStrategy @@ -33,7 +34,7 @@ internal class LegacySyncStrategy : SyncStrategy val taskLog = context.taskLog val totalCount = context.totalCount val progressListener = context.progressListener - val handle = context.handle + val token = context.handle.cancellation taskLog.info("Legacy sync start: target=`{}`.`{}` batchSize={}", database, table, batchSize) val startNanos = System.nanoTime() @@ -50,48 +51,44 @@ internal class LegacySyncStrategy : SyncStrategy // 目标端能流式：用 BatchWriter 安全 + 节省内存 if (outputPlugin.supportsStreaming()) { - val targetColumns = originColumns.map { it.name } - val sourceKeys = originColumns.map { it.original } - var written = 0L + val targetColumns = context.targetColumns + val sourceKeys = context.sourceKeys + var read = 0L val writer = outputPlugin.openBatchWriter( outputConfigure, database, table, targetColumns, batchSize ) - writer.use { writer -> + writer.runCancelable(token) { w -> for (item in rows) { - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } + token.throwIfCancelled(w.writtenCount()) val node = item as? ObjectNode ?: continue val projected = ArrayList(sourceKeys.size) for (key in sourceKeys) { projected.add(ValueCodec.jsonNodeToJdbcValue(node.get(key))) } - writer.addRow(projected) - written ++ - if (written % PROGRESS_INTERVAL == 0L) + w.addRow(projected) + read ++ + if (read % PROGRESS_INTERVAL == 0L) { - taskLog.info("Legacy sync progress: written={} committed={}", written, writer.writtenCount()) - progressListener?.onProgress(writer.writtenCount(), effectiveTotal) + taskLog.info("Legacy sync progress: read={} committed={}", read, w.writtenCount()) + progressListener?.onProgress(w.writtenCount(), effectiveTotal) } } } + val committed = writer.writtenCount() val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - taskLog.info("Legacy sync done: rows={} committed={} elapsed={}s", written, writer.writtenCount(), "%.1f".format(totalSeconds)) - return written + taskLog.info("Legacy sync done: read={} committed={} elapsed={}s", read, committed, "%.1f".format(totalSeconds)) + return committed } - // 双端都不支持流式：用 INSERT 字符串，但按 batch 提交，不再拼一个巨大字符串 + // 双端都不支持流式：用 INSERT 字符串，但按 batch 提交，不再拼一个巨大字符串。 + // written 只在 flush 成功后累加，因此它就是“已提交行数”。 var written = 0L val batch = ArrayList(batchSize) for (item in rows) { - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } + token.throwIfCancelled(written) val node = item as? ObjectNode ?: continue val sqlColumns = ArrayList(originColumns.size) for (col in originColumns) diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt index 4565555587..fdacbee76c 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt @@ -116,12 +116,13 @@ class LocalExecutorService : ExecutorService } catch (ex: Exception) { - // 取消可能以包装异常的形式抛出（例如 SQLException by Statement.cancel()），通过 handle 反向判别 - if (handle.cancelled.get()) + // 正常情况下取消都会被策略归一成 TaskCancelledException（见上）。 + // 这里只兜底“取消发生在写入循环之外”（如 openBatchWriter / 全量读取阶段就被取消）： + // 此时尚无已提交行数，count 记 0。 + if (handle.cancellation.isCancelled) { - val rows = handle.rowsAtStop - taskLog.warn("Local executor task stopped by user (via JDBC cancel): rows≈{} task={}", rows, request.taskName) - response.count = if (rows > Int.MAX_VALUE.toLong()) Int.MAX_VALUE else rows.toInt() + taskLog.warn("Local executor task stopped by user (outside write loop): task={}", request.taskName) + response.count = 0 response.successful = false response.state = RunState.STOPPED response.message = "Stopped by user" @@ -182,7 +183,7 @@ class LocalExecutorService : ExecutorService return ExecutorResponse(false, false, RunState.FAILURE, "Task [ $taskName ] is not running on this node") } // 设标志位 + 写日志：纯内存操作，立即完成 - handle.cancelled.set(true) + handle.cancellation.cancel() log.info("Cancel requested for task [ {} ]", taskName) handle.taskLog?.warn("Stop requested by user for task [ {} ]", taskName) // Statement.cancel() 实现里通常会新开一个 JDBC 连接发 KILL QUERY， diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt index e2d016aa08..2ef18a0fff 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/StreamingSyncStrategy.kt @@ -8,7 +8,8 @@ import io.edurt.datacap.spi.adapter.RowCallback * 流式路径：源端 fetchSize 拉取，目标端 PreparedStatement 批量写。 * 全程不在 JVM 内物化整个结果集。 * - * 仅当源与汇【都】支持流式时选用。逻辑从 LocalExecutorService.runStreaming 原样迁移，行为不变。 + * 仅当源与汇【都】支持流式时选用。取消处理统一由 [runCancelable] 收敛， + * 返回值与取消时上报的行数均为“已提交行数”（writer.writtenCount()）。 */ @SuppressFBWarnings(value = ["RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"]) internal class StreamingSyncStrategy : SyncStrategy @@ -29,73 +30,68 @@ internal class StreamingSyncStrategy : SyncStrategy val taskLog = context.taskLog val totalCount = context.totalCount val progressListener = context.progressListener - val handle = context.handle + val token = context.handle.cancellation taskLog.info( "Streaming sync start: target=`{}`.`{}` columns={} fetchSize={} batchSize={} total={}", database, table, targetColumns.size, fetchSize, batchSize, totalCount ) val startNanos = System.nanoTime() - var written = 0L + var read = 0L val writer: BatchWriter = outputPlugin.openBatchWriter( outputConfigure, database, table, targetColumns, batchSize ) - writer.use { writer -> - val indexByHeader = HashMap() + writer.runCancelable(token) { w -> + // onSchema 时把 sourceKeys 一次性解析成列下标数组，避免每行做 HashMap 查找 + var keyIndexes = IntArray(0) inputPlugin.executeStream(inputConfigure, query, fetchSize, object : RowCallback { override fun onSchema(headers: List, types: List) { - indexByHeader.clear() + val indexByHeader = HashMap(headers.size * 2) headers.forEachIndexed { i, h -> indexByHeader[h.lowercase()] = i } + keyIndexes = IntArray(sourceKeys.size) { k -> + indexByHeader[sourceKeys[k].lowercase()] + ?: throw IllegalStateException("Source column '${sourceKeys[k]}' not found in query result") + } taskLog.info("Streaming sync: source returned headers={}", headers) } override fun onStatement(statement: java.sql.Statement) { - handle.sourceStatement = statement + context.handle.sourceStatement = statement } override fun onRow(row: List) { - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } - val projected = ArrayList(sourceKeys.size) - for (key in sourceKeys) + // 路径 1：源端还在吐行时检测到取消，立即中断（携带已提交行数） + token.throwIfCancelled(w.writtenCount()) + val projected = ArrayList(keyIndexes.size) + for (idx in keyIndexes) { - val idx = indexByHeader[key.lowercase()] - ?: throw IllegalStateException("Source column '$key' not found in query result") projected.add(row[idx]) } - writer.addRow(projected) - written ++ - if (written % PROGRESS_INTERVAL == 0L) + w.addRow(projected) + read ++ + if (read % PROGRESS_INTERVAL == 0L) { val seconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - val rps = if (seconds > 0) (written / seconds).toLong() else 0L + val rps = if (seconds > 0) (read / seconds).toLong() else 0L taskLog.info( "Streaming sync progress: read={} committed={} elapsed={}s rps={}", - written, writer.writtenCount(), "%.1f".format(seconds), rps + read, w.writtenCount(), "%.1f".format(seconds), rps ) - progressListener?.onProgress(writer.writtenCount(), totalCount) - // 顺手记录最近一次进度行数，给 stop 后 catch 取整时使用 - handle.rowsAtStop = writer.writtenCount() + progressListener?.onProgress(w.writtenCount(), totalCount) } } }) } - // 某些驱动在 Statement.cancel() 后直接让 rs.next() 返回 false（不抛异常），需要在循环结束后再判一次 - if (handle.cancelled.get()) - { - throw TaskCancelledException(written) - } + val committed = writer.writtenCount() val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 taskLog.info( - "Streaming sync done: rows={} committed={} elapsed={}s", - written, writer.writtenCount(), "%.1f".format(totalSeconds) + "Streaming sync done: read={} committed={} elapsed={}s", + read, committed, "%.1f".format(totalSeconds) ) - return written + return committed } } diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt index 13e23d0c42..2b34618a02 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/SyncStrategy.kt @@ -4,12 +4,52 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.edurt.datacap.executor.configure.ExecutorProgressListener import io.edurt.datacap.executor.configure.OriginColumn import io.edurt.datacap.spi.PluginService +import io.edurt.datacap.spi.adapter.BatchWriter import io.edurt.datacap.spi.model.Configure import org.slf4j.Logger /** 每处理多少行上报一次进度 / 打一次日志。行为与重构前保持一致。 */ internal const val PROGRESS_INTERVAL = 1_000L +/** + * 用 BatchWriter 跑一段写入循环，并把【所有】取消路径统一收敛成 + * `TaskCancelledException(committed)`（committed = writer.writtenCount()）： + * + * 1. 循环内 onRow 主动抛的 TaskCancelledException（源端还在吐行时检测到取消） + * 2. 循环正常结束、但取消标志已置位（某些驱动 cancel 后 rs.next() 直接返回 false，不抛异常） + * 3. 取消导致底层 fetch 抛出的任意异常（如 Statement.cancel() 引发的 SQLException） + * + * 无论走哪条，writer 都已 use{} 关闭并 flush，故 writtenCount() 是准确的已落库行数。 + * 非取消导致的异常原样抛出，交由上层判为 FAILURE。 + * + * SA_LOCAL_SELF_ASSIGNMENT 是 Kotlin `use{}` 内联展开产生的字节码假阳性，非真实自赋值。 + */ +@SuppressFBWarnings(value = ["SA_LOCAL_SELF_ASSIGNMENT"]) +internal fun BatchWriter.runCancelable(token: CancellationToken, block: (BatchWriter) -> T): T +{ + return try + { + val result = this.use { block(it) } + // 路径 2：循环没抛异常，但期间收到了停止请求 + token.throwIfCancelled(this.writtenCount()) + result + } + catch (ex: TaskCancelledException) + { + // 路径 1：统一用已提交行数覆盖 processed + throw TaskCancelledException(this.writtenCount()) + } + catch (ex: Exception) + { + // 路径 3：取消引发的底层异常 -> 归一为取消；否则是真失败，原样抛出 + if (token.isCancelled) + { + throw TaskCancelledException(this.writtenCount()) + } + throw ex + } +} + /** * 一次同步任务所需的全部上下文。由 LocalExecutorService.start() 组装，交给具体 [SyncStrategy]。 * 把原先 runStreaming / runLegacy 的一长串形参收敛到这里，行为不变，只是不再逐个透传。 diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt index 7eac9a55dc..ee81810e03 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt @@ -4,30 +4,54 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import org.slf4j.Logger import java.util.concurrent.atomic.AtomicBoolean +/** + * 单一取消信号源。stop() 只调 [cancel]，各 sync 策略只读 [isCancelled] / 调 [throwIfCancelled]。 + * 取消时统一抛 [TaskCancelledException]，并携带“已提交行数”（committed），由 start() 转成 STOPPED。 + * + * The single source of truth for cancellation. stop() calls [cancel]; strategies observe it via + * [isCancelled] / [throwIfCancelled]. Cancellation always surfaces as a [TaskCancelledException] + * carrying the committed row count. + */ +internal class CancellationToken +{ + private val cancelled: AtomicBoolean = AtomicBoolean(false) + + val isCancelled: Boolean + get() = cancelled.get() + + fun cancel() + { + cancelled.set(true) + } + + fun throwIfCancelled(processed: Long) + { + if (cancelled.get()) + { + throw TaskCancelledException(processed) + } + } +} + /** * 已注册的活跃任务句柄。 - * - cancelled: 调度循环每条行检查 + * - cancellation: 取消信号，stop() 设置、策略读取 * - sourceStatement: stop() 会调 cancel() 立即终止源端 JDBC 查询，不必等下一行 * - taskLog: 让 stop() 把"用户停止"事件写到任务专属日志里 - * - rowsAtStop: 记录被停止时已处理的行数，供 history 落库 * - * An in-flight task handle shared between start() and stop(). Extracted verbatim from the former - * inner class in LocalExecutorService; behaviour is unchanged. + * An in-flight task handle shared between start() and stop(). */ @SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"]) internal class TaskHandle { - val cancelled: AtomicBoolean = AtomicBoolean(false) + val cancellation: CancellationToken = CancellationToken() @Volatile var sourceStatement: java.sql.Statement? = null @Volatile var taskLog: Logger? = null - - @Volatile - var rowsAtStop: Long = 0L } -/** 取消传播专用异常，携带已处理行数便于上游记录 */ +/** 取消传播专用异常，携带已提交行数便于上游记录 */ internal class TaskCancelledException(val processed: Long) : RuntimeException("Task cancelled by user") diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt index 650420d74c..ce3006258b 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt @@ -22,8 +22,9 @@ import java.util.concurrent.CopyOnWriteArrayList * LocalExecutorService 的“特性测试”（characterization test）： * 只依赖内存 fake，不连任何数据库，用来在重构前锁定当前可观测行为。 * - * 注意：这里断言的是【当前实际行为】，其中个别断言（见 TODO(phase2)）刻画的是已知的、 - * 计划在 Phase 2 修正的可疑行为。重构后如果这些断言需要修改，说明改动是有意为之、可见的。 + * 注意：这里断言的是【当前实际行为】。Phase 2 已把“取消时上报已提交行数”落地， + * 相关断言（streamingDriverAbortAfterCancelIsReportedAsStopped）已从 count=0 更新为 count=2； + * 断言的每次变更都应对应一次有意为之、可见的行为调整。 */ class LocalExecutorCharacterizationTest { @@ -191,9 +192,9 @@ class LocalExecutorCharacterizationTest assertEquals(RunState.STOPPED, response.state) // 前 2 行确实已落库 assertEquals(2, sink.committedRows.size) - // TODO(phase2): 当前实现用 rowsAtStop 上报，而 rowsAtStop 每 1000 行才更新一次， - // 所以小数据量下 count=0（丢失了已处理行数）。这是计划在 Phase 2 修正的已知问题。 - assertEquals(0, response.count) + // Phase 2 修正：取消统一携带“已提交行数”（writer.writtenCount()）， + // 不再依赖每 1000 行才更新一次的 rowsAtStop，故这里能正确报告 count=2。 + assertEquals(2, response.count) } // --------------------------------------------------------------------- From ded497f2c4b6ee4fa535d191dbfb9ee7b608347f Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 09:43:10 +0800 Subject: [PATCH 04/17] feat(executor): enforce optional whole-task timeout in local executor Add a per-executor timeout (seconds) to the local executor. A watchdog cancels the running sync once the limit is exceeded, surfacing it as RunState.TIMEOUT with the committed row count via the shared CancellationToken (USER stop vs TIMEOUT). The timeout defaults to 0 (disabled) in LocalExecutor's config, so existing syncs keep running to completion. DataSetServiceImpl now reads the value from each executor's effective config; executors without the field (Seatunnel) still fall back to 600s. TIMEOUT is treated as a terminal state like STOPPED so it is not overwritten to FAILURE. --- .../service/impl/DataSetServiceImpl.java | 10 ++- .../datacap/executor/local/LocalExecutor.kt | 8 ++ .../executor/local/LocalExecutorService.kt | 82 +++++++++++++++---- .../datacap/executor/local/TaskHandle.kt | 28 +++++-- .../datacap/executor/local/TaskRegistry.kt | 14 ++++ .../LocalExecutorCharacterizationTest.kt | 26 ++++++ 6 files changed, 141 insertions(+), 27 deletions(-) diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java index 7f624167c2..92f98383e0 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java @@ -1232,6 +1232,9 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut int fetchSize = parseIntOrDefault(executorCfg.get("fetchSize"), 1000); int batchSize = parseIntOrDefault(executorCfg.get("batchSize"), 1000); boolean preCount = Boolean.parseBoolean(executorCfg.getOrDefault("preCount", "false")); + // timeout 由各 executor 自己的配置决定：LocalExecutor schema 默认 0（不限时）， + // 未声明该字段的 executor（如 Seatunnel）回退到 600 秒，保持原有行为 + int timeout = parseIntOrDefault(executorCfg.get("timeout"), 600); ExecutorRequest request = new ExecutorRequest( taskName, entity.getUser().getUsername(), @@ -1240,7 +1243,7 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut executorCfg.get("home"), workHome, this.pluginManager, - 600, + timeout, RunWay.valueOf(executorCfg.getOrDefault("way", "LOCAL")), RunMode.valueOf(executorCfg.getOrDefault("mode", "CLIENT")), executorCfg.get("startScript"), @@ -1355,8 +1358,9 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut history.setProgress(java.math.BigDecimal.valueOf(100.0).setScale(2, java.math.RoundingMode.HALF_UP)); } historyRepository.save(history); - // STOPPED 是用户主动停止，已经把最终状态写入 history，无需当作异常抛出 / 也不刷 table metadata - if (response.getState() == RunState.STOPPED) { + // STOPPED（用户主动停止）与 TIMEOUT（超时取消）都是终态：已把最终状态写入 history， + // 无需当作异常抛出 / 也不刷 table metadata + if (response.getState() == RunState.STOPPED || response.getState() == RunState.TIMEOUT) { return; } Preconditions.checkArgument(response.getSuccessful(), response.getMessage()); diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutor.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutor.kt index 750c2c80ac..47c472bf15 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutor.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutor.kt @@ -28,6 +28,14 @@ class LocalExecutor : ExecutorPlugin() "Run SELECT COUNT(*) over the user query before sync to populate the progress denominator. " + "Default OFF: derived-table wrapping is expensive on MySQL/InnoDB for large tables.", true + ), + PluginConfigureField( + "timeout", + PluginFieldType.NUMBER, + "0", + "Whole-task timeout in seconds; when exceeded the sync is cancelled and marked TIMEOUT. " + + "Default 0 disables the timeout (sync runs to completion).", + true ) ) } diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt index fdacbee76c..433272ac33 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LocalExecutorService.kt @@ -11,6 +11,7 @@ import io.edurt.datacap.spi.PluginService import io.edurt.datacap.spi.model.Configure import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.util.concurrent.ScheduledFuture /** * 本地执行器：在 DataCap 进程内把“源查询结果”搬运到“目标表”，支持流式与回退两种同步策略、 @@ -35,6 +36,20 @@ class LocalExecutorService : ExecutorService { TaskRegistry.register(request.taskName, handle) } + // 整任务超时看门狗：timeout<=0 表示不限时（默认）。到点后触发一次“超时取消”。 + val timeoutFuture: ScheduledFuture<*>? = + if (request.timeout > 0) + { + TaskRegistry.scheduleTimeout(request.timeout) + { + taskLog.warn("Task [ {} ] exceeded timeout of {}s, cancelling", request.taskName, request.timeout) + requestCancel(handle, request.taskName, CancelReason.TIMEOUT) + } + } + else + { + null + } try { taskLog.info("Local executor task starting: task={} user={}", request.taskName, request.userName) @@ -108,11 +123,7 @@ class LocalExecutorService : ExecutorService } catch (ex: TaskCancelledException) { - taskLog.warn("Local executor task stopped by user: rows={} task={}", ex.processed, request.taskName) - response.count = if (ex.processed > Int.MAX_VALUE.toLong()) Int.MAX_VALUE else ex.processed.toInt() - response.successful = false - response.state = RunState.STOPPED - response.message = "Stopped by user" + applyCancelled(response, handle, request, ex.processed, taskLog) } catch (ex: Exception) { @@ -121,11 +132,7 @@ class LocalExecutorService : ExecutorService // 此时尚无已提交行数，count 记 0。 if (handle.cancellation.isCancelled) { - taskLog.warn("Local executor task stopped by user (outside write loop): task={}", request.taskName) - response.count = 0 - response.successful = false - response.state = RunState.STOPPED - response.message = "Stopped by user" + applyCancelled(response, handle, request, 0L, taskLog) } else { @@ -137,6 +144,8 @@ class LocalExecutorService : ExecutorService } finally { + // 任务已结束：撤掉尚未触发的超时看门狗，避免事后误触发 + timeoutFuture?.cancel(false) if (request.taskName.isNotBlank()) { TaskRegistry.unregister(request.taskName, handle) @@ -182,12 +191,23 @@ class LocalExecutorService : ExecutorService { return ExecutorResponse(false, false, RunState.FAILURE, "Task [ $taskName ] is not running on this node") } - // 设标志位 + 写日志：纯内存操作，立即完成 - handle.cancellation.cancel() - log.info("Cancel requested for task [ {} ]", taskName) - handle.taskLog?.warn("Stop requested by user for task [ {} ]", taskName) - // Statement.cancel() 实现里通常会新开一个 JDBC 连接发 KILL QUERY， - // 如果源 DB 网络异常会阻塞 HTTP 请求线程。所以放到独立线程，不让 HTTP 等 + requestCancel(handle, taskName, CancelReason.USER) + return ExecutorResponse(false, true, RunState.STOPPED, null) + } + + /** + * 触发取消：设标志位（纯内存、立即完成）+ 异步 cancel 源端 Statement。 + * 用户停止与超时看门狗共用此逻辑，仅取消原因不同。 + * + * Statement.cancel() 实现里通常会新开一个 JDBC 连接发 KILL QUERY， + * 如果源 DB 网络异常会阻塞调用线程，所以放到独立线程执行。 + */ + private fun requestCancel(handle: TaskHandle, taskName: String, reason: CancelReason) + { + handle.cancellation.cancel(reason) + val what = if (reason == CancelReason.TIMEOUT) "Timeout cancel" else "Cancel" + log.info("{} requested for task [ {} ]", what, taskName) + handle.taskLog?.warn("{} requested for task [ {} ]", what, taskName) val stmt = handle.sourceStatement if (stmt != null) { @@ -207,7 +227,35 @@ class LocalExecutorService : ExecutorService } } } - return ExecutorResponse(false, true, RunState.STOPPED, null) + } + + /** + * 把“取消”落到 response 上：按取消原因区分 STOPPED（用户停止）与 TIMEOUT（超时）。 + * processed 为已提交行数（committed）。 + */ + private fun applyCancelled( + response: ExecutorResponse, + handle: TaskHandle, + request: ExecutorRequest, + processed: Long, + taskLog: Logger + ) + { + response.count = if (processed > Int.MAX_VALUE.toLong()) Int.MAX_VALUE else processed.toInt() + response.successful = false + if (handle.cancellation.reason == CancelReason.TIMEOUT) + { + taskLog.warn("Local executor task timed out: rows={} task={}", processed, request.taskName) + response.state = RunState.TIMEOUT + response.timeout = true + response.message = "Timed out after ${request.timeout}s" + } + else + { + taskLog.warn("Local executor task stopped by user: rows={} task={}", processed, request.taskName) + response.state = RunState.STOPPED + response.message = "Stopped by user" + } } /** diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt index ee81810e03..b6e957ed07 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskHandle.kt @@ -4,24 +4,38 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import org.slf4j.Logger import java.util.concurrent.atomic.AtomicBoolean +/** 取消原因：区分“用户主动停止”与“任务超时”，供 start() 映射成 STOPPED / TIMEOUT。 */ +internal enum class CancelReason +{ + USER, + TIMEOUT +} + /** - * 单一取消信号源。stop() 只调 [cancel]，各 sync 策略只读 [isCancelled] / 调 [throwIfCancelled]。 - * 取消时统一抛 [TaskCancelledException]，并携带“已提交行数”（committed），由 start() 转成 STOPPED。 + * 单一取消信号源。stop() / 超时看门狗调 [cancel]，各 sync 策略只读 [isCancelled] / 调 [throwIfCancelled]。 + * 取消时统一抛 [TaskCancelledException]，并携带“已提交行数”（committed），由 start() 按 [reason] 转成 STOPPED / TIMEOUT。 * - * The single source of truth for cancellation. stop() calls [cancel]; strategies observe it via - * [isCancelled] / [throwIfCancelled]. Cancellation always surfaces as a [TaskCancelledException] - * carrying the committed row count. + * The single source of truth for cancellation. Cancellation always surfaces as a + * [TaskCancelledException] carrying the committed row count; [reason] tells USER stop from TIMEOUT. */ internal class CancellationToken { private val cancelled: AtomicBoolean = AtomicBoolean(false) + /** 首个取消原因胜出（用户停止与超时若竞争，以先到者为准） */ + @Volatile + var reason: CancelReason? = null + private set + val isCancelled: Boolean get() = cancelled.get() - fun cancel() + fun cancel(reason: CancelReason = CancelReason.USER) { - cancelled.set(true) + if (cancelled.compareAndSet(false, true)) + { + this.reason = reason + } } fun throwIfCancelled(processed: Long) diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt index c8de1ee8e0..bebb0fca9c 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/TaskRegistry.kt @@ -3,6 +3,8 @@ package io.edurt.datacap.executor.local import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit /** * 进程内活跃任务表 + 取消线程池。 @@ -26,6 +28,14 @@ internal object TaskRegistry t } + // 单线程调度器：仅负责“到点触发超时取消”，动作本身很轻（设标志位 + 提交 cancel 到 cancelExecutor） + private val timeoutScheduler: java.util.concurrent.ScheduledExecutorService = + Executors.newSingleThreadScheduledExecutor { r -> + val t = Thread(r, "local-executor-timeout") + t.isDaemon = true + t + } + fun register(taskName: String, handle: TaskHandle) { runningTasks[taskName] = handle @@ -43,4 +53,8 @@ internal object TaskRegistry { cancelExecutor.submit(block) } + + /** 安排一次性超时触发；返回的 future 供任务正常结束时取消，避免事后误触发。 */ + fun scheduleTimeout(seconds: Long, block: () -> Unit): ScheduledFuture<*> = + timeoutScheduler.schedule(block, seconds, TimeUnit.SECONDS) } diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt index ce3006258b..6380e08dc7 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt @@ -197,6 +197,32 @@ class LocalExecutorCharacterizationTest assertEquals(2, response.count) } + @Test + fun streamingTimeoutIsReportedAsTimeout() + { + // 源端“慢”：每行前 sleep，使整体远超 timeout；看门狗到点后取消，报告 TIMEOUT + val source = FakePluginService( + streaming = true, + headers = listOf("id"), + streamRows = (1..200).map { listOf(it) } + ) + source.beforeRow = { Thread.sleep(50) } + val sink = FakePluginService(streaming = true) + val request = buildRequest( + source, sink, + taskName = "timeout-task", + originColumns = linkedSetOf(OriginColumn("id", "id")) + ).apply { + timeout = 1L + } + + val response = LocalExecutorService().start(request) + + assertFalse(response.successful) + assertEquals(RunState.TIMEOUT, response.state) + assertTrue(response.timeout) + } + // --------------------------------------------------------------------- // Legacy 回退路径（源或汇任一不支持流式） // --------------------------------------------------------------------- From da698802f1b8e2432137efaeb7511bdd2cb1c77e Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 10:01:51 +0800 Subject: [PATCH 05/17] refactor(executor): drop hand-built SQL fallback from local executor Every plugin is reached through the datacap JDBC-conversion driver (type() defaults to JDBC, supportsStreaming()==true) and the sync target is always the JDBC dataset store, so the both-ends-non-streaming branch that hand-built INSERT strings was unreachable in practice. Remove that branch along with ValueCodec.formatSqlLiteral / escapeSqlString (manual quoting was an injection and type-coercion hazard, and carried a stray NUL byte in a comment). The legacy path now requires a batch-write-capable target and fails fast with a clear message otherwise; the streaming path is unaffected. --- .../executor/local/LegacySyncStrategy.kt | 133 +++++------------- .../datacap/executor/local/ValueCodec.kt | Bin 2462 -> 1273 bytes .../LocalExecutorCharacterizationTest.kt | 34 ++--- 3 files changed, 50 insertions(+), 117 deletions(-) diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt index 1fabc1b2b7..d965cf4b35 100644 --- a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt +++ b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/LegacySyncStrategy.kt @@ -2,20 +2,14 @@ package io.edurt.datacap.executor.local import com.fasterxml.jackson.databind.node.ObjectNode import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.edurt.datacap.common.sql.SqlBuilder -import io.edurt.datacap.common.sql.configure.SqlBody -import io.edurt.datacap.common.sql.configure.SqlColumn -import io.edurt.datacap.common.sql.configure.SqlType -import io.edurt.datacap.spi.PluginService -import io.edurt.datacap.spi.model.Configure /** - * 回退路径：源或汇任一不支持流式（如 HTTP / Native 插件）。仍然使用旧的全量读取， - * 但目标端按 batch 切片提交，避免一次性拼接巨大 SQL 字符串；同时修复 NULL、类型、转义问题。 + * 回退路径：源端不支持流式（executeStream 不可用）时，先用 execute() 全量读取， + * 再由 BatchWriter 分批写入目标端。 * - * 内部再按“目标端是否支持流式”分两条子路：BatchWriter / 拼 INSERT 字符串。 - * 取消处理与流式路径一致：BatchWriter 子路由 [runCancelable] 收敛为“已提交行数”； - * SQL 子路的 written 本身就是已 flush（已提交）行数。 + * 说明：本仓库所有插件都经 “datacap” JDBC 转换驱动访问（PluginService.type() 默认 JDBC， + * supportsStreaming()==true），因此实际运行中几乎总是走 [StreamingSyncStrategy]，此回退仅为 + * 未来可能 opt-out 流式的插件保留。目标端若不支持流式则直接失败——不再退回到不安全的手拼 SQL。 */ @SuppressFBWarnings(value = ["BC_BAD_CAST_TO_ABSTRACT_COLLECTION", "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"]) internal class LegacySyncStrategy : SyncStrategy @@ -29,7 +23,6 @@ internal class LegacySyncStrategy : SyncStrategy val outputConfigure = context.outputConfigure val database = context.database val table = context.table - val originColumns = context.originColumns val batchSize = context.batchSize val taskLog = context.taskLog val totalCount = context.totalCount @@ -37,6 +30,17 @@ internal class LegacySyncStrategy : SyncStrategy val token = context.handle.cancellation taskLog.info("Legacy sync start: target=`{}`.`{}` batchSize={}", database, table, batchSize) + + // 目标端必须支持批量/流式写入。全部插件经 JDBC 转换后都满足；这里显式兜底， + // 避免历史上“双端非流式”时静默拼接 INSERT 字符串（存在注入与类型/转义隐患）。 + if (!outputPlugin.supportsStreaming()) + { + throw IllegalStateException( + "Output plugin '${outputPlugin.name()}' does not support batch write (supportsStreaming=false); " + + "the local executor no longer falls back to hand-built SQL. Use a JDBC/streaming-capable target." + ) + } + val startNanos = System.nanoTime() val inputResult = inputPlugin.execute(inputConfigure, query) if (inputResult.isSuccessful != true) @@ -48,95 +52,34 @@ internal class LegacySyncStrategy : SyncStrategy // 全量路径已经能拿到精确总数，覆盖 pre-count 的估算 val effectiveTotal = if (totalCount >= 0) totalCount else rows.size.toLong() - // 目标端能流式：用 BatchWriter 安全 + 节省内存 - if (outputPlugin.supportsStreaming()) - { - val targetColumns = context.targetColumns - val sourceKeys = context.sourceKeys - var read = 0L - val writer = outputPlugin.openBatchWriter( - outputConfigure, database, table, targetColumns, batchSize - ) - writer.runCancelable(token) { w -> - for (item in rows) + val targetColumns = context.targetColumns + val sourceKeys = context.sourceKeys + var read = 0L + val writer = outputPlugin.openBatchWriter( + outputConfigure, database, table, targetColumns, batchSize + ) + writer.runCancelable(token) { w -> + for (item in rows) + { + token.throwIfCancelled(w.writtenCount()) + val node = item as? ObjectNode ?: continue + val projected = ArrayList(sourceKeys.size) + for (key in sourceKeys) { - token.throwIfCancelled(w.writtenCount()) - val node = item as? ObjectNode ?: continue - val projected = ArrayList(sourceKeys.size) - for (key in sourceKeys) - { - projected.add(ValueCodec.jsonNodeToJdbcValue(node.get(key))) - } - w.addRow(projected) - read ++ - if (read % PROGRESS_INTERVAL == 0L) - { - taskLog.info("Legacy sync progress: read={} committed={}", read, w.writtenCount()) - progressListener?.onProgress(w.writtenCount(), effectiveTotal) - } + projected.add(ValueCodec.jsonNodeToJdbcValue(node.get(key))) } - } - val committed = writer.writtenCount() - val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - taskLog.info("Legacy sync done: read={} committed={} elapsed={}s", read, committed, "%.1f".format(totalSeconds)) - return committed - } - - // 双端都不支持流式：用 INSERT 字符串，但按 batch 提交，不再拼一个巨大字符串。 - // written 只在 flush 成功后累加，因此它就是“已提交行数”。 - var written = 0L - val batch = ArrayList(batchSize) - for (item in rows) - { - token.throwIfCancelled(written) - val node = item as? ObjectNode ?: continue - val sqlColumns = ArrayList(originColumns.size) - for (col in originColumns) - { - sqlColumns.add( - SqlColumn.builder() - .column("`${col.name}`") - .value(ValueCodec.formatSqlLiteral(node.get(col.original))) - .build() - ) - } - val body = SqlBody.builder() - .type(SqlType.INSERT) - .database(database) - .table(table) - .columns(sqlColumns) - .build() - batch.add(SqlBuilder(body).sql) - if (batch.size >= batchSize) - { - flushLegacyBatch(outputPlugin, outputConfigure, batch) - written += batch.size - batch.clear() - if (written % PROGRESS_INTERVAL == 0L) + w.addRow(projected) + read ++ + if (read % PROGRESS_INTERVAL == 0L) { - taskLog.info("Legacy sync progress (sql batch): written={}", written) - progressListener?.onProgress(written, effectiveTotal) + taskLog.info("Legacy sync progress: read={} committed={}", read, w.writtenCount()) + progressListener?.onProgress(w.writtenCount(), effectiveTotal) } } } - if (batch.isNotEmpty()) - { - flushLegacyBatch(outputPlugin, outputConfigure, batch) - written += batch.size - batch.clear() - } + val committed = writer.writtenCount() val totalSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0 - taskLog.info("Legacy sync done: rows={} elapsed={}s", written, "%.1f".format(totalSeconds)) - return written - } - - private fun flushLegacyBatch(plugin: PluginService, configure: Configure, batch: List) - { - val joined = batch.joinToString("\n") - val result = plugin.execute(configure, joined) - if (result.isSuccessful != true) - { - throw RuntimeException(result.message ?: "Output plugin failed") - } + taskLog.info("Legacy sync done: read={} committed={} elapsed={}s", read, committed, "%.1f".format(totalSeconds)) + return committed } } diff --git a/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/ValueCodec.kt b/executor/datacap-executor-local/src/main/kotlin/io/edurt/datacap/executor/local/ValueCodec.kt index 7812dd3c1a491b3cb98b8cb00fc26ae560cefc7f..1d5dddeb7608e6649e4927f502e23526622f5eee 100644 GIT binary patch delta 185 zcmbOy{F8G+$;6h32sp*Nyl?qOYCCM4#MVTe3MG7yruYbC)ph zGymzjdCxcQembxFX~Uk!4NY7MT0o@WT2WGzm|T*YqL5aUpR3@LpPZQET9KMuT9RKB zoLW?tnVf2+kT}_nQOaJ|PQlB?$yuQ+F{d2#8sS;#dNk%F* S*Ayg{WNdC>%x7k*?n$6JPvNOtzv9A52V0fJmHCvk-KUXu?ZZE~81;&fdET!3BmQ z5H^NxxJdjF6@Ns9DJo2v;6LJ%y>0ijXP@+T>xL3AJY!qQph$5!4)}=mp4aFxs zTmzYK!x$q-xT{~HniOYQfWepoJ2`OOnTADG$Y_Jgb_n|J{nWpKCwX#A#Oq6pEtU{q7xG{rIWb3!5KE+1P{&qDZ$MJfF{jR3y aA4TH7a@ew*vwf?Z9BO3B$wk7QaQ^`{!nUvg diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt index 6380e08dc7..603cddef4f 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt @@ -263,38 +263,28 @@ class LocalExecutorCharacterizationTest } @Test - fun legacyBothNonStreamingBuildsInsertSql() + fun legacyNonStreamingOutputIsRejected() { - // 双端都不支持流式 -> 拼 INSERT 字符串，验证列名反引号、单引号转义、NULL、数字不加引号 - val rows = listOf( - objectNode("id" to 1, "name" to "O'Brien"), - objectNode("id" to 2, "name" to null) - ) + // Phase 3-C：目标端不支持流式写入时，不再静默拼接手写 INSERT（注入/转义隐患）， + // 而是快速失败。（实运行中所有插件经 JDBC 转换后 supportsStreaming==true，此分支仅为兜底。） val source = FakePluginService( streaming = false, - executeHandler = { rowsResponse(rows) } - ) - val sink = FakePluginService( - streaming = false, - executeHandler = { Response.builder().isSuccessful(true).columns(emptyList()).build() } + executeHandler = { rowsResponse(listOf(objectNode("id" to 1, "name" to "a"))) } ) + val sink = FakePluginService(streaming = false) val request = buildRequest( source, sink, - originColumns = linkedSetOf( - OriginColumn("uid", "id"), - OriginColumn("full_name", "name") - ) + originColumns = linkedSetOf(OriginColumn("uid", "id")) ) val response = LocalExecutorService().start(request) - assertTrue(response.successful) - assertEquals(2, response.count) - val sql = sink.executedSql.joinToString("\n") - assertTrue("should build INSERT: $sql", sql.contains("INSERT INTO `target_db`.`target_tbl`")) - assertTrue("should quote columns: $sql", sql.contains("`uid`") && sql.contains("`full_name`")) - assertTrue("should escape single quote: $sql", sql.contains("'O''Brien'")) - assertTrue("should emit NULL literal: $sql", sql.contains("NULL")) + assertFalse(response.successful) + assertEquals(RunState.FAILURE, response.state) + assertNotNull(response.message) + assertTrue(response.message!!.contains("does not support batch write")) + // 没有拼接/下发任何 INSERT 语句 + assertTrue(sink.executedSql.isEmpty()) } // --------------------------------------------------------------------- From 456533644e8a15e2c1055d80d85deef9629328b3 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 10:42:04 +0800 Subject: [PATCH 06/17] test(executor): add PostgreSQL Testcontainers end-to-end sync tests Drive LocalExecutorService against a real PostgreSQL container (source and target) through a real-JDBC PluginService, verifying an actual source->target sync: streaming copy with column projection/rename, NULL and numeric handling, pre-count totals, and a real Statement.cancel() stop. Pin the docker-java API version (1.41) in-code so Testcontainers negotiates with newer Docker Desktop engines, and skip gracefully when Docker is absent. Add spotbugs suppressions on the test helpers for false positives (Kotlin use{} obligations, list casts, test data exposure). --- test/datacap-test-executor/pom.xml | 13 + .../LocalExecutorCharacterizationTest.kt | 2 + .../local/LocalExecutorPostgresE2ETest.kt | 256 ++++++++++++++++++ .../test/local/support/FakePluginService.kt | 3 + .../test/local/support/JdbcPluginService.kt | 214 +++++++++++++++ 5 files changed, 488 insertions(+) create mode 100644 test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt create mode 100644 test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/JdbcPluginService.kt diff --git a/test/datacap-test-executor/pom.xml b/test/datacap-test-executor/pom.xml index 90cd43b5b4..047e6f6e89 100644 --- a/test/datacap-test-executor/pom.xml +++ b/test/datacap-test-executor/pom.xml @@ -49,6 +49,19 @@ ${project.version} test + + + org.testcontainers + postgresql + ${testcontainers.version} + test + + + org.postgresql + postgresql + ${datacap.pgsql.version} + test + diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt index 603cddef4f..3051f537d0 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt @@ -2,6 +2,7 @@ package io.edurt.datacap.test.local import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.edurt.datacap.executor.common.RunState import io.edurt.datacap.executor.configure.ExecutorConfigure import io.edurt.datacap.executor.configure.ExecutorProgressListener @@ -26,6 +27,7 @@ import java.util.concurrent.CopyOnWriteArrayList * 相关断言（streamingDriverAbortAfterCancelIsReportedAsStopped）已从 count=0 更新为 count=2； * 断言的每次变更都应对应一次有意为之、可见的行为调整。 */ +@SuppressFBWarnings(value = ["BC_BAD_CAST_TO_ABSTRACT_COLLECTION"]) class LocalExecutorCharacterizationTest { private val mapper = ObjectMapper() diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt new file mode 100644 index 0000000000..21f62064dd --- /dev/null +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt @@ -0,0 +1,256 @@ +package io.edurt.datacap.test.local + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.edurt.datacap.executor.common.RunState +import io.edurt.datacap.executor.configure.ExecutorConfigure +import io.edurt.datacap.executor.configure.ExecutorProgressListener +import io.edurt.datacap.executor.configure.ExecutorRequest +import io.edurt.datacap.executor.configure.ExecutorResponse +import io.edurt.datacap.executor.configure.OriginColumn +import io.edurt.datacap.executor.local.LocalExecutorService +import io.edurt.datacap.spi.model.Configure +import io.edurt.datacap.test.local.support.JdbcPluginService +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.AfterClass +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.testcontainers.DockerClientFactory +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.utility.DockerImageName +import java.sql.DriverManager +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicReference + +/** + * 真实数据库端到端测试：用 PostgreSQL 容器作为【源】和【目标】， + * 通过 [JdbcPluginService]（真实 JDBC）驱动 [LocalExecutorService] 完成一次真正的 source -> target 同步， + * 然后直接查目标表核对落库数据。覆盖：流式搬运 + 投影/改名 + NULL/数值类型、preCount 总数、真实 stop 取消。 + * + * 需要本机可用的 Docker。容器由 @BeforeClass/@AfterClass 管理，整个测试类只起停一次； + * Docker 不可用时优雅跳过。 + */ +@SuppressFBWarnings(value = ["OBL_UNSATISFIED_OBLIGATION", "OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"]) +class LocalExecutorPostgresE2ETest +{ + companion object + { + private var container: PostgreSQLContainer? = null + + @BeforeClass + @JvmStatic + fun startContainer() + { + // Docker Desktop（Engine 29 / API 1.55）会以 HTTP 400 拒绝 docker-java 默认过低的 API 版本。 + // 钉一个双方都支持的版本（守护进程最低 1.40）。尊重外部已显式指定的值。 + if (System.getProperty("api.version") == null && System.getenv("DOCKER_API_VERSION") == null) + { + System.setProperty("api.version", "1.41") + } + // Docker 不可用时优雅跳过（例如未装 Docker 的 CI），而不是让整个类失败。 + Assume.assumeTrue( + "Docker is not available; skipping PostgreSQL E2E tests", + DockerClientFactory.instance().isDockerAvailable + ) + container = PostgreSQLContainer(DockerImageName.parse("postgres:16-alpine")) + .apply { start() } + } + + @AfterClass + @JvmStatic + fun stopContainer() + { + container?.stop() + container = null + } + + private fun container(): PostgreSQLContainer = + container ?: throw IllegalStateException("PostgreSQL container is not started") + } + + private lateinit var service: JdbcPluginService + + @Before + fun setup() + { + service = JdbcPluginService(container().jdbcUrl, container().username, container().password) + exec("DROP TABLE IF EXISTS source_t") + exec("DROP TABLE IF EXISTS target_t") + exec("CREATE TABLE source_t (id int, name text, amount numeric)") + exec("CREATE TABLE target_t (uid int, full_name text, amt numeric)") + } + + @Test + fun streamingCopiesRealDataWithProjectionAndNull() + { + exec("INSERT INTO source_t (id, name, amount) VALUES (1,'alice',10.5),(2,NULL,20),(3,'carol',NULL)") + + val request = buildRequest( + query = "SELECT id, name, amount FROM source_t ORDER BY id", + targetTable = "target_t", + columns = linkedSetOf( + OriginColumn("uid", "id"), + OriginColumn("full_name", "name"), + OriginColumn("amt", "amount") + ) + ) + + val response = LocalExecutorService().start(request) + + assertTrue(response.successful) + assertEquals(RunState.SUCCESS, response.state) + assertEquals(3, response.count) + + val rows = query("SELECT uid, full_name, amt FROM target_t ORDER BY uid") + assertEquals( + listOf( + listOf("1", "alice", "10.5"), + listOf("2", null, "20"), + listOf("3", "carol", null) + ), + rows + ) + } + + @Test + fun preCountReportsTotalFromRealDb() + { + exec("INSERT INTO source_t (id, name, amount) SELECT g, 'n' || g, g FROM generate_series(1, 250) g") + + val progress = CopyOnWriteArrayList>() + val request = buildRequest( + query = "SELECT id, name, amount FROM source_t", + targetTable = "target_t", + columns = linkedSetOf( + OriginColumn("uid", "id"), + OriginColumn("full_name", "name"), + OriginColumn("amt", "amount") + ) + ).apply { + preCount = true + progressListener = ExecutorProgressListener { processed, total -> progress.add(processed to total) } + } + + val response = LocalExecutorService().start(request) + + assertTrue(response.successful) + assertEquals(250, response.count) + assertEquals(250L, count("target_t")) + // 首个进度事件应是 pre-count 后上报的 (0, 250) + assertEquals(0L to 250L, progress.first()) + assertEquals(250L, progress.last().second) + } + + @Test + fun stopCancelsRunningSyncOnRealDb() + { + // 用 generate_series 造一个足够大的慢结果集，跑起来后从另一线程 stop，验证真实 Statement.cancel 生效 + val taskName = "pg-stop" + val request = buildRequest( + query = "SELECT g AS id, 'x' AS name, g AS amount FROM generate_series(1, 3000000) g", + targetTable = "target_t", + columns = linkedSetOf( + OriginColumn("uid", "id"), + OriginColumn("full_name", "name"), + OriginColumn("amt", "amount") + ), + taskName = taskName + ) + + val executor = LocalExecutorService() + val responseRef = AtomicReference() + val worker = Thread { responseRef.set(executor.start(request)) } + worker.start() + + // 等到目标表已经落了一些行，确保同步确实在跑，再发停止 + var waited = 0 + while (count("target_t") == 0L && waited < 20_000) + { + Thread.sleep(50) + waited += 50 + } + assertTrue("sync did not start writing in time", count("target_t") > 0L) + + val stopResponse = executor.stop(stopRequest(taskName)) + assertEquals(RunState.STOPPED, stopResponse.state) + + worker.join(60_000) + val response = responseRef.get() + assertNotNull("worker did not finish after stop", response) + assertFalse(response.successful) + assertEquals(RunState.STOPPED, response.state) + // 被中途取消：落库行数应远小于 3,000,000 + assertTrue("expected partial rows, got ${count("target_t")}", count("target_t") < 3_000_000L) + } + + // --------------------------------------------------------------------- + // helpers + // --------------------------------------------------------------------- + + private fun buildRequest( + query: String, + targetTable: String, + columns: LinkedHashSet, + taskName: String = "" + ): ExecutorRequest + { + val input = ExecutorConfigure("postgresql", null, emptySet()).apply { + plugin = service + this.query = query + database = "public" + table = targetTable + originConfigure = Configure() + originColumns = columns + } + val output = ExecutorConfigure("postgresql", null, emptySet()).apply { + plugin = service + originConfigure = Configure() + } + return ExecutorRequest(null, input, output).apply { + this.taskName = taskName + userName = "tester" + } + } + + private fun stopRequest(taskName: String): ExecutorRequest + { + val placeholder = ExecutorConfigure(null) + return ExecutorRequest(taskName, "", placeholder, placeholder) + } + + private fun exec(sql: String) + { + DriverManager.getConnection(container().jdbcUrl, container().username, container().password).use { conn -> + conn.createStatement().use { st -> st.execute(sql) } + } + } + + private fun query(sql: String): List> + { + DriverManager.getConnection(container().jdbcUrl, container().username, container().password).use { conn -> + conn.createStatement().use { st -> + st.executeQuery(sql).use { rs -> + val n = rs.metaData.columnCount + val out = ArrayList>() + while (rs.next()) + { + val row = ArrayList(n) + for (i in 1..n) + { + row.add(rs.getString(i)) + } + out.add(row) + } + return out + } + } + } + } + + private fun count(table: String): Long = + query("SELECT count(*) FROM $table").first().first()!!.toLong() +} diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt index 7ec8d31309..4d71434cd6 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/FakePluginService.kt @@ -1,5 +1,6 @@ package io.edurt.datacap.test.local.support +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.edurt.datacap.spi.PluginService import io.edurt.datacap.spi.PluginType import io.edurt.datacap.spi.adapter.BatchWriter @@ -27,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger * - streaming=true -> openBatchWriter() 返回 [CapturingBatchWriter]，把落库行收集到 [committedRows] * - streaming=false -> 走 execute()，[executedSql] 收集所有拼出来的 INSERT 语句 */ +@SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"]) class FakePluginService( private val streaming: Boolean = true, private val headers: List = emptyList(), @@ -139,6 +141,7 @@ class FakePluginService( * 内存版 BatchWriter：模拟 JdbcBatchWriter 的“攒够 batchSize 才 flush、close 时 flush 剩余”语义， * writtenCount() 只统计已 flush（已提交）的行数，与真实实现保持一致。 */ +@SuppressFBWarnings(value = ["EI_EXPOSE_REP2"]) class CapturingBatchWriter( private val columns: List, private val batchSize: Int, diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/JdbcPluginService.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/JdbcPluginService.kt new file mode 100644 index 0000000000..0f9f3fa837 --- /dev/null +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/support/JdbcPluginService.kt @@ -0,0 +1,214 @@ +package io.edurt.datacap.test.local.support + +import com.fasterxml.jackson.databind.ObjectMapper +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.edurt.datacap.spi.PluginService +import io.edurt.datacap.spi.PluginType +import io.edurt.datacap.spi.adapter.BatchWriter +import io.edurt.datacap.spi.adapter.RowCallback +import io.edurt.datacap.spi.model.Configure +import io.edurt.datacap.spi.model.Response +import java.math.BigDecimal +import java.sql.Connection +import java.sql.DriverManager +import java.sql.ResultSet + +/** + * 直连真实 JDBC 数据库的 PluginService，用于对真实容器库（Testcontainers）做端到端同步测试。 + * + * 与 [FakePluginService] 不同：这里 executeStream / openBatchWriter / execute 都打真实 JDBC—— + * 真实的流式 ResultSet 读取、真实的 PreparedStatement 批量写、真实的 Statement.cancel()。 + * 它绕开 datacap 自己的 JdbcConnection/PluginClassLoader 封装（那部分由 driver/plugin 测试模块覆盖）， + * 从而稳定地验证 LocalExecutorService 在真实源/目标库之间的搬运正确性。 + */ +@SuppressFBWarnings(value = [ + "BC_BAD_CAST_TO_ABSTRACT_COLLECTION", + "OBL_UNSATISFIED_OBLIGATION", + "OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE", + "SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING" +]) +class JdbcPluginService( + private val url: String, + private val user: String, + private val password: String, + private val defaultFetchSize: Int = 1000 +) : PluginService +{ + override fun supportsStreaming(): Boolean = true + + override fun type(): PluginType = PluginType.JDBC + + private fun open(): Connection + { + val conn = DriverManager.getConnection(url, user, password) + conn.autoCommit = false + return conn + } + + override fun executeStream(configure: Configure, content: String, fetchSize: Int, callback: RowCallback) + { + val conn = open() + try + { + conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY).use { st -> + st.fetchSize = if (fetchSize > 0) fetchSize else defaultFetchSize + callback.onStatement(st) + st.executeQuery(content).use { rs -> + val md = rs.metaData + val n = md.columnCount + val headers = (1..n).map { i -> md.getColumnLabel(i) ?: md.getColumnName(i) } + val types = (1..n).map { i -> md.getColumnTypeName(i) } + callback.onSchema(headers, types) + while (rs.next()) + { + val row = ArrayList(n) + for (i in 1..n) + { + row.add(rs.getObject(i)) + } + callback.onRow(row) + } + } + } + } + finally + { + try + { + conn.close() + } + catch (ignore: Exception) + { + } + } + } + + override fun openBatchWriter( + configure: Configure, + database: String, + table: String, + columns: List, + batchSize: Int + ): BatchWriter + { + val conn = open() + val cols = columns.joinToString(", ") { "\"$it\"" } + val placeholders = columns.joinToString(", ") { "?" } + val target = if (database.isNotEmpty()) "\"$database\".\"$table\"" else "\"$table\"" + val sql = "INSERT INTO $target ($cols) VALUES ($placeholders)" + val ps = conn.prepareStatement(sql) + return RealJdbcBatchWriter(conn, ps, columns.size, if (batchSize > 0) batchSize else 1000) + } + + override fun execute(configure: Configure, content: String): Response + { + return try + { + open().use { conn -> + conn.createStatement().use { st -> + val isResultSet = st.execute(content) + if (!isResultSet) + { + conn.commit() + return Response.builder().isSuccessful(true).build() + } + st.resultSet.use { rs -> + val md = rs.metaData + val n = md.columnCount + val mapper = ObjectMapper() + val rows = ArrayList() + while (rs.next()) + { + val node = mapper.createObjectNode() + for (i in 1..n) + { + val name = md.getColumnLabel(i) ?: md.getColumnName(i) + when (val v = rs.getObject(i)) + { + null -> node.putNull(name) + is Int -> node.put(name, v) + is Long -> node.put(name, v) + is Boolean -> node.put(name, v) + is Double -> node.put(name, v) + is Float -> node.put(name, v.toDouble()) + is BigDecimal -> node.put(name, v) + else -> node.put(name, v.toString()) + } + } + rows.add(node) + } + Response.builder().isSuccessful(true).columns(rows).build() + } + } + } + } + catch (ex: Exception) + { + Response.builder().isSuccessful(false).message(ex.message).build() + } + } + + private class RealJdbcBatchWriter( + private val conn: Connection, + private val ps: java.sql.PreparedStatement, + private val columnCount: Int, + private val batchSize: Int + ) : BatchWriter + { + private var pending = 0 + private var committed = 0L + + override fun addRow(row: List<*>) + { + for (i in 0 until columnCount) + { + ps.setObject(i + 1, row[i]) + } + ps.addBatch() + pending++ + if (pending >= batchSize) + { + flush() + } + } + + override fun writtenCount(): Long = committed + + override fun close() + { + try + { + if (pending > 0) + { + flush() + } + } + finally + { + try + { + ps.close() + } + catch (ignore: Exception) + { + } + try + { + conn.close() + } + catch (ignore: Exception) + { + } + } + } + + private fun flush() + { + ps.executeBatch() + conn.commit() + committed += pending + pending = 0 + ps.clearBatch() + } + } +} From 0e9b8949a6a68c83b059de58ceaf80116143aeed Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 10:51:39 +0800 Subject: [PATCH 07/17] test(executor): pin seatunnel request-to-command mapping Extract SeaTunnelCommander construction from SeatunnelExecutorService.start() into a package-visible buildCommander(request) (no behaviour change) and add tests asserting the resulting command line for the Spark and SeaTunnel engines. This locks the ExecutorRequest-to-command mapping so the upcoming ExecutorRequest restructure can be verified against a stable expected command. --- .../seatunnel/SeatunnelExecutorService.java | 25 ++++--- .../SeatunnelCommanderMappingTest.kt | 68 +++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt diff --git a/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java b/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java index 87e50d169b..7f67d17c64 100644 --- a/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java +++ b/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java @@ -38,14 +38,7 @@ public class SeatunnelExecutorService public ExecutorResponse start(ExecutorRequest request) { try { - SeaTunnelCommander commander = new SeaTunnelCommander( - request.getExecutorHome() + "/bin", - request.getStartScript(), - request.getRunWay().name().toLowerCase(), - request.getRunMode().name().toLowerCase(), - String.join(File.separator, request.getWorkHome(), request.getTaskName() + ".configure"), - request.getTaskName(), - request.getRunEngine()); + SeaTunnelCommander commander = buildCommander(request); LoggerExecutor loggerExecutor = new LogbackExecutor(request.getWorkHome(), request.getTaskName() + ".log"); String result = before(request, loggerExecutor.getLogger()); @@ -83,6 +76,22 @@ public ExecutorResponse stop(ExecutorRequest request) throw new UnsupportedOperationException(); } + /** + * 从请求构造 SeaTunnel 命令。单独抽出以便测试“请求字段 -> 命令行”的映射， + * 这是 ExecutorRequest 结构调整时最容易出错、也最需要回归保护的一段。 + */ + SeaTunnelCommander buildCommander(ExecutorRequest request) + { + return new SeaTunnelCommander( + request.getExecutorHome() + "/bin", + request.getStartScript(), + request.getRunWay().name().toLowerCase(), + request.getRunMode().name().toLowerCase(), + String.join(File.separator, request.getWorkHome(), request.getTaskName() + ".configure"), + request.getTaskName(), + request.getRunEngine()); + } + /** * Writes a child element to the JSON output using the specified type, JSON generator, and executor configure. * diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt new file mode 100644 index 0000000000..2c5fe724d7 --- /dev/null +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt @@ -0,0 +1,68 @@ +package io.edurt.datacap.executor.seatunnel + +import io.edurt.datacap.executor.common.RunEngine +import io.edurt.datacap.executor.common.RunMode +import io.edurt.datacap.executor.common.RunWay +import io.edurt.datacap.executor.configure.ExecutorConfigure +import io.edurt.datacap.executor.configure.ExecutorRequest +import org.junit.Assert.assertEquals +import org.junit.Test +import java.io.File + +/** + * Seatunnel 安全网：锁定“ExecutorRequest 字段 -> seatunnel 命令行”的映射。 + * + * 这段映射在 ExecutorRequest 结构调整（把 executor 专属字段收敛进 options）时最易出错。 + * 断言的是最终命令字符串——只要重构后 buildCommander 仍读到等价的值，命令就不变、测试保持绿。 + */ +class SeatunnelCommanderMappingTest +{ + private val service = SeatunnelExecutorService() + + @Test + fun sparkEngineCommand() + { + val request = ExecutorRequest( + taskName = "task1", + userName = "tester", + input = ExecutorConfigure("Jdbc"), + output = ExecutorConfigure("Jdbc"), + executorHome = "/opt/seatunnel", + workHome = "/work", + runWay = RunWay.LOCAL, + runMode = RunMode.CLIENT, + startScript = "start-seatunnel-spark-connector-v2.sh", + runEngine = RunEngine.SPARK + ) + + val config = "/work" + File.separator + "task1.configure" + assertEquals( + "/opt/seatunnel/bin/start-seatunnel-spark-connector-v2.sh " + + "--master local --deploy-mode client --config $config --name task1", + service.buildCommander(request).toCommand() + ) + } + + @Test + fun seatunnelEngineLocalCommand() + { + val request = ExecutorRequest( + taskName = "task2", + userName = "tester", + input = ExecutorConfigure("Jdbc"), + output = ExecutorConfigure("Jdbc"), + executorHome = "/opt/seatunnel", + workHome = "/work", + runWay = RunWay.LOCAL, + runMode = RunMode.CLIENT, + startScript = "seatunnel.sh", + runEngine = RunEngine.SEATUNNEL + ) + + val config = "/work" + File.separator + "task2.configure" + assertEquals( + "/opt/seatunnel/bin/seatunnel.sh -e local --config $config --name task2", + service.buildCommander(request).toCommand() + ) + } +} From 00faa07fa78fd20d504eb63c5f9ed031f111f2fb Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Fri, 4 Sep 2026 11:31:18 +0800 Subject: [PATCH 08/17] refactor(executor): split ExecutorRequest into common fields and options Collapse the executor-specific typed fields (executorHome, startScript, runWay, runMode, runEngine, fetchSize, batchSize, preCount) into a single options string map, leaving only executor-agnostic fields on ExecutorRequest (taskName, userName, input, output, workHome, timeout, transform, progressListener, pluginManager). Reduce the six telescoping constructors to the primary plus one workHome convenience constructor. Each executor now reads what it needs from options: the local executor takes fetchSize/batchSize/preCount, seatunnel takes home/startScript/way/mode/engine. Callers pass the executor's effective config map straight through as options. Adding a new executor no longer requires touching this shared class. Behaviour is preserved: local characterization + PostgreSQL E2E tests stay green, and the seatunnel command-mapping test asserts the same command line. --- .../service/impl/DataSetServiceImpl.java | 27 ++----- .../service/impl/WorkflowServiceImpl.java | 19 ++--- .../executor/local/LocalExecutorService.kt | 11 ++- .../seatunnel/SeatunnelExecutorService.java | 18 +++-- .../executor/configure/ExecutorRequest.kt | 78 +++++-------------- .../SeatunnelCommanderMappingTest.kt | 34 ++++---- .../LocalExecutorCharacterizationTest.kt | 2 +- .../local/LocalExecutorPostgresE2ETest.kt | 2 +- 8 files changed, 72 insertions(+), 119 deletions(-) diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java index 92f98383e0..ea88a00f43 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/DataSetServiceImpl.java @@ -13,11 +13,8 @@ import io.edurt.datacap.common.sql.configure.SqlOrder; import io.edurt.datacap.common.sql.configure.SqlType; import io.edurt.datacap.executor.ExecutorService; -import io.edurt.datacap.executor.common.RunEngine; -import io.edurt.datacap.executor.common.RunMode; import io.edurt.datacap.executor.common.RunProtocol; import io.edurt.datacap.executor.common.RunState; -import io.edurt.datacap.executor.common.RunWay; import io.edurt.datacap.executor.configure.ExecutorConfigure; import io.edurt.datacap.executor.configure.ExecutorRequest; import io.edurt.datacap.executor.configure.ExecutorResponse; @@ -1229,9 +1226,6 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut log.warn("Serialize effective executor configure failed: {}", ex.getMessage()); } historyRepository.save(history); - int fetchSize = parseIntOrDefault(executorCfg.get("fetchSize"), 1000); - int batchSize = parseIntOrDefault(executorCfg.get("batchSize"), 1000); - boolean preCount = Boolean.parseBoolean(executorCfg.getOrDefault("preCount", "false")); // timeout 由各 executor 自己的配置决定：LocalExecutor schema 默认 0（不限时）， // 未声明该字段的 executor（如 Seatunnel）回退到 600 秒，保持原有行为 int timeout = parseIntOrDefault(executorCfg.get("timeout"), 600); @@ -1239,21 +1233,14 @@ private DataSetEntity syncData(DataSetEntity entity, java.util.concurrent.Execut taskName, entity.getUser().getUsername(), input, - output, - executorCfg.get("home"), - workHome, - this.pluginManager, - timeout, - RunWay.valueOf(executorCfg.getOrDefault("way", "LOCAL")), - RunMode.valueOf(executorCfg.getOrDefault("mode", "CLIENT")), - executorCfg.get("startScript"), - RunEngine.valueOf(executorCfg.getOrDefault("engine", "SPARK")), - null, - fetchSize, - batchSize, - preCount, - null + output ); + request.setWorkHome(workHome); + request.setTimeout(timeout); + request.setPluginManager(this.pluginManager); + // 把该 executor 的 effective 配置整体作为 options，各执行器各取所需： + // Local 读 fetchSize/batchSize/preCount；Seatunnel 读 home/startScript/way/mode/engine + request.setOptions(executorCfg); // 进度回调：每个 batch 写完后更新 datacap_dataset_history 以及数据集自身的 totalRows / totalSize // totalRows 直接用已写入计数；totalSize 必须查 ClickHouse system.parts，开销大且不可控， diff --git a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/WorkflowServiceImpl.java b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/WorkflowServiceImpl.java index 69f7f138ff..3e1ddcc186 100644 --- a/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/WorkflowServiceImpl.java +++ b/core/datacap-service/src/main/java/io/edurt/datacap/service/service/impl/WorkflowServiceImpl.java @@ -6,10 +6,7 @@ import io.edurt.datacap.common.response.CommonResponse; import io.edurt.datacap.common.utils.NullAwareBeanUtils; import io.edurt.datacap.executor.ExecutorService; -import io.edurt.datacap.executor.common.RunEngine; -import io.edurt.datacap.executor.common.RunMode; import io.edurt.datacap.executor.common.RunState; -import io.edurt.datacap.executor.common.RunWay; import io.edurt.datacap.executor.configure.ExecutorConfigure; import io.edurt.datacap.executor.configure.ExecutorRequest; import io.edurt.datacap.executor.configure.ExecutorResponse; @@ -155,22 +152,18 @@ public CommonResponse saveOrUpdate(BaseRepository 0) request.fetchSize else DEFAULT_FETCH_SIZE - val batchSize = if (request.batchSize > 0) request.batchSize else DEFAULT_BATCH_SIZE + // Local 专属可调项从 options 读取（调用方把 executor 的 effective 配置塞进 options） + val options = request.options + val fetchSize = (options["fetchSize"]?.toIntOrNull()?.takeIf { it > 0 }) ?: DEFAULT_FETCH_SIZE + val batchSize = (options["batchSize"]?.toIntOrNull()?.takeIf { it > 0 }) ?: DEFAULT_BATCH_SIZE + val preCountEnabled = options["preCount"]?.toBoolean() ?: false val progressListener = request.progressListener taskLog.info( "Resolved input plugin={} output plugin={} streaming={}/{} fetchSize={} batchSize={} preCount={}", inputPlugin.name(), outputPlugin.name(), inputPlugin.supportsStreaming(), outputPlugin.supportsStreaming(), - fetchSize, batchSize, request.preCount + fetchSize, batchSize, preCountEnabled ) taskLog.info("Source query: {}", query) // 可选：先跑 SELECT COUNT(*) 拿到源端总行数，作为进度分母 // Optional: pre-count to populate the total row count used for progress percentage - val totalCount: Long = if (request.preCount) preCount(inputPlugin, inputConfigure, query, taskLog) else -1L + val totalCount: Long = if (preCountEnabled) preCount(inputPlugin, inputConfigure, query, taskLog) else -1L if (totalCount >= 0) { taskLog.info("Pre-count: source total rows = {}", totalCount) diff --git a/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java b/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java index 7f67d17c64..f3696137f3 100644 --- a/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java +++ b/executor/datacap-executor-seatunnel/src/main/java/io/edurt/datacap/executor/seatunnel/SeatunnelExecutorService.java @@ -6,7 +6,10 @@ import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.edurt.datacap.executor.ExecutorService; +import io.edurt.datacap.executor.common.RunEngine; +import io.edurt.datacap.executor.common.RunMode; import io.edurt.datacap.executor.common.RunState; +import io.edurt.datacap.executor.common.RunWay; import io.edurt.datacap.executor.configure.ExecutorConfigure; import io.edurt.datacap.executor.configure.ExecutorRequest; import io.edurt.datacap.executor.configure.ExecutorResponse; @@ -82,14 +85,19 @@ public ExecutorResponse stop(ExecutorRequest request) */ SeaTunnelCommander buildCommander(ExecutorRequest request) { + // Seatunnel 专属项来自 request.options：home / startScript / way / mode / engine + Map options = request.getOptions(); + RunWay runWay = RunWay.valueOf(options.getOrDefault("way", RunWay.LOCAL.name())); + RunMode runMode = RunMode.valueOf(options.getOrDefault("mode", RunMode.CLIENT.name())); + RunEngine runEngine = RunEngine.valueOf(options.getOrDefault("engine", RunEngine.SPARK.name())); return new SeaTunnelCommander( - request.getExecutorHome() + "/bin", - request.getStartScript(), - request.getRunWay().name().toLowerCase(), - request.getRunMode().name().toLowerCase(), + options.get("home") + "/bin", + options.get("startScript"), + runWay.name().toLowerCase(), + runMode.name().toLowerCase(), String.join(File.separator, request.getWorkHome(), request.getTaskName() + ".configure"), request.getTaskName(), - request.getRunEngine()); + runEngine); } /** diff --git a/executor/datacap-executor-spi/src/main/kotlin/io/edurt/datacap/executor/configure/ExecutorRequest.kt b/executor/datacap-executor-spi/src/main/kotlin/io/edurt/datacap/executor/configure/ExecutorRequest.kt index 7af6e66af9..58598eea80 100644 --- a/executor/datacap-executor-spi/src/main/kotlin/io/edurt/datacap/executor/configure/ExecutorRequest.kt +++ b/executor/datacap-executor-spi/src/main/kotlin/io/edurt/datacap/executor/configure/ExecutorRequest.kt @@ -1,80 +1,40 @@ package io.edurt.datacap.executor.configure import edu.umd.cs.findbugs.annotations.SuppressFBWarnings -import io.edurt.datacap.executor.common.RunEngine -import io.edurt.datacap.executor.common.RunMode -import io.edurt.datacap.executor.common.RunWay import io.edurt.datacap.plugin.PluginManager +/** + * 执行器请求。字段分两类： + * - 【所有执行器通用】taskName / userName / input / output / workHome / timeout / + * transform / pluginManager / progressListener + * - 【各执行器专属】统一放进 [options]（字符串 KV），由各执行器自行解析，例如： + * Local: fetchSize / batchSize / preCount + * Seatunnel: home / startScript / way / mode / engine + * + * 这样新增一个执行器无需改动本类，只需读取自己关心的 option。调用方通常把 + * “该执行器的 effective 配置 Map” 直接塞进 [options]。 + * + * Executor request split into executor-agnostic fields and an [options] bag of executor-specific + * string settings, so adding an executor no longer requires touching this class. + */ @SuppressFBWarnings(value = ["EI_EXPOSE_REP", "EI_EXPOSE_REP2"]) data class ExecutorRequest @JvmOverloads constructor( var taskName: String, var userName: String, var input: ExecutorConfigure, var output: ExecutorConfigure, - var executorHome: String? = null, var workHome: String? = null, - var pluginManager: PluginManager? = null, var timeout: Long = 600, - var runWay: RunWay = RunWay.LOCAL, - var runMode: RunMode = RunMode.CLIENT, - var startScript: String? = null, - var runEngine: RunEngine = RunEngine.SPARK, var transform: ExecutorConfigure? = null, - var fetchSize: Int = 1000, - var batchSize: Int = 1000, - var preCount: Boolean = false, - var progressListener: ExecutorProgressListener? = null + var options: Map = emptyMap(), + var progressListener: ExecutorProgressListener? = null, + var pluginManager: PluginManager? = null ) { + /** 便捷构造：仅指定 workHome 与源/汇，其余用默认值。主要给测试和简单调用使用。 */ constructor( workHome: String?, input: ExecutorConfigure, output: ExecutorConfigure - ) : this("", "", input, output, null, workHome, null, 600, RunWay.LOCAL, RunMode.CLIENT) - - constructor( - workHome: String?, - input: ExecutorConfigure, - output: ExecutorConfigure, - transform: ExecutorConfigure?, - runEngine: RunEngine = RunEngine.SPARK - ) : this("", "", input, output, null, workHome, null, 600, RunWay.LOCAL, RunMode.CLIENT, null, runEngine, transform) - - constructor( - workHome: String? = null, - executorHome: String? = null, - taskName: String, - userName: String, - input: ExecutorConfigure, - output: ExecutorConfigure, - runMode: RunMode = RunMode.CLIENT, - runWay: RunWay = RunWay.LOCAL - ) : this(taskName, userName, input, output, executorHome, workHome, null, 600, runWay, runMode) - - constructor( - workHome: String? = null, - executorHome: String? = null, - taskName: String, - userName: String, - input: ExecutorConfigure, - output: ExecutorConfigure, - runMode: RunMode = RunMode.CLIENT, - runWay: RunWay = RunWay.LOCAL, - startScript: String? - ) : this(taskName, userName, input, output, executorHome, workHome, null, 600, runWay, runMode, startScript) - - constructor( - workHome: String? = null, - executorHome: String? = null, - taskName: String, - userName: String, - input: ExecutorConfigure, - output: ExecutorConfigure, - runMode: RunMode = RunMode.CLIENT, - runWay: RunWay = RunWay.LOCAL, - startScript: String?, - runEngine: RunEngine = RunEngine.SPARK, - transform: ExecutorConfigure? = null - ) : this(taskName, userName, input, output, executorHome, workHome, null, 600, runWay, runMode, startScript, runEngine, transform) + ) : this("", "", input, output, workHome) } diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt index 2c5fe724d7..0506d04920 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/executor/seatunnel/SeatunnelCommanderMappingTest.kt @@ -1,8 +1,5 @@ package io.edurt.datacap.executor.seatunnel -import io.edurt.datacap.executor.common.RunEngine -import io.edurt.datacap.executor.common.RunMode -import io.edurt.datacap.executor.common.RunWay import io.edurt.datacap.executor.configure.ExecutorConfigure import io.edurt.datacap.executor.configure.ExecutorRequest import org.junit.Assert.assertEquals @@ -10,10 +7,11 @@ import org.junit.Test import java.io.File /** - * Seatunnel 安全网：锁定“ExecutorRequest 字段 -> seatunnel 命令行”的映射。 + * Seatunnel 安全网：锁定“ExecutorRequest -> seatunnel 命令行”的映射。 * - * 这段映射在 ExecutorRequest 结构调整（把 executor 专属字段收敛进 options）时最易出错。 - * 断言的是最终命令字符串——只要重构后 buildCommander 仍读到等价的值，命令就不变、测试保持绿。 + * ExecutorRequest 已把 executor 专属项收敛进 options（home/startScript/way/mode/engine）。 + * 这里断言最终命令字符串——只要 buildCommander 从 options 读到等价的值，命令就不变、测试保持绿。 + * 命令期望值与字段拆分前完全一致，用来证明拆分没有改变行为。 */ class SeatunnelCommanderMappingTest { @@ -27,12 +25,14 @@ class SeatunnelCommanderMappingTest userName = "tester", input = ExecutorConfigure("Jdbc"), output = ExecutorConfigure("Jdbc"), - executorHome = "/opt/seatunnel", workHome = "/work", - runWay = RunWay.LOCAL, - runMode = RunMode.CLIENT, - startScript = "start-seatunnel-spark-connector-v2.sh", - runEngine = RunEngine.SPARK + options = mapOf( + "home" to "/opt/seatunnel", + "startScript" to "start-seatunnel-spark-connector-v2.sh", + "way" to "LOCAL", + "mode" to "CLIENT", + "engine" to "SPARK" + ) ) val config = "/work" + File.separator + "task1.configure" @@ -51,12 +51,14 @@ class SeatunnelCommanderMappingTest userName = "tester", input = ExecutorConfigure("Jdbc"), output = ExecutorConfigure("Jdbc"), - executorHome = "/opt/seatunnel", workHome = "/work", - runWay = RunWay.LOCAL, - runMode = RunMode.CLIENT, - startScript = "seatunnel.sh", - runEngine = RunEngine.SEATUNNEL + options = mapOf( + "home" to "/opt/seatunnel", + "startScript" to "seatunnel.sh", + "way" to "LOCAL", + "mode" to "CLIENT", + "engine" to "SEATUNNEL" + ) ) val config = "/work" + File.separator + "task2.configure" diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt index 3051f537d0..f0ea7b3f24 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorCharacterizationTest.kt @@ -95,7 +95,7 @@ class LocalExecutorCharacterizationTest source, sink, originColumns = linkedSetOf(OriginColumn("id", "id")) ).apply { - preCount = true + options = mapOf("preCount" to "true") progressListener = ExecutorProgressListener { processed, total -> progress.add(processed to total) } } diff --git a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt index 21f62064dd..1c2ff34e6a 100644 --- a/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt +++ b/test/datacap-test-executor/src/test/kotlin/io/edurt/datacap/test/local/LocalExecutorPostgresE2ETest.kt @@ -131,7 +131,7 @@ class LocalExecutorPostgresE2ETest OriginColumn("amt", "amount") ) ).apply { - preCount = true + options = mapOf("preCount" to "true") progressListener = ExecutorProgressListener { processed, total -> progress.add(processed to total) } } From 0d5fadbf5c1fbe9f33484aa7c4becc32a44518d4 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 5 Sep 2026 10:49:44 +0800 Subject: [PATCH 09/17] feat(ui): add ant-design-vue foundation alongside view-shadcn-ui Introduce ant-design-vue 4 and wire it globally next to the existing view-shadcn-ui so components can be migrated page by page without breaking untouched pages. Add composables for cross-cutting concerns: - useTheme: single-source dark/light switch (persisted, toggles the html `dark` class and drives antd's dark/default algorithm) - useAntdLocale: maps the vue-i18n locale to antd's built-in locale App.vue now wraps the router view in a-config-provider (theme + locale). No reset.css yet to avoid affecting not-yet-migrated pages. Pin pnpm onlyBuiltDependencies to keep pnpm 10 CI from failing on ignored builds. --- core/datacap-ui/package.json | 9 + core/datacap-ui/pnpm-lock.yaml | 199 ++++++++++++++++++ core/datacap-ui/src/App.vue | 14 +- .../src/composables/useAntdLocale.ts | 31 +++ core/datacap-ui/src/composables/useTheme.ts | 63 ++++++ core/datacap-ui/src/main.ts | 5 + 6 files changed, 318 insertions(+), 3 deletions(-) create mode 100644 core/datacap-ui/src/composables/useAntdLocale.ts create mode 100644 core/datacap-ui/src/composables/useTheme.ts diff --git a/core/datacap-ui/package.json b/core/datacap-ui/package.json index 52ff84dd88..15e5be3b42 100644 --- a/core/datacap-ui/package.json +++ b/core/datacap-ui/package.json @@ -9,6 +9,7 @@ "preview": "vite preview" }, "dependencies": { + "@ant-design/icons-vue": "^7.0.1", "@antv/x6": "^2.18.1", "@fortawesome/fontawesome-svg-core": "^6.5.1", "@fortawesome/free-solid-svg-icons": "^6.5.1", @@ -22,6 +23,7 @@ "ag-grid-community": "^31.3.4", "ag-grid-vue3": "^31.3.4", "ansi_up": "^6.0.2", + "ant-design-vue": "^4.2.6", "axios": "^1.7.4", "clsx": "^2.1.0", "lodash": "^4.17.21", @@ -45,5 +47,12 @@ "typescript": "^5.2.2", "vite": "^5.4.8", "vue-tsc": "^1.8.27" + }, + "pnpm": { + "onlyBuiltDependencies": [ + "esbuild", + "vue-demi", + "core-js" + ] } } diff --git a/core/datacap-ui/pnpm-lock.yaml b/core/datacap-ui/pnpm-lock.yaml index 557238bf6b..efc37d89ec 100644 --- a/core/datacap-ui/pnpm-lock.yaml +++ b/core/datacap-ui/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@ant-design/icons-vue': + specifier: ^7.0.1 + version: 7.0.1(vue@3.5.12(typescript@5.2.2)) '@antv/x6': specifier: ^2.18.1 version: 2.18.1 @@ -47,6 +50,9 @@ importers: ansi_up: specifier: ^6.0.2 version: 6.0.2 + ant-design-vue: + specifier: ^4.2.6 + version: 4.2.6(vue@3.5.12(typescript@5.2.2)) axios: specifier: ^1.7.4 version: 1.7.7 @@ -114,6 +120,17 @@ importers: packages: + '@ant-design/colors@6.0.0': + resolution: {integrity: sha512-qAZRvPzfdWHtfameEGP2Qvuf838NhergR35o+EuVyB5XvSA98xod5r4utvi4TJ3ywmevm290g9nsCG5MryrdWQ==} + + '@ant-design/icons-svg@4.6.0': + resolution: {integrity: sha512-PRomU725ABMf/lnQp5HiB7my1kjEbFY0D10N4lXYxK6TIB1gKjIVD5MRThpDaezLgw1D774J8eOeeDB0M6wHrQ==} + + '@ant-design/icons-vue@7.0.1': + resolution: {integrity: sha512-eCqY2unfZK6Fe02AwFlDHLfoyEFreP6rBwAZMIJ1LugmfMiVgwWDYlp1YsRugaPtICYOabV1iWxXdP12u9U43Q==} + peerDependencies: + vue: '>=3.0.3' + '@antv/x6-common@2.0.17': resolution: {integrity: sha512-37g7vmRkNdYzZPdwjaMSZEGv/MMH0S4r70/Jwoab1mioycmuIBN73iyziX8m56BvJSDucZ3J/6DU07otWqzS6A==} @@ -136,6 +153,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + '@babel/types@7.26.0': resolution: {integrity: sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==} engines: {node: '>=6.9.0'} @@ -235,6 +256,16 @@ packages: '@codemirror/view@6.34.1': resolution: {integrity: sha512-t1zK/l9UiRqwUNPm+pdIT0qzJlzuVckbTEMVNFhfWkGiBQClstzg+78vedCvLSX0xJEZ6lwZbPpnljL7L6iwMQ==} + '@ctrl/tinycolor@3.6.1': + resolution: {integrity: sha512-SITSV6aIXsuVNV3f3O0f2n/cgyEDWoSqtZMYiAmcsYHydcKrOz3gUxB/iXd/Qf08+IZX4KpgNbvUdMBmWz+kcA==} + engines: {node: '>=10'} + + '@emotion/hash@0.9.2': + resolution: {integrity: sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==} + + '@emotion/unitless@0.8.1': + resolution: {integrity: sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ==} + '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} engines: {node: '>=12'} @@ -540,24 +571,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@resvg/resvg-js-linux-arm64-musl@2.4.1': resolution: {integrity: sha512-6mT0+JBCsermKMdi/O2mMk3m7SqOjwi9TKAwSngRZ/nQoL3Z0Z5zV+572ztgbWr0GODB422uD8e9R9zzz38dRQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@resvg/resvg-js-linux-x64-gnu@2.4.1': resolution: {integrity: sha512-60KnrscLj6VGhkYOJEmmzPlqqfcw1keDh6U+vMcNDjPhV3B5vRSkpP/D/a8sfokyeh4VEacPSYkWGezvzS2/mg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@resvg/resvg-js-linux-x64-musl@2.4.1': resolution: {integrity: sha512-0AMyZSICC1D7ge115cOZQW8Pcad6PjWuZkBFF3FJuSxC6Dgok0MQnLTs2MfMdKBlAcwO9dXsf3bv9tJZj8pATA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@resvg/resvg-js-win32-arm64-msvc@2.4.1': resolution: {integrity: sha512-76XDFOFSa3d0QotmcNyChh2xHwk+JTFiEQBVxMlHpHMeq7hNrQJ1IpE1zcHSQvrckvkdfLboKRrlGB86B10Qjw==} @@ -615,46 +650,55 @@ packages: resolution: {integrity: sha512-KRSFHyE/RdxQ1CSeOIBVIAxStFC/hnBgVcaiCkQaVC+EYDtTe4X7z5tBkFyRoBgUGtB6Xg6t9t2kulnX6wJc6A==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.24.3': resolution: {integrity: sha512-h6Q8MT+e05zP5BxEKz0vi0DhthLdrNEnspdLzkoFqGwnmOzakEHSlXfVyA4HJ322QtFy7biUAVFPvIDEDQa6rw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.24.3': resolution: {integrity: sha512-fKElSyXhXIJ9pqiYRqisfirIo2Z5pTTve5K438URf08fsypXrEkVmShkSfM8GJ1aUyvjakT+fn2W7Czlpd/0FQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.24.3': resolution: {integrity: sha512-YlddZSUk8G0px9/+V9PVilVDC6ydMz7WquxozToozSnfFK6wa6ne1ATUjUvjin09jp34p84milxlY5ikueoenw==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-powerpc64le-gnu@4.24.3': resolution: {integrity: sha512-yNaWw+GAO8JjVx3s3cMeG5Esz1cKVzz8PkTJSfYzE5u7A+NvGmbVFEHP+BikTIyYWuz0+DX9kaA3pH9Sqxp69g==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.24.3': resolution: {integrity: sha512-lWKNQfsbpv14ZCtM/HkjCTm4oWTKTfxPmr7iPfp3AHSqyoTz5AgLemYkWLwOBWc+XxBbrU9SCokZP0WlBZM9lA==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-s390x-gnu@4.24.3': resolution: {integrity: sha512-HoojGXTC2CgCcq0Woc/dn12wQUlkNyfH0I1ABK4Ni9YXyFQa86Fkt2Q0nqgLfbhkyfQ6003i3qQk9pLh/SpAYw==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.24.3': resolution: {integrity: sha512-mnEOh4iE4USSccBOtcrjF5nj+5/zm6NcNhbSEfR3Ot0pxBwvEn5QVUXcuOwwPkapDtGZ6pT02xLoPaNv06w7KQ==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.24.3': resolution: {integrity: sha512-rMTzawBPimBQkG9NKpNHvquIUTQPzrnPxPbCY1Xt+mFkW7pshvyIS5kYgcf74goxXOQk0CP3EoOC1zcEezKXhw==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.24.3': resolution: {integrity: sha512-2lg1CE305xNvnH3SyiKwPVsTVLCg4TmNCF1z7PSHX2uZY2VbUpdkgAllVoISD7JO7zu+YynpWNSKAtOrX3AiuA==} @@ -671,6 +715,9 @@ packages: cpu: [x64] os: [win32] + '@simonwep/pickr@1.8.2': + resolution: {integrity: sha512-/l5w8BIkrpP6n1xsetx9MWPWlU6OblN5YgZZphxan0Tq4BByTCETL6lyIeY8lagalS2Nbt4F2W034KHLIiunKA==} + '@turf/boolean-clockwise@6.5.0': resolution: {integrity: sha512-45+C7LC5RMbRWrxh3Z0Eihsc8db1VGBO5d9BLTOAwU4jR6SgsunTfRWR16X7JUwIDYlCVEmnjcXJNi/kIU3VIw==} @@ -917,12 +964,24 @@ packages: ansi_up@6.0.2: resolution: {integrity: sha512-3G3vKvl1ilEp7J1u6BmULpMA0xVoW/f4Ekqhl8RTrJrhEBkonKn5k3bUc5Xt+qDayA6iDX0jyUh3AbZjB/l0tw==} + ant-design-vue@4.2.6: + resolution: {integrity: sha512-t7eX13Yj3i9+i5g9lqFyYneoIb3OzTvQjq9Tts1i+eiOd3Eva/6GagxBSXM1fOCjqemIu0FYVE1ByZ/38epR3Q==} + engines: {node: '>=12.22.0'} + peerDependencies: + vue: '>=3.2.0' + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} array-source@0.0.4: resolution: {integrity: sha512-frNdc+zBn80vipY+GdcJkLEbMWj3xmzArYApmUGxoiV8uAu/ygcs9icPdsGdA26h0MkHUMW6EN2piIvVx+M5Mw==} + array-tree-filter@2.1.0: + resolution: {integrity: sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==} + + async-validator@4.2.5: + resolution: {integrity: sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==} + asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -975,6 +1034,9 @@ packages: commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} + compute-scroll-into-view@1.0.20: + resolution: {integrity: sha512-UCB0ioiyj8CRjtrvaceBLqqhZCVP+1B8+NWQhmdsm0VXOJtobBCf1dBQmebCCo34qZmUwZfIH2MZLqNHazrfjg==} + computeds@0.0.1: resolution: {integrity: sha512-7CEBgcMjVmitjYo5q8JTJVra6X5mQ20uTThdK+0kR7UEaDrAWEQcRiBtWJzga4eRpP6afNwwLsX2SET2JhVB1Q==} @@ -993,6 +1055,9 @@ packages: copy-to-clipboard@3.3.3: resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} + core-js@3.50.0: + resolution: {integrity: sha512-BRWgOLKkFeCgRudR6zrs8p9XJZcE14grzKMMssoYrk6krtuEZ7MTKPIY5RzOnqsEKIR9kst7wNzphttraT+Yqw==} + core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} @@ -1086,6 +1151,12 @@ packages: dijkstrajs@1.0.3: resolution: {integrity: sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA==} + dom-align@1.12.4: + resolution: {integrity: sha512-R8LUSEay/68zE5c8/3BDxiTEvgb4xZTF0RKmAHfiEVN3klfIpXfi2/QCoiWPccVQ0J/ZGdz9OjzL4uJEP/MRAw==} + + dom-scroll-into-view@2.0.1: + resolution: {integrity: sha512-bvVTQe1lfaUr1oFzZX80ce9KLDlZ3iU+XGNE/bz9HnGdklTieqsbmsLHe+rT2XWqopvL0PckkYqN7ksmm5pe3w==} + element-resize-detector@1.2.4: resolution: {integrity: sha512-Fl5Ftk6WwXE0wqCgNoseKWndjzZlDCwuPTcoVZfCP9R3EHQF8qUtr3YUPNETegRBOKqQKPW3n4kiIWngGi8tKg==} @@ -1181,6 +1252,10 @@ packages: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + is-plain-object@3.0.1: + resolution: {integrity: sha512-Xnpx182SBMrr/aBik8y+GuR4U1L9FqMSojwDQwPMmxyC6bvEqly9UBCxhauBF5vNh2gwWJNX6oDV7O+OM4z34g==} + engines: {node: '>=0.10.0'} + is-what@4.1.16: resolution: {integrity: sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==} engines: {node: '>=12.13'} @@ -1188,6 +1263,9 @@ packages: isarray@0.0.1: resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + linkify-it@5.0.0: resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} @@ -1201,6 +1279,10 @@ packages: lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -1276,6 +1358,9 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + nanopop@2.4.2: + resolution: {integrity: sha512-NzOgmMQ+elxxHeIha+OG/Pv3Oc3p4RU2aBhwWwAqDpXrdTbtRylbRLQztLy8dMMwfl6pclznBdfUhccEn9ZIzw==} + p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -1395,6 +1480,9 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + scroll-into-view-if-needed@2.2.31: + resolution: {integrity: sha512-dGCXy99wZQivjmjIqihaBQNjryrz5rueJY7eHfTdyWEiR4ttYpsajb14rn9s5d4DY4EcY6+4+U/maARBXJedkA==} + select@1.1.2: resolution: {integrity: sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==} @@ -1406,6 +1494,9 @@ packages: set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} + shallow-equal@1.2.1: + resolution: {integrity: sha512-S4vJDjHHMBaiZuT9NPb616CSmLf618jawtv3sufLl6ivK8WocjAo58cXwbRV1cgqxH0Qbv+iUt6m05eqEa2IRA==} + shapefile@0.6.6: resolution: {integrity: sha512-rLGSWeK2ufzCVx05wYd+xrWnOOdSV7xNUW5/XFgx3Bc02hBkpMlrd2F1dDII7/jhWzv0MSyBFh5uJIy9hLdfuw==} hasBin: true @@ -1454,6 +1545,9 @@ packages: style-mod@4.1.2: resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==} + stylis@4.4.0: + resolution: {integrity: sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA==} + superjson@2.2.2: resolution: {integrity: sha512-5JRxVqC8I8NuOUjzBbvVJAKNM8qoVuH0O77h4WInc/qC2q5IreqKxYwgkga3PfA22OayK2ikceb/B26dztPl+Q==} engines: {node: '>=16'} @@ -1464,6 +1558,10 @@ packages: text-encoding@0.6.4: resolution: {integrity: sha512-hJnc6Qg3dWoOMkqP53F0dzRIgtmsAge09kxUIqGrEUS4qr5rWLckGYaQAVr+opBrIMRErGgy6f5aPnyPpyGRfg==} + throttle-debounce@5.0.2: + resolution: {integrity: sha512-B71/4oyj61iNH0KeCamLuE2rmKuTO5byTOSVwECM5FA7TiAiAW+UqTKZ9ERueC4qvgSttUhdmq1mXC3kJqGX7A==} + engines: {node: '>=12.22'} + tiny-emitter@2.1.0: resolution: {integrity: sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==} @@ -1579,6 +1677,12 @@ packages: peerDependencies: typescript: '*' + vue-types@3.0.2: + resolution: {integrity: sha512-IwUC0Aq2zwaXqy74h4WCvFCUtoV0iSWr0snWnE9TnU18S66GAQyqQbRf2qfJtUuiFsBf6qp0MEwdonlwznlcrw==} + engines: {node: '>=10.15.0'} + peerDependencies: + vue: ^3.0.0 + vue3-ace-editor@2.2.4: resolution: {integrity: sha512-FZkEyfpbH068BwjhMyNROxfEI8135Sc+x8ouxkMdCNkuj/Tuw83VP/gStFQqZHqljyX9/VfMTCdTqtOnJZGN8g==} peerDependencies: @@ -1604,6 +1708,9 @@ packages: w3c-keyname@2.2.8: resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + warning@4.0.3: + resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} + which-module@2.0.1: resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==} @@ -1629,6 +1736,18 @@ packages: snapshots: + '@ant-design/colors@6.0.0': + dependencies: + '@ctrl/tinycolor': 3.6.1 + + '@ant-design/icons-svg@4.6.0': {} + + '@ant-design/icons-vue@7.0.1(vue@3.5.12(typescript@5.2.2))': + dependencies: + '@ant-design/colors': 6.0.0 + '@ant-design/icons-svg': 4.6.0 + vue: 3.5.12(typescript@5.2.2) + '@antv/x6-common@2.0.17': dependencies: lodash-es: 4.17.21 @@ -1650,6 +1769,8 @@ snapshots: dependencies: '@babel/types': 7.26.0 + '@babel/runtime@7.29.7': {} + '@babel/types@7.26.0': dependencies: '@babel/helper-string-parser': 7.25.9 @@ -1910,6 +2031,12 @@ snapshots: style-mod: 4.1.2 w3c-keyname: 2.2.8 + '@ctrl/tinycolor@3.6.1': {} + + '@emotion/hash@0.9.2': {} + + '@emotion/unitless@0.8.1': {} + '@esbuild/aix-ppc64@0.21.5': optional: true @@ -2263,6 +2390,11 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.24.3': optional: true + '@simonwep/pickr@1.8.2': + dependencies: + core-js: 3.50.0 + nanopop: 2.4.2 + '@turf/boolean-clockwise@6.5.0': dependencies: '@turf/helpers': 6.5.0 @@ -2691,10 +2823,40 @@ snapshots: ansi_up@6.0.2: {} + ant-design-vue@4.2.6(vue@3.5.12(typescript@5.2.2)): + dependencies: + '@ant-design/colors': 6.0.0 + '@ant-design/icons-vue': 7.0.1(vue@3.5.12(typescript@5.2.2)) + '@babel/runtime': 7.29.7 + '@ctrl/tinycolor': 3.6.1 + '@emotion/hash': 0.9.2 + '@emotion/unitless': 0.8.1 + '@simonwep/pickr': 1.8.2 + array-tree-filter: 2.1.0 + async-validator: 4.2.5 + csstype: 3.1.3 + dayjs: 1.11.13 + dom-align: 1.12.4 + dom-scroll-into-view: 2.0.1 + lodash: 4.17.21 + lodash-es: 4.17.21 + resize-observer-polyfill: 1.5.1 + scroll-into-view-if-needed: 2.2.31 + shallow-equal: 1.2.1 + stylis: 4.4.0 + throttle-debounce: 5.0.2 + vue: 3.5.12(typescript@5.2.2) + vue-types: 3.0.2(vue@3.5.12(typescript@5.2.2)) + warning: 4.0.3 + argparse@2.0.1: {} array-source@0.0.4: {} + array-tree-filter@2.1.0: {} + + async-validator@4.2.5: {} + asynckit@0.4.0: {} axios@1.7.7: @@ -2757,6 +2919,8 @@ snapshots: commander@2.20.3: {} + compute-scroll-into-view@1.0.20: {} + computeds@0.0.1: {} concat-stream@1.4.11: @@ -2780,6 +2944,8 @@ snapshots: dependencies: toggle-selection: 1.0.6 + core-js@3.50.0: {} + core-util-is@1.0.3: {} crelt@1.0.6: {} @@ -2856,6 +3022,10 @@ snapshots: dijkstrajs@1.0.3: {} + dom-align@1.12.4: {} + + dom-scroll-into-view@2.0.1: {} + element-resize-detector@1.2.4: dependencies: batch-processor: 1.0.0 @@ -2961,10 +3131,14 @@ snapshots: is-fullwidth-code-point@3.0.0: {} + is-plain-object@3.0.1: {} + is-what@4.1.16: {} isarray@0.0.1: {} + js-tokens@4.0.0: {} + linkify-it@5.0.0: dependencies: uc.micro: 2.1.0 @@ -2977,6 +3151,10 @@ snapshots: lodash@4.17.21: {} + loose-envify@1.4.0: + dependencies: + js-tokens: 4.0.0 + lru-cache@10.4.3: {} lucide-vue-next@0.360.0(vue@3.5.12(typescript@5.2.2)): @@ -3052,6 +3230,8 @@ snapshots: nanoid@3.3.7: {} + nanopop@2.4.2: {} + p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -3185,12 +3365,18 @@ snapshots: safer-buffer@2.1.2: {} + scroll-into-view-if-needed@2.2.31: + dependencies: + compute-scroll-into-view: 1.0.20 + select@1.1.2: {} semver@7.6.3: {} set-blocking@2.0.0: {} + shallow-equal@1.2.1: {} + shapefile@0.6.6: dependencies: array-source: 0.0.4 @@ -3238,6 +3424,8 @@ snapshots: style-mod@4.1.2: {} + stylis@4.4.0: {} + superjson@2.2.2: dependencies: copy-anything: 3.0.5 @@ -3246,6 +3434,8 @@ snapshots: text-encoding@0.6.4: {} + throttle-debounce@5.0.2: {} + tiny-emitter@2.1.0: {} toggle-selection@1.0.6: {} @@ -3334,6 +3524,11 @@ snapshots: semver: 7.6.3 typescript: 5.2.2 + vue-types@3.0.2(vue@3.5.12(typescript@5.2.2)): + dependencies: + is-plain-object: 3.0.1 + vue: 3.5.12(typescript@5.2.2) + vue3-ace-editor@2.2.4(ace-builds@1.36.3)(vue@3.5.12(typescript@5.2.2)): dependencies: ace-builds: 1.36.3 @@ -3373,6 +3568,10 @@ snapshots: w3c-keyname@2.2.8: {} + warning@4.0.3: + dependencies: + loose-envify: 1.4.0 + which-module@2.0.1: {} wrap-ansi@6.2.0: diff --git a/core/datacap-ui/src/App.vue b/core/datacap-ui/src/App.vue index d3b95e045c..60c2b34aa6 100644 --- a/core/datacap-ui/src/App.vue +++ b/core/datacap-ui/src/App.vue @@ -1,8 +1,16 @@ diff --git a/core/datacap-ui/src/composables/useAntdLocale.ts b/core/datacap-ui/src/composables/useAntdLocale.ts new file mode 100644 index 0000000000..2334374803 --- /dev/null +++ b/core/datacap-ui/src/composables/useAntdLocale.ts @@ -0,0 +1,31 @@ +import { computed } from 'vue' +import { useI18n } from 'vue-i18n' +import zhCN from 'ant-design-vue/es/locale/zh_CN' +import enUS from 'ant-design-vue/es/locale/en_US' +import type { Locale } from 'ant-design-vue/es/locale' + +/** + * 把应用当前 vue-i18n 语言映射到 ant-design-vue 的内置 locale， + * 传给 ，让 antd 组件（分页、表格空数据、日期等）文案跟随全局语言。 + * + * 这样迁移后不再需要 view-shadcn-ui 的 setLocale——antd 内置文案与业务 i18n 收敛为同一套来源。 + */ +const LOCALE_MAP: Record = { + zh_cn: zhCN, + 'zh-cn': zhCN, + zh: zhCN, + en: enUS, + en_us: enUS, + 'en-us': enUS +} + +export function useAntdLocale() { + const { locale } = useI18n() + + const antdLocale = computed(() => { + const key = (locale.value || '').toLowerCase() + return LOCALE_MAP[key] || enUS + }) + + return { antdLocale } +} diff --git a/core/datacap-ui/src/composables/useTheme.ts b/core/datacap-ui/src/composables/useTheme.ts new file mode 100644 index 0000000000..87a8d3cdf5 --- /dev/null +++ b/core/datacap-ui/src/composables/useTheme.ts @@ -0,0 +1,63 @@ +import { computed, ref, watch } from 'vue' +import { theme } from 'ant-design-vue' + +/** + * 全局主题（明/暗）composable。 + * + * - 单例状态：模块级 `isDark`，全应用共享，任意组件调用 useTheme() 都拿到同一个开关。 + * - 持久化：写入 localStorage，刷新后保持。 + * - 作用于两处：给 加 `dark` class（供自定义 CSS 用 `.dark xxx` 定制）， + * 同时导出 antd 的主题算法给 。 + * + * 迁移到 ant-design-vue 后，暗黑模式由 antd 的 darkAlgorithm 负责组件； + * 自定义（CSS 手写）组件则通过 上的类名做暗色适配。 + */ +const STORAGE_KEY = 'datacap-theme' + +const isDark = ref(localStorage.getItem(STORAGE_KEY) === 'dark') + +function applyToDocument(dark: boolean): void { + const el = document.documentElement + if (dark) { + el.classList.add('dark') + } + else { + el.classList.remove('dark') + } + el.setAttribute('data-theme', dark ? 'dark' : 'light') +} + +// 首次加载即应用一次 +applyToDocument(isDark.value) + +watch(isDark, (value) => { + localStorage.setItem(STORAGE_KEY, value ? 'dark' : 'light') + applyToDocument(value) +}) + +export function useTheme() { + /** 传给 的主题配置。 */ + const antdTheme = computed(() => ({ + algorithm: isDark.value ? theme.darkAlgorithm : theme.defaultAlgorithm, + token: { + // 品牌主色，后续可按 DataCap 设计调整 + colorPrimary: '#1677ff', + borderRadius: 6 + } + })) + + const toggle = (): void => { + isDark.value = !isDark.value + } + + const setDark = (value: boolean): void => { + isDark.value = value + } + + return { + isDark, + antdTheme, + toggle, + setDark + } +} diff --git a/core/datacap-ui/src/main.ts b/core/datacap-ui/src/main.ts index a413fd90ef..7ff1c9b0b6 100644 --- a/core/datacap-ui/src/main.ts +++ b/core/datacap-ui/src/main.ts @@ -5,6 +5,7 @@ import App from './App.vue' import router from '@/router' import i18n from '@/i18n/I18n' import { createIcons } from '@/fontawesome' +import Antd from 'ant-design-vue' // @ts-ignore import ShadcnViewUI from 'view-shadcn-ui' @@ -15,5 +16,9 @@ app.config.warnHandler = () => null app.use(router) app.use(i18n) app.use(createPinia()) +// ant-design-vue 与旧的 view-shadcn-ui 暂时共存：a-* 用 antd，Shadcn* 仍走旧库， +// 逐页迁移完成后再移除 view-shadcn-ui。antd v4 为 CSS-in-JS，无需全局引入 reset.css， +// 以免影响尚未迁移的旧库页面（迁移收尾时再统一引入 reset）。 +app.use(Antd) app.use(ShadcnViewUI) app.provide('$t', i18n.global.t).mount('#app') From 144a626d3c3a1a30197b1360296bf15cbcada0c0 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 5 Sep 2026 10:52:39 +0800 Subject: [PATCH 10/17] refactor(ui): migrate language switcher to ant-design-vue Replace ShadcnSelect with a-select (grouped options) and drop the view-shadcn-ui setLocale call; antd's built-in strings now follow the vue-i18n locale through a-config-provider. loadLocale already updates the global locale, so switching language reactively updates both app and antd text. --- .../components/LanguageSwitcher.vue | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/core/datacap-ui/src/views/layouts/common/components/components/LanguageSwitcher.vue b/core/datacap-ui/src/views/layouts/common/components/components/LanguageSwitcher.vue index 1e06cb317e..f3957e4107 100644 --- a/core/datacap-ui/src/views/layouts/common/components/components/LanguageSwitcher.vue +++ b/core/datacap-ui/src/views/layouts/common/components/components/LanguageSwitcher.vue @@ -1,43 +1,38 @@ From 0380d6eadb91a9db9db70924d5531e2ce483dd9e Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 5 Sep 2026 11:06:45 +0800 Subject: [PATCH 12/17] refactor(ui): migrate sign-in page to ant-design-vue Rebuild the sign-in form on antd: a-form (:model + @finish/@finishFailed) with a-form-item rules, a-input / a-input-password, submit via html-type=submit, and a-card / a-avatar / a-divider / a-space / a-spin. Global $Message calls become antd message; validation-error reporting adapts to antd's errorFields shape. Field rules and behaviour are unchanged. --- .../src/views/auth/signin/AuthSignin.vue | 160 ++++++++---------- 1 file changed, 74 insertions(+), 86 deletions(-) diff --git a/core/datacap-ui/src/views/auth/signin/AuthSignin.vue b/core/datacap-ui/src/views/auth/signin/AuthSignin.vue index 340ece59c4..93a1458a7c 100644 --- a/core/datacap-ui/src/views/auth/signin/AuthSignin.vue +++ b/core/datacap-ui/src/views/auth/signin/AuthSignin.vue @@ -2,92 +2,90 @@
- + - +
+ {{ $t('user.auth.signinTip') }} +
- - - - - +
+ +
+ + + + - - - + + + - +
- - + + - +
- + - - + + {{ $t('user.common.signin') }} - + - + + {{ $t('user.auth.notUserTip') }} + - + {{ $t('user.common.signup') }} - - - + + +
- +
@@ -95,6 +93,7 @@ \ No newline at end of file + From 4da644d710f5e985ba3025098af3f216fc04b740 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 5 Sep 2026 11:10:14 +0800 Subject: [PATCH 13/17] refactor(ui): migrate sign-up page to ant-design-vue Mirror the sign-in migration for sign-up: antd a-form with a-input / a-input-password, submit via html-type=submit, a-card / a-avatar / a-divider / a-space / a-spin, and antd message. The confirm-password custom validator now uses antd's (rule, value) signature; rules and behaviour unchanged. --- .../src/views/auth/signup/AuthSignup.vue | 170 +++++++++--------- 1 file changed, 81 insertions(+), 89 deletions(-) diff --git a/core/datacap-ui/src/views/auth/signup/AuthSignup.vue b/core/datacap-ui/src/views/auth/signup/AuthSignup.vue index 1ee3b327d9..cf9812725f 100644 --- a/core/datacap-ui/src/views/auth/signup/AuthSignup.vue +++ b/core/datacap-ui/src/views/auth/signup/AuthSignup.vue @@ -2,104 +2,100 @@
- + - +
+ {{ $t('user.auth.signupTip') }} +
- - - - - +
+ +
+ + + + - - - + + + - - - + + + - +
- - + + - +
- + - - + + {{ $t('user.common.signup') }} - + - + + {{ $t('user.auth.hasUserTip') }} + - + {{ $t('user.common.signin') }} - - - + + +
- +
@@ -107,6 +103,7 @@ diff --git a/core/datacap-ui/src/views/pages/system/user/UserUtils.ts b/core/datacap-ui/src/views/pages/system/user/UserUtils.ts index d009c4c221..8c79d53b09 100644 --- a/core/datacap-ui/src/views/pages/system/user/UserUtils.ts +++ b/core/datacap-ui/src/views/pages/system/user/UserUtils.ts @@ -5,13 +5,15 @@ export function useHeaders() { const { t } = useI18n() + // ant-design-vue Table 列格式：title + dataIndex/key； + // role / action 列没有 dataIndex，通过 - diff --git a/core/datacap-ui/src/views/common/error/NotNetwork.vue b/core/datacap-ui/src/views/common/error/NotNetwork.vue index 8785198a3c..34068224ba 100644 --- a/core/datacap-ui/src/views/common/error/NotNetwork.vue +++ b/core/datacap-ui/src/views/common/error/NotNetwork.vue @@ -20,13 +20,9 @@ - From a2f0ddcfb53abce6822440d7328434bb137b55a2 Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 5 Sep 2026 11:22:34 +0800 Subject: [PATCH 16/17] refactor(ui): convert auth pages to script setup composition Convert AuthSignin / AuthSignup from Options API to diff --git a/core/datacap-ui/src/views/auth/signup/AuthSignup.vue b/core/datacap-ui/src/views/auth/signup/AuthSignup.vue index cf9812725f..ec37eca4be 100644 --- a/core/datacap-ui/src/views/auth/signup/AuthSignup.vue +++ b/core/datacap-ui/src/views/auth/signup/AuthSignup.vue @@ -101,8 +101,9 @@ - From 8a829e0fe5124d69083fb1f213bc84c67f3c9ada Mon Sep 17 00:00:00 2001 From: qianmoQ Date: Sat, 5 Sep 2026 11:31:10 +0800 Subject: [PATCH 17/17] refactor(ui): migrate user-role dialog to ant-design-vue + composition API Proves the modal pattern: ShadcnModal -> a-modal (v-model:open, footer=null with the form's own submit button); ShadcnCheckboxGroup/Checkbox -> a-checkbox-group / a-checkbox; a-spin wrapper. Convert to