From ee0e14b1f6b8474872cbd6179e626de65f448c8a Mon Sep 17 00:00:00 2001 From: Magnus Persson <1431685+magnusp@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:42:10 +0200 Subject: [PATCH 1/2] docs: rewrite TESTCONTAINERS-USAGE.md against the current code --- TESTCONTAINERS-USAGE.md | 163 ++++++++++++++++++++++++++++++++-------- 1 file changed, 131 insertions(+), 32 deletions(-) diff --git a/TESTCONTAINERS-USAGE.md b/TESTCONTAINERS-USAGE.md index b41b1b1..2cafd7b 100644 --- a/TESTCONTAINERS-USAGE.md +++ b/TESTCONTAINERS-USAGE.md @@ -6,7 +6,14 @@ This document explains how Testcontainers is configured and utilized in this pro ## 1. Overview & Image Selection -Testing a pure-Java JDBC driver for Turso requires verifying real SQLite behavior, lock contention, HTTP pipeline batons, and network edge cases over the wire. Rather than mocking HTTP endpoints, we run real `sqld` instances using Testcontainers. +Testing a pure-Java JDBC driver for libSQL requires verifying real SQLite behavior, lock contention, HTTP pipeline batons, and network edge cases over the wire. Rather than mocking HTTP endpoints, we run real `sqld` instances using Testcontainers. + +The integration tests live in `libsql-itests/src/test/java/com/github/magnusp/libsql/itest/`: + +* `SqldContainerBase` — shared singleton `sqld` container that the JDBC and client tests extend. +* `LibsqlJdbcIT`, `LibsqlHttpClientIT`, `HikariCpIntegrationIT`, `TransactionLifecycleIT`, `GeneratedKeysIT`, and `DatabaseMetaDataIT` — extend `SqldContainerBase`. +* `NetworkToxicityIT` — standalone Toxiproxy scenario with its own `sqld` container on a dedicated network. +* `NamespacePathIT` — standalone `sqld` container started with namespaces enabled and an admin port. ### Container Image * **Image**: `ghcr.io/tursodatabase/libsql-server:v0.24.33` @@ -25,11 +32,11 @@ test JVMs. To test a different version, override the property: ## 2. Base Container Setup (`SqldContainerBase.java`) -All standard integration tests inherit from [`SqldContainerBase`](file:///home/magnus/src/local/turso-java/src/test/java/org/libsql/jdbc/base/SqldContainerBase.java) to share a managed, singleton container instance across test executions. +All standard integration tests inherit from [`SqldContainerBase`](libsql-itests/src/test/java/com/github/magnusp/libsql/itest/SqldContainerBase.java) in the `com.github.magnusp.libsql.itest` package to share a managed, singleton container instance across test executions. ### Implementation ```java -package org.libsql.jdbc.base; +package com.github.magnusp.libsql.itest; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.GenericContainer; @@ -42,20 +49,29 @@ import java.time.Duration; @Testcontainers public abstract class SqldContainerBase { - // Supplied by the root pom's libsql.image property via surefire/failsafe. - private static final String LIBSQL_IMAGE = System.getProperty("libsql.image"); + static final String LIBSQL_IMAGE = requireLibsqlImage(); private static final int PORT = 8080; + private static String requireLibsqlImage() { + String image = System.getProperty("libsql.image"); + if (image == null || image.isBlank()) { + throw new IllegalStateException( + "libsql.image is not set. Run the integration tests with Maven so the" + + " pinned sqld image from the root pom is used."); + } + return image; + } + @Container public static final GenericContainer LIBSQL_CONTAINER = new GenericContainer<>(LIBSQL_IMAGE) .withExposedPorts(PORT) .withEnv("SQLD_NODE", "primary") .withEnv("SQLD_DISABLE_INTELLIGENT_THROTTLING", "true") .withEnv("SQLD_CONNECTION_CREATION_TIMEOUT_SEC", "30") - .waitingFor(Wait.forHttp("/v2/pipeline") - .forPort(PORT) - .forStatusCode(405) - .withStartupTimeout(Duration.ofSeconds(60))); + .waitingFor(Wait.forHttp("/v3/pipeline") + .forPort(PORT) + .forStatusCode(405) + .withStartupTimeout(Duration.ofSeconds(60))); @BeforeAll public static void startContainer() { @@ -67,13 +83,23 @@ public abstract class SqldContainerBase { public static String getHttpUrl() { return "http://" + LIBSQL_CONTAINER.getHost() + ":" + LIBSQL_CONTAINER.getMappedPort(PORT); } +} +``` - public static String getJdbcUrl() { - return "jdbc:libsql:" + getHttpUrl(); - } +`SqldContainerBase` exposes only `getHttpUrl()`. Each JDBC test class builds its own +driver URL from it, for example `LibsqlJdbcIT`, `GeneratedKeysIT`, +`TransactionLifecycleIT`, and `DatabaseMetaDataIT` each define: + +```java +private String getDriverJdbcUrl() { + return "jdbc:libsql:" + getHttpUrl(); } ``` +`HikariCpIntegrationIT` passes the same URL to HikariCP with +`config.setJdbcUrl("jdbc:libsql:" + getHttpUrl())`, and `LibsqlHttpClientIT` passes +`getHttpUrl()` directly to `new LibsqlClientConfig(getHttpUrl(), null)`. + --- ## 3. Key Configuration Parameters @@ -84,15 +110,17 @@ public abstract class SqldContainerBase { * **`SQLD_CONNECTION_CREATION_TIMEOUT_SEC=30`**: Extends the SQLite file connection creation timeout to avoid transient lock timeouts during heavy parallel integration test runs. ### B. Healthcheck & Wait Strategy -Because `/v2/pipeline` requires an HTTP `POST` request with a JSON body, standard `GET /` health checks can be misleading: -* A `GET` request to `/v2/pipeline` returns **HTTP 405 Method Not Allowed**, which confirms the HTTP server and routing engine are fully initialized and listening. -* `Wait.forHttp("/v2/pipeline").forPort(PORT).forStatusCode(405)` provides a fast and reliable readiness probe. +The driver talks Hrana 3 to `sqld` at `/v3/pipeline` (see `LibsqlHttpClient`, which +posts pipeline requests to `base + "/v3/pipeline"`). Because `/v3/pipeline` requires +an HTTP `POST` request with a JSON body, standard `GET /` health checks can be misleading: +* A `GET` request to `/v3/pipeline` returns **HTTP 405 Method Not Allowed**, which confirms the HTTP server and routing engine are fully initialized and listening. +* `Wait.forHttp("/v3/pipeline").forPort(PORT).forStatusCode(405)` with a 60-second startup timeout provides a fast and reliable readiness probe. --- ## 4. Advanced: Network Toxicity Simulation with Toxiproxy -To verify driver resiliency against real-world network anomalies (e.g. latency spikes, connection cuts, and slow timeouts), [`NetworkToxicityIntegrationIT`](file:///home/magnus/src/local/turso-java/src/test/java/org/libsql/jdbc/integration/NetworkToxicityIntegrationIT.java) pairs `sqld` with a **Shopify Toxiproxy** container within a dedicated Docker bridge network. +To verify driver resiliency against real-world network anomalies (e.g. latency spikes, connection cuts, and packet fragmentation), [`NetworkToxicityIT`](libsql-itests/src/test/java/com/github/magnusp/libsql/itest/NetworkToxicityIT.java) pairs `sqld` with a **Shopify Toxiproxy** container within a dedicated Docker bridge network. Unlike the tests above, `NetworkToxicityIT` does not extend `SqldContainerBase`; it declares its own `sqld` container on that network and reuses `SqldContainerBase.LIBSQL_IMAGE` for the image name. ### Architecture @@ -109,46 +137,75 @@ To verify driver resiliency against real-world network anomalies (e.g. latency s ### Setup Code ```java @Testcontainers -public class NetworkToxicityIntegrationIT { +public class NetworkToxicityIT { private static final Network NETWORK = Network.newNetwork(); - private static final GenericContainer LIBSQL = new GenericContainer<>(SqldContainerBase.LIBSQL_IMAGE) + private static final GenericContainer LIBSQL = + new GenericContainer<>(SqldContainerBase.LIBSQL_IMAGE) .withNetwork(NETWORK) .withNetworkAliases("libsql-srv") .withExposedPorts(8080) - .withEnv("SQLD_NODE", "primary"); + .withEnv("SQLD_NODE", "primary") + .withEnv("SQLD_DISABLE_INTELLIGENT_THROTTLING", "true") + .withEnv("SQLD_CONNECTION_CREATION_TIMEOUT_SEC", "30") + .waitingFor(Wait.forHttp("/v3/pipeline") + .forPort(8080) + .forStatusCode(405) + .withStartupTimeout(Duration.ofSeconds(60))); @Container - private static final ToxiproxyContainer TOXIPROXY = new ToxiproxyContainer("ghcr.io/shopify/toxiproxy:latest") + private static final ToxiproxyContainer TOXIPROXY = new ToxiproxyContainer( + DockerImageName.parse("ghcr.io/shopify/toxiproxy:latest")) .withNetwork(NETWORK); - private static ToxiproxyContainer.ContainerProxy proxy; + private static ToxiproxyClient toxiproxyClient; + private static Proxy proxy; + private static int mappedProxyPort; @BeforeAll - static void setUpAll() { + static void setUpAll() throws IOException { LIBSQL.start(); TOXIPROXY.start(); - proxy = TOXIPROXY.getProxy(LIBSQL, 8080); + toxiproxyClient = new ToxiproxyClient(TOXIPROXY.getHost(), TOXIPROXY.getControlPort()); + proxy = toxiproxyClient.createProxy("libsql-proxy", "0.0.0.0:8666", "libsql-srv:8080"); + mappedProxyPort = TOXIPROXY.getMappedPort(8666); } } ``` +Tests connect through the proxy with +`"jdbc:libsql:http://" + TOXIPROXY.getHost() + ":" + mappedProxyPort + "?connectTimeout=" + timeoutMs`. +After each test, `resetToxiproxy()` re-enables the proxy and removes all toxics. + +`NamespacePathIT` follows the same standalone pattern for a different scenario: it +starts its own container with ports `8080` and `9090` exposed, runs +`sqld --http-listen-addr 0.0.0.0:8080 --enable-namespaces --admin-listen-addr 0.0.0.0:9090`, +creates namespaces through the admin API, and then verifies isolation through JDBC +URLs such as `"jdbc:libsql:" + httpBaseUrl + "/dev/ns_alpha"`. + ### Example Toxics Tested -1. **Network Latency & Jitter**: +1. **Connection Cut**: ```java - proxy.toxics().latency("latency-toxic", ToxicDirection.DOWNSTREAM, 300).setJitter(50); + proxy.disable(); + // Verifies that the driver throws SQLException rather than hanging indefinitely ``` -2. **Abrupt Connection Drop (TCP RST / Cut)**: +2. **High Latency, Then Recovery**: ```java - proxy.setConnectionCut(true); - // Verifies that driver throws SQLException rather than hanging indefinitely + Latency latency = proxy.toxics().latency("latency_toxic", ToxicDirection.DOWNSTREAM, 1500); + // A query with a 500 ms connect timeout fails with SQLException; + // after latency.remove(), the same proxy serves SELECT 42 normally ``` -3. **Bandwidth Throttling / Slowloris**: +3. **Packet Fragmentation (Slicer)**: ```java - proxy.toxics().bandwidth("bandwidth-toxic", ToxicDirection.DOWNSTREAM, 10); // 10 KB/s + proxy.toxics().slicer("slice_toxic", ToxicDirection.DOWNSTREAM, 16, 20); ``` +The class also verifies clean traffic through the proxy +(`testNormalTrafficThroughProxy`) and that a failed rollback under a cut connection +throws `SQLException` and keeps the transaction available for retry once the proxy +is re-enabled (`testFailedRollbackThrowsAndKeepsTransactionForRetry`). + --- ## 5. Running the Tests @@ -157,11 +214,53 @@ To run the integration tests using Maven: ```bash # Run unit tests only -./mvnw test +mise exec -- ./mvnw test # Run full integration test suite against Testcontainers -./mvnw verify +mise exec -- ./mvnw clean verify # Override the sqld container image ./mvnw verify -Dlibsql.image=ghcr.io/tursodatabase/libsql-server:v0.24.33 ``` + +To run a single integration test, use the Failsafe selector: + +```bash +DOCKER_CONFIG="$PWD/libsql-conformance/src/test/docker-config" \ + ./mvnw -pl libsql-itests -am -Dit.test=HikariCpIntegrationIT verify +``` + +### Docker Config Under the `nono` Sandbox + +This environment may run inside a `nono` filesystem sandbox. The sandbox can block +reads of `~/.docker/config.json` even when `docker info` works. Testcontainers then +fails with: + +```text +ServiceConfigurationError: ... EnvironmentAndSystemPropertyClientProviderStrategy could not be instantiated +Caused by: java.io.FileNotFoundException: /home//.docker/config.json (Permission denied) +``` + +Check the exact path with: + +```bash +nono why --self --path ~/.docker/config.json --op read +``` + +Do not change file permissions or the `nono` profile from inside the sandbox. + +The repository ships one shared readable Docker config at +`libsql-conformance/src/test/docker-config/config.json`. It is an empty auth config, +so it works only for public images such as the `sqld` and Toxiproxy images that the +integration tests use. + +`libsql-itests` has no copy step. Point `DOCKER_CONFIG` at the shared config +directory in the shell, and the Failsafe fork inherits it: + +```bash +DOCKER_CONFIG="$PWD/libsql-conformance/src/test/docker-config" \ + ./mvnw -pl libsql-itests -am verify +``` + +Use the absolute path shown (here expanded through `$PWD`). A relative +`DOCKER_CONFIG` is resolved against the fork's working directory, not the shell's. From d444d92cad6a8650f3764aac0c69a0e9319e6c0b Mon Sep 17 00:00:00 2001 From: Magnus Persson <1431685+magnusp@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:08:29 +0200 Subject: [PATCH 2/2] docs: keep the readiness probe endpoint faithful to the real classes --- TESTCONTAINERS-USAGE.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/TESTCONTAINERS-USAGE.md b/TESTCONTAINERS-USAGE.md index 2cafd7b..46fbace 100644 --- a/TESTCONTAINERS-USAGE.md +++ b/TESTCONTAINERS-USAGE.md @@ -68,7 +68,7 @@ public abstract class SqldContainerBase { .withEnv("SQLD_NODE", "primary") .withEnv("SQLD_DISABLE_INTELLIGENT_THROTTLING", "true") .withEnv("SQLD_CONNECTION_CREATION_TIMEOUT_SEC", "30") - .waitingFor(Wait.forHttp("/v3/pipeline") + .waitingFor(Wait.forHttp("/v2/pipeline") .forPort(PORT) .forStatusCode(405) .withStartupTimeout(Duration.ofSeconds(60))); @@ -110,11 +110,14 @@ private String getDriverJdbcUrl() { * **`SQLD_CONNECTION_CREATION_TIMEOUT_SEC=30`**: Extends the SQLite file connection creation timeout to avoid transient lock timeouts during heavy parallel integration test runs. ### B. Healthcheck & Wait Strategy -The driver talks Hrana 3 to `sqld` at `/v3/pipeline` (see `LibsqlHttpClient`, which -posts pipeline requests to `base + "/v3/pipeline"`). Because `/v3/pipeline` requires -an HTTP `POST` request with a JSON body, standard `GET /` health checks can be misleading: -* A `GET` request to `/v3/pipeline` returns **HTTP 405 Method Not Allowed**, which confirms the HTTP server and routing engine are fully initialized and listening. -* `Wait.forHttp("/v3/pipeline").forPort(PORT).forStatusCode(405)` with a 60-second startup timeout provides a fast and reliable readiness probe. +The test containers probe `/v2/pipeline` for readiness, matching the code in +`SqldContainerBase`, `NetworkToxicityIT`, `NamespacePathIT`, and +`XerialTestServer`. The driver itself talks Hrana 3 to `sqld` at `/v3/pipeline` +(see `LibsqlHttpClient`, which posts pipeline requests to +`base + "/v3/pipeline"`). Both endpoints require an HTTP `POST` request with a +JSON body, so standard `GET /` health checks can be misleading: +* A `GET` request to `/v2/pipeline` returns **HTTP 405 Method Not Allowed**, which confirms the HTTP server and routing engine are fully initialized and listening. +* `Wait.forHttp("/v2/pipeline").forPort(PORT).forStatusCode(405)` with a 60-second startup timeout provides a fast and reliable readiness probe. --- @@ -149,7 +152,7 @@ public class NetworkToxicityIT { .withEnv("SQLD_NODE", "primary") .withEnv("SQLD_DISABLE_INTELLIGENT_THROTTLING", "true") .withEnv("SQLD_CONNECTION_CREATION_TIMEOUT_SEC", "30") - .waitingFor(Wait.forHttp("/v3/pipeline") + .waitingFor(Wait.forHttp("/v2/pipeline") .forPort(8080) .forStatusCode(405) .withStartupTimeout(Duration.ofSeconds(60)));