From 15e8bb93e4f96e0c2e4264f2875ea428996cb864 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Thu, 24 Sep 2026 17:06:17 +0300 Subject: [PATCH 1/4] [#1085] Stop the server cleanly on SIGTERM in a container that bootstrapped its instance On the first start run.sh kept bash as PID 1 behind `tail -f /dev/null`, so SIGTERM was never delivered and the container was killed at the end of the stop timeout without stopping the server. The server that setup started is now stopped once the bootstrap is through, before the health marker is written, and started again with exec, so it is PID 1 as on every restart. The Docker test stopped and started the server inside the container, which only worked because of this; it now imports online, checks that `docker stop` stops the server cleanly, and checks the restarted container. Fixes #1085 --- .github/workflows/build.yml | 50 ++++++++++++++++++++-------- opendj-packages/opendj-docker/run.sh | 18 +++++----- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8469aec276..a71f29aeeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -523,17 +523,28 @@ jobs: shell: bash run: | trap 'code=$?; echo "::group::container logs (test)"; docker logs test 2>&1 || true; echo "::endgroup::"; exit $code' ERR - docker run --rm -it -d --memory="512m" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} + docker run -it -d --memory="512m" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done' docker exec test 'sh' '-c' '/opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=example2 --type je --set=base-dn:dc=example2,dc=com --set=enabled:true --no-prompt --trustAll' docker exec test 'sh' '-c' '/opt/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example2,dc=com /opt/opendj/data/config/MakeLDIF/example.template' - docker exec test 'sh' '-c' '/opt/opendj/bin/stop-ds' - docker exec test 'sh' '-c' '/opt/opendj/bin/import-ldif --offline --ldifFile /tmp/test.ldif --backendID=example2' - docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --offline --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll' - docker exec test 'sh' '-c' '/opt/opendj/bin/start-ds' - docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll --trustAll' + docker exec test 'sh' '-c' '/opt/opendj/bin/import-ldif --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --ldifFile /tmp/test.ldif --backendID=example2 --trustAll' + # the server is PID 1 of the container, so stopping it stops the container; the + # container that bootstrapped the instance has to stop the server on SIGTERM, not + # sit out the timeout and be killed. The server run in the foreground reports its + # shutdown only in the error log, which is read once the container is back up + stopped=$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors || true) + start=$SECONDS + docker stop -t 60 test + echo "stopped in $((SECONDS - start)) s, exit code $(docker inspect --format='{{.State.ExitCode}}' test)" + test $((SECONDS - start)) -lt 50 + test "$(docker inspect --format='{{.State.ExitCode}}' test)" -ne 137 + # a restart runs the server of the instance already there + docker start test + timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done' + test "$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors)" -gt "$stopped" + docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll --trustAll' docker exec test 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example2,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 10000' - docker kill test + docker rm -f test - name: Docker test custom password shell: bash run: | @@ -736,17 +747,28 @@ jobs: shell: bash run: | trap 'code=$?; echo "::group::container logs (test)"; docker logs test 2>&1 || true; echo "::endgroup::"; exit $code' ERR - docker run --rm -it -d --memory="1g" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine + docker run -it -d --memory="1g" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done' docker exec test 'sh' '-c' '/opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=example2 --type je --set=base-dn:dc=example2,dc=com --set=enabled:true --no-prompt --trustAll' docker exec test 'sh' '-c' '/opt/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example2,dc=com /opt/opendj/data/config/MakeLDIF/example.template' - docker exec test 'sh' '-c' '/opt/opendj/bin/stop-ds' - docker exec test 'sh' '-c' '/opt/opendj/bin/import-ldif --offline --ldifFile /tmp/test.ldif --backendID=example2' - docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --offline --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll' - docker exec test 'sh' '-c' '/opt/opendj/bin/start-ds' - docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll --trustAll' + docker exec test 'sh' '-c' '/opt/opendj/bin/import-ldif --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --ldifFile /tmp/test.ldif --backendID=example2 --trustAll' + # the server is PID 1 of the container, so stopping it stops the container; the + # container that bootstrapped the instance has to stop the server on SIGTERM, not + # sit out the timeout and be killed. The server run in the foreground reports its + # shutdown only in the error log, which is read once the container is back up + stopped=$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors || true) + start=$SECONDS + docker stop -t 60 test + echo "stopped in $((SECONDS - start)) s, exit code $(docker inspect --format='{{.State.ExitCode}}' test)" + test $((SECONDS - start)) -lt 50 + test "$(docker inspect --format='{{.State.ExitCode}}' test)" -ne 137 + # a restart runs the server of the instance already there + docker start test + timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done' + test "$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors)" -gt "$stopped" + docker exec test 'sh' '-c' '/opt/opendj/bin/rebuild-index --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --baseDN "dc=example2,dc=com" --rebuildAll --trustAll' docker exec test 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example2,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 10000' - docker kill test + docker rm -f test - name: Docker test custom password shell: bash run: | diff --git a/opendj-packages/opendj-docker/run.sh b/opendj-packages/opendj-docker/run.sh index d1ca2bdc81..0ee06d1a6d 100755 --- a/opendj-packages/opendj-docker/run.sh +++ b/opendj-packages/opendj-docker/run.sh @@ -81,6 +81,16 @@ if [ -n "${MASTER_SERVER}" ] && [ -n "${OPENDJ_REPLICATION_TYPE}" ]; then fi fi +# Setup has usually left the server running in the background. It is stopped here and +# started again below with exec, so that the server is PID 1 on the first start just as +# on a restart: a shell as PID 1 without a SIGTERM handler never receives the signal, and +# the container would then be killed at the end of the stop timeout instead of stopping +# the server. It is stopped before the marker below is written, so that the health check +# never reports the server of the bootstrap healthy just before it goes down. stop-ds +# exits 0 when the server is not running. +echo "Stopping the server started by the bootstrap" +./bin/stop-ds + # Check if keystores are mounted as a volume, and if so # Copy any keystores over SECRET_VOLUME=${SECRET_VOLUME:-/var/secrets/opendj} @@ -97,13 +107,5 @@ if [ "$BOOTSTRAPPED" = true ]; then touch "$BOOTSTRAP_COMPLETE" fi -# Opendj is probably already started in detach mode at the install -if (bin/status -n | grep Started); then - echo "OpenDJ is started" - - # Use tail instead of sleep to allow the container to be stopped with SIGTERM - tail -f /dev/null -fi - echo "Starting OpenDJ" exec ./bin/start-ds --nodetach From eaad637fe7c6f1119700ae27296023f242d57559 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Fri, 25 Sep 2026 11:26:53 +0300 Subject: [PATCH 2/4] [#1085] Wait for the container to be healthy before seeding the benchmark, and check that the bootstrap's server is down before the marker On a first start the server the bootstrap started is stopped and started again, so waiting for it to answer is no longer enough: the benchmark seeded ou=People into that gap and ran against a directory without its users. Both benchmark waits now require a healthy container where the image has a health check, still search the base entry, which older images need, and last as long as the health check's start period. The Docker test steps check that the bootstrap's server has logged its stop by the time the marker is written. run.sh says so when stop-ds fails, and still tries the start. --- .github/benchmark/compare-opendj.sh | 17 +++++++++++++---- .github/workflows/benchmark.yml | 11 +++++++++-- .github/workflows/build.yml | 8 ++++++++ opendj-packages/opendj-docker/README.md | 4 ++++ opendj-packages/opendj-docker/run.sh | 8 ++++++-- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.github/benchmark/compare-opendj.sh b/.github/benchmark/compare-opendj.sh index 65f03c41e7..eed3cac090 100644 --- a/.github/benchmark/compare-opendj.sh +++ b/.github/benchmark/compare-opendj.sh @@ -50,10 +50,19 @@ if [ ! -x "$JM" ]; then tar -xzf /tmp/jmeter.tgz -C "$HOME/jmeter" fi -wait_dj() { # poll OpenDJ readiness on localhost:1389 - for _ in $(seq 1 90); do - ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ - -b "$BASEDN" -s base dn >/dev/null 2>&1 && return 0 +# Poll OpenDJ readiness on localhost:1389. An image with a HEALTHCHECK has to report healthy +# first: on a first start the server the bootstrap started answers, then is stopped and +# started again, and whatever is added in between is lost. An older image's health check +# may pass before the bootstrap is done, so the base entry is searched for in both cases. +# The wait lasts as long as the start period of the health check, 5 minutes. +wait_dj() { + local health + for _ in $(seq 1 150); do + health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' opendj-bench 2>/dev/null || true)" + if [ -z "$health" ] || [ "$health" = healthy ]; then + ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ + -b "$BASEDN" -s base dn >/dev/null 2>&1 && return 0 + fi sleep 2 done return 1 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e1e1c1b382..f2971e32a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -177,8 +177,15 @@ jobs: - name: Wait for OpenDJ + seed ou=People run: | - for i in $(seq 1 90); do - if ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ + # An image with a HEALTHCHECK has to report healthy first: on a first start the + # server the bootstrap started answers, then is stopped and started again, and + # whatever is added in between is lost. An older image's health check may pass + # before the bootstrap is done, so the base entry is searched for in both cases. + # The wait lasts as long as the start period of the health check, 5 minutes. + for i in $(seq 1 150); do + health="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' opendj 2>/dev/null || true)" + if { [ -z "$health" ] || [ "$health" = healthy ]; } && \ + ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -b "$BASEDN" -s base dn >/dev/null 2>&1; then echo "OpenDJ is up"; break fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a71f29aeeb..0d54a2a7f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -524,6 +524,10 @@ jobs: run: | trap 'code=$?; echo "::group::container logs (test)"; docker logs test 2>&1 || true; echo "::endgroup::"; exit $code' ERR docker run -it -d --memory="512m" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} + # the bootstrap's server is down before the marker the health check waits for is + # written: the stop has logged its record by the time the marker is there + timeout 3m bash -c 'until docker exec test sh -c "test -f \"\$BOOTSTRAP_COMPLETE\"" 2>/dev/null; do sleep 0.2; done' + test "$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors)" -ge 1 timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done' docker exec test 'sh' '-c' '/opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=example2 --type je --set=base-dn:dc=example2,dc=com --set=enabled:true --no-prompt --trustAll' docker exec test 'sh' '-c' '/opt/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example2,dc=com /opt/opendj/data/config/MakeLDIF/example.template' @@ -748,6 +752,10 @@ jobs: run: | trap 'code=$?; echo "::group::container logs (test)"; docker logs test 2>&1 || true; echo "::endgroup::"; exit $code' ERR docker run -it -d --memory="1g" --name=test localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine + # the bootstrap's server is down before the marker the health check waits for is + # written: the stop has logged its record by the time the marker is there + timeout 3m bash -c 'until docker exec test sh -c "test -f \"\$BOOTSTRAP_COMPLETE\"" 2>/dev/null; do sleep 0.2; done' + test "$(docker exec test grep -c "The Directory Server is now stopped" /opt/opendj/data/logs/errors)" -ge 1 timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test | grep -q \"healthy\"; do sleep 10; done' docker exec test 'sh' '-c' '/opt/opendj/bin/dsconfig create-backend --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=example2 --type je --set=base-dn:dc=example2,dc=com --set=enabled:true --no-prompt --trustAll' docker exec test 'sh' '-c' '/opt/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example2,dc=com /opt/opendj/data/config/MakeLDIF/example.template' diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md index 0b85aa0471..d444926186 100644 --- a/opendj-packages/opendj-docker/README.md +++ b/opendj-packages/opendj-docker/README.md @@ -30,6 +30,10 @@ without `ADD_BASE_ENTRY` nothing creates the base entry, so `BASE_DN` is an empt a healthy container - the health check itself searches the root DSE, which every instance serves whatever it was set up to hold. +The server answering is not enough on a first start: the bootstrap starts the server, and +once it is done that server is stopped and started again in the foreground, so a client +that only waits for the port can have its first requests fail in between. + A bootstrap that imports `SAMPLE_DATA` can take minutes on a small container, which is what the start period allows for. A bootstrap that fails - or an upgrade that fails when starting over an instance that is already there - never reports healthy: what failed is in `docker diff --git a/opendj-packages/opendj-docker/run.sh b/opendj-packages/opendj-docker/run.sh index 0ee06d1a6d..aa8733e0ba 100755 --- a/opendj-packages/opendj-docker/run.sh +++ b/opendj-packages/opendj-docker/run.sh @@ -87,9 +87,13 @@ fi # the container would then be killed at the end of the stop timeout instead of stopping # the server. It is stopped before the marker below is written, so that the health check # never reports the server of the bootstrap healthy just before it goes down. stop-ds -# exits 0 when the server is not running. +# exits 0 when the server is not running. When it fails the server may still be stopping, +# so the start below is tried anyway: it either runs the server or fails on the lock of +# the one still there. echo "Stopping the server started by the bootstrap" -./bin/stop-ds +if ! ./bin/stop-ds; then + echo "Could not stop the server started by the bootstrap, starting OpenDJ may fail" +fi # Check if keystores are mounted as a volume, and if so # Copy any keystores over From df6b6327ea571855febe48fb7fd59a2cec317c64 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Fri, 25 Sep 2026 15:00:08 +0300 Subject: [PATCH 3/4] [#1085] Let a replica try again a master restarting at the end of its first start, and fail the benchmark when its seed missed A replica started together with its master reached it while the master stopped the server its bootstrap started and started it again, and its one dsreplication enable failed with ERROR_CONNECTING. replicate.sh now runs enable again when it exits 8, the only code it returns before changing anything, and initialize, initialize-all and the dsconfig on the master again after any failure, every 10 s for up to 5 minutes. A failed enable ends the script instead of running initialize. Both Docker test steps take the master off the network while the replica sleeps before its first try, and bring it back once the replica says it tries again. The benchmark checks that ou=People was seeded, and stops when it was not. --- .github/benchmark/compare-opendj.sh | 11 ++++- .github/workflows/benchmark.yml | 8 +++- .github/workflows/build.yml | 16 +++++++ opendj-packages/opendj-docker/README.md | 4 +- .../opendj-docker/bootstrap/replicate.sh | 43 ++++++++++++++----- 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/.github/benchmark/compare-opendj.sh b/.github/benchmark/compare-opendj.sh index eed3cac090..e8321f1d5e 100644 --- a/.github/benchmark/compare-opendj.sh +++ b/.github/benchmark/compare-opendj.sh @@ -52,7 +52,7 @@ fi # Poll OpenDJ readiness on localhost:1389. An image with a HEALTHCHECK has to report healthy # first: on a first start the server the bootstrap started answers, then is stopped and -# started again, and whatever is added in between is lost. An older image's health check +# started again, and a request sent in between fails. An older image's health check # may pass before the bootstrap is done, so the base entry is searched for in both cases. # The wait lasts as long as the start period of the health check, 5 minutes. wait_dj() { @@ -81,6 +81,15 @@ bench_one() { wait_dj || echo "WARN: $image not ready in time" >&2 ldapadd -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -f "$HERE/people.ldif" >/dev/null 2>&1 || true + # without its users JMeter binds as users that do not exist and the numbers mean nothing, + # so a seed that missed stops the comparison (A_VER="$(bench_one ...)" under set -e) + if ! ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ + -b "ou=People,$BASEDN" -s base dn >/dev/null 2>&1; then + echo "ERROR: $image: ou=People was not seeded" >&2 + docker logs opendj-bench > "$out.docker.log" 2>&1 || true + docker rm -f opendj-bench >/dev/null 2>&1 || true + return 1 + fi ver="$( { ldapsearch -x -LLL -H ldap://localhost:1389 -D 'cn=Directory Manager' -w password \ -b '' -s base fullVendorVersion 2>/dev/null || true; } | sed -n 's/^fullVendorVersion: //p')" rm -rf "$out" "$out.jtl" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f2971e32a4..58efae20e1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -178,8 +178,8 @@ jobs: - name: Wait for OpenDJ + seed ou=People run: | # An image with a HEALTHCHECK has to report healthy first: on a first start the - # server the bootstrap started answers, then is stopped and started again, and - # whatever is added in between is lost. An older image's health check may pass + # server the bootstrap started answers, then is stopped and started again, and a + # request sent in between fails. An older image's health check may pass # before the bootstrap is done, so the base entry is searched for in both cases. # The wait lasts as long as the start period of the health check, 5 minutes. for i in $(seq 1 150); do @@ -193,6 +193,10 @@ jobs: done ldapadd -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ -f .github/benchmark/people.ldif || true + # without its users JMeter binds as users that do not exist, so a seed that missed fails the job + ldapsearch -x -H ldap://localhost:1389 -D "cn=Directory Manager" -w password \ + -b "ou=People,$BASEDN" -s base dn >/dev/null \ + || { echo "::error::ou=People was not seeded"; docker logs opendj; false; } - name: Configure OpenDJ password policy (SSHA hash-on-write) run: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0d54a2a7f7..16caf0ec09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -597,6 +597,14 @@ jobs: # a password file waits as a killed replicate.sh would have left it, and its run.sh has to remove it (checked below) docker exec test_master sh -c 'printf "%s\n" "$ROOT_PASSWORD" >/dev/shm/opendj-replicate.killed' docker run --rm -it -d --memory="512m" --network test_replication --ipc=container:test_master --name=test_replica --hostname=dj-replica -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=simple "$IMAGE" + # on a first start the master stops the server its bootstrap started and starts it again, and a replica + # started together with it can reach it in between: replicate.sh tries a dsreplication that could not + # connect again. The master is taken off the network while the replica sleeps before its first try, and + # comes back only once the replica has said it will try again + timeout 5m bash -c 'until docker logs test_replica 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' + docker network disconnect test_replication test_master + timeout 2m bash -c 'until docker logs test_replica 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' + docker network connect --alias dj-master test_replication test_master timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica | grep -q \"healthy\"; do sleep 10; done' docker run --rm -it -d --memory="512m" --network test_replication --name=test_replica_sdsr --hostname=dj-replica-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica_sdsr | grep -q \"healthy\"; do sleep 10; done' @@ -825,6 +833,14 @@ jobs: # a password file waits as a killed replicate.sh would have left it, and its run.sh has to remove it (checked below) docker exec test_master sh -c 'printf "%s\n" "$ROOT_PASSWORD" >/dev/shm/opendj-replicate.killed' docker run --rm -it -d --memory="1g" --network test_replication --ipc=container:test_master --name=test_replica --hostname=dj-replica -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=simple "$IMAGE" + # on a first start the master stops the server its bootstrap started and starts it again, and a replica + # started together with it can reach it in between: replicate.sh tries a dsreplication that could not + # connect again. The master is taken off the network while the replica sleeps before its first try, and + # comes back only once the replica has said it will try again + timeout 5m bash -c 'until docker logs test_replica 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' + docker network disconnect test_replication test_master + timeout 2m bash -c 'until docker logs test_replica 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' + docker network connect --alias dj-master test_replication test_master timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica | grep -q \"healthy\"; do sleep 10; done' docker run --rm -it -d --memory="1g" --network test_replication --name=test_replica_sdsr --hostname=dj-replica-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica_sdsr | grep -q \"healthy\"; do sleep 10; done' diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md index d444926186..76ba8d8719 100644 --- a/opendj-packages/opendj-docker/README.md +++ b/opendj-packages/opendj-docker/README.md @@ -32,7 +32,9 @@ serves whatever it was set up to hold. The server answering is not enough on a first start: the bootstrap starts the server, and once it is done that server is stopped and started again in the foreground, so a client -that only waits for the port can have its first requests fail in between. +that only waits for the port can have its first requests fail in between. A replica set up +with `MASTER_SERVER` tries a master it cannot connect to again, every 10 s for up to 5 +minutes, so that window does not fail its replication setup. A bootstrap that imports `SAMPLE_DATA` can take minutes on a small container, which is what the start period allows for. A bootstrap that fails - or an upgrade that fails when starting diff --git a/opendj-packages/opendj-docker/bootstrap/replicate.sh b/opendj-packages/opendj-docker/bootstrap/replicate.sh index e996e6f9cd..f87f40d74b 100755 --- a/opendj-packages/opendj-docker/bootstrap/replicate.sh +++ b/opendj-packages/opendj-docker/bootstrap/replicate.sh @@ -46,6 +46,29 @@ PASSWORD_FILE=$(mktemp -p /dev/shm opendj-replicate.XXXXXX 2>/dev/null || mktemp trap 'rm -f "$PASSWORD_FILE"' EXIT printf '%s\n' "$ROOT_PASSWORD" >"$PASSWORD_FILE" || exit 1 +# The master stops the server its bootstrap started and starts it again in the foreground once +# that bootstrap is done (run.sh), so a replica started together with it can reach it while it +# is down. A tool that fails that way is run again, every 10 s for up to 5 minutes. +# dsreplication enable exits 8 (ERROR_CONNECTING) when it cannot connect to one of the servers, +# which it does before it changes anything, so only that exit code is tried again: an enable +# that got further may have replicated the base DN already, and a second one then fails on that. +# initialize and dsconfig set-* can be run again whatever made them fail. +# retry [...] +retry() { + local on=$1 rc + shift + for _ in $(seq 1 30); do + "$@" && return 0 + rc=$? + if [ "$on" != any ] && [ "$rc" -ne "$on" ]; then + return $rc + fi + echo "$(basename "$1") $2 exited with $rc, trying again in 10 s" + sleep 10 + done + return $rc +} + # todo: Replace with command to test for master being reachable and up # This is hacky.... echo "Will sleep for a bit to ensure master is up" @@ -54,7 +77,7 @@ sleep 5 if [ "$OPENDJ_REPLICATION_TYPE" == "simple" ]; then echo "Enabling Standard Replication..." - /opt/opendj/bin/dsreplication \ + retry 8 /opt/opendj/bin/dsreplication \ enable \ --host1 $MASTER_SERVER \ --port1 4444 \ @@ -63,19 +86,19 @@ if [ "$OPENDJ_REPLICATION_TYPE" == "simple" ]; then --host2 $MYHOSTNAME --port2 4444 --bindDN2 "$ROOT_USER_DN" \ --bindPasswordFile2 "$PASSWORD_FILE" --replicationPort2 8989 \ --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ - --baseDN "$BASE_DN" -X -n + --baseDN "$BASE_DN" -X -n || exit echo "initializing replication" # replicating data in MASTER_SERVER to MYHOSTNAME: - /opt/opendj/bin/dsreplication initialize --baseDN "$BASE_DN" \ + retry any /opt/opendj/bin/dsreplication initialize --baseDN "$BASE_DN" \ --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ --hostSource $MASTER_SERVER --portSource 4444 \ --hostDestination $MYHOSTNAME --portDestination 4444 -X -n elif [ "$OPENDJ_REPLICATION_TYPE" == "srs" ]; then echo "Enabling Standalone Replication Servers..." - dsreplication enable \ + retry 8 dsreplication enable \ --adminUID admin \ --adminPasswordFile "$PASSWORD_FILE" \ --baseDN "$BASE_DN" \ @@ -91,11 +114,11 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "srs" ]; then --replicationPort2 8989 \ --onlyReplicationServer2 \ --trustAll \ - --no-prompt + --no-prompt || exit echo "initializing replication" - dsreplication \ + retry any dsreplication \ initialize-all \ --adminUID admin \ --adminPasswordFile "$PASSWORD_FILE" \ @@ -107,7 +130,7 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "srs" ]; then elif [ "$OPENDJ_REPLICATION_TYPE" == "sdsr" ]; then echo "Enabling Standalone Directory Server Replicas...." - dsreplication \ + retry 8 dsreplication \ enable \ --adminUID admin \ --adminPasswordFile "$PASSWORD_FILE" \ @@ -122,11 +145,11 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "sdsr" ]; then --bindPasswordFile2 "$PASSWORD_FILE" \ --noReplicationServer2 \ --trustAll \ - --no-prompt + --no-prompt || exit echo "initializing replication" - dsreplication \ + retry any dsreplication \ initialize \ --adminUID admin \ --adminPasswordFile "$PASSWORD_FILE" \ @@ -153,7 +176,7 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "rg" ]; then --trustAll \ --no-prompt - dsconfig \ + retry any dsconfig \ set-replication-server-prop \ --port 4444 \ --hostname $MASTER_SERVER \ From 76bdaf0cfed1e889bdeef6fbb3d99c0ae1a75285 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Fri, 25 Sep 2026 17:28:39 +0300 Subject: [PATCH 4/4] [#1085] Pin which dsreplication enable failures a replica tries again, and document what the retry does not cover replicate.sh - The comment on retry no longer says that exit 8 means nothing was written: dsreplication enable also exits 8 when another server of the topology cannot be reached after it has written to the first two. It also says that a wrong root DN or password exits 8, so it fails only once the 5 minutes are over. - retry returns after its 30th failure instead of saying it will try again and sleeping another 10 s. README.md - A master that stops while dsreplication enable is writing to it still fails the replica's setup; the retry covers only a master that cannot be reached. - With OPENDJ_REPLICATION_TYPE=srs the directory server replicas are to be started one at a time: each pushes its data to the replicas connected at that moment, and one restarting at the end of its own first start misses it and stays in a bad generation ID. build.yml (Docker test replication, both docker jobs) - The sdsr replica's enable is pinned the same way as the simple one's: the master is taken off the network until the replica logs that it tries again. - replicate.sh is run once more on the healthy simple replica. Its enable of a base DN already replicated has to end it with exit 5, with no "trying again" and no "initializing replication". --- .github/workflows/build.yml | 22 +++++++++++++++++++ opendj-packages/opendj-docker/README.md | 7 +++++- .../opendj-docker/bootstrap/replicate.sh | 15 +++++++------ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16caf0ec09..701e8bd62e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -607,6 +607,11 @@ jobs: docker network connect --alias dj-master test_replication test_master timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica | grep -q \"healthy\"; do sleep 10; done' docker run --rm -it -d --memory="512m" --network test_replication --name=test_replica_sdsr --hostname=dj-replica-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" + # the same for the sdsr replica, whose dsreplication enable is another command of replicate.sh + timeout 5m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' + docker network disconnect test_replication test_master + timeout 2m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' + docker network connect --alias dj-master test_replication test_master timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica_sdsr | grep -q \"healthy\"; do sleep 10; done' # the replicas were initialized from the master, and a change made on the master reaches them for c in $REPLICAS; do @@ -616,6 +621,12 @@ jobs: for c in $REPLICAS; do timeout 1m bash -c 'until docker exec $1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" $c done + # replicate.sh tries dsreplication enable again only when it exits 8, and a failed enable ends it: run once more + # on the replica, the enable of a base DN already replicated exits 5 and nothing is tried again or initialized + rc=0; out=$(docker exec -e BASE_DN=dc=example,dc=com -e ROOT_USER_DN="cn=Directory Manager" test_replica timeout 90 /opt/opendj/bootstrap/replicate.sh 2>&1) || rc=$? + if [ $rc -ne 5 ] || grep -qE "trying again|initializing replication" <<<"$out"; then + echo "$out"; echo "::error::a second replicate.sh exited with $rc, not with the 5 of its dsreplication enable, or went on after it"; false + fi # the root password shows in no container log, and the file replicate.sh passed it in is gone (#1084) for c in test_master $REPLICAS; do if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi @@ -843,6 +854,11 @@ jobs: docker network connect --alias dj-master test_replication test_master timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica | grep -q \"healthy\"; do sleep 10; done' docker run --rm -it -d --memory="1g" --network test_replication --name=test_replica_sdsr --hostname=dj-replica-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" + # the same for the sdsr replica, whose dsreplication enable is another command of replicate.sh + timeout 5m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' + docker network disconnect test_replication test_master + timeout 2m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' + docker network connect --alias dj-master test_replication test_master timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica_sdsr | grep -q \"healthy\"; do sleep 10; done' # the replicas were initialized from the master, and a change made on the master reaches them for c in $REPLICAS; do @@ -852,6 +868,12 @@ jobs: for c in $REPLICAS; do timeout 1m bash -c 'until docker exec $1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" $c done + # replicate.sh tries dsreplication enable again only when it exits 8, and a failed enable ends it: run once more + # on the replica, the enable of a base DN already replicated exits 5 and nothing is tried again or initialized + rc=0; out=$(docker exec -e BASE_DN=dc=example,dc=com -e ROOT_USER_DN="cn=Directory Manager" test_replica timeout 90 /opt/opendj/bootstrap/replicate.sh 2>&1) || rc=$? + if [ $rc -ne 5 ] || grep -qE "trying again|initializing replication" <<<"$out"; then + echo "$out"; echo "::error::a second replicate.sh exited with $rc, not with the 5 of its dsreplication enable, or went on after it"; false + fi # the root password shows in no container log, and the file replicate.sh passed it in is gone (#1084) for c in test_master $REPLICAS; do if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md index 76ba8d8719..9e7d4f2fac 100644 --- a/opendj-packages/opendj-docker/README.md +++ b/opendj-packages/opendj-docker/README.md @@ -34,7 +34,12 @@ The server answering is not enough on a first start: the bootstrap starts the se once it is done that server is stopped and started again in the foreground, so a client that only waits for the port can have its first requests fail in between. A replica set up with `MASTER_SERVER` tries a master it cannot connect to again, every 10 s for up to 5 -minutes, so that window does not fail its replication setup. +minutes, so a master that is down when the replica reaches it does not fail its replication +setup; one that stops while `dsreplication enable` is writing to it still does. + +With `OPENDJ_REPLICATION_TYPE=srs`, start the directory server replicas one at a time, each +once the previous one is healthy: every replica pushes its data to the replicas connected at +that moment, and one that is restarting at the end of its own first start misses it. A bootstrap that imports `SAMPLE_DATA` can take minutes on a small container, which is what the start period allows for. A bootstrap that fails - or an upgrade that fails when starting diff --git a/opendj-packages/opendj-docker/bootstrap/replicate.sh b/opendj-packages/opendj-docker/bootstrap/replicate.sh index f87f40d74b..5d07ba8a57 100755 --- a/opendj-packages/opendj-docker/bootstrap/replicate.sh +++ b/opendj-packages/opendj-docker/bootstrap/replicate.sh @@ -49,24 +49,25 @@ printf '%s\n' "$ROOT_PASSWORD" >"$PASSWORD_FILE" || exit 1 # The master stops the server its bootstrap started and starts it again in the foreground once # that bootstrap is done (run.sh), so a replica started together with it can reach it while it # is down. A tool that fails that way is run again, every 10 s for up to 5 minutes. -# dsreplication enable exits 8 (ERROR_CONNECTING) when it cannot connect to one of the servers, -# which it does before it changes anything, so only that exit code is tried again: an enable -# that got further may have replicated the base DN already, and a second one then fails on that. +# dsreplication enable exits 8 (ERROR_CONNECTING) when it cannot connect or bind to one of the two +# servers, which it checks before it changes anything, and also when another server of the topology +# cannot be reached after it has written to the first two. Only that exit code is tried again: an +# enable that failed otherwise may have replicated the base DN already, and a second one then fails. +# A wrong root DN or password also exits 8, so it fails only once the 5 minutes are over. # initialize and dsconfig set-* can be run again whatever made them fail. # retry [...] retry() { - local on=$1 rc + local on=$1 rc i shift - for _ in $(seq 1 30); do + for i in $(seq 1 30); do "$@" && return 0 rc=$? - if [ "$on" != any ] && [ "$rc" -ne "$on" ]; then + if [ "$on" != any ] && [ "$rc" -ne "$on" ] || [ "$i" -eq 30 ]; then return $rc fi echo "$(basename "$1") $2 exited with $rc, trying again in 10 s" sleep 10 done - return $rc } # todo: Replace with command to test for master being reachable and up