CASSANDRA-21628 Add CI support for repeated tests across machines

.build/README.md

@@ -134,6 +134,11 @@ Fail fast with repeating tests is done with REPEATED_TESTS_STOP_ON_FAILURE
 
     .build/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2 -e REPEATED_TESTS_STOP_ON_FAILURE=false
 
+Each split chunk runs the full set of iterations on its own machine, so -c multiplies the number of
+machines doing the same REPEATED_TESTS_COUNT runs (the chunk does not partition the iterations):
+
+    .build/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2 -c 1/2
+
 Running python dtests without docker:
 
     .build/run-python-dtests.sh dtest

.build/docker/run-tests.sh

@@ -287,7 +287,12 @@ logfile="${build_dir}/test/logs/docker_attach_${container_name}.log"
 # Docker commands:
 #  set java to java_version
 #  execute the run_script
-[ -n "${test_name_regexp}" ] && test_name_regexp_arg="-t ${test_name_regexp}" || split_chunk_arg="-c ${split_chunk}"
+[ -n "${test_name_regexp}" ] && test_name_regexp_arg="-t ${test_name_regexp}"
+# Named tests normally disable splitting. For *-repeat targets the chunk instead identifies the
+# machine, with each machine running every iteration.
+if [ -z "${test_name_regexp}" ] || [[ "${test_target}" == *"-repeat" ]]; then
+    split_chunk_arg="-c ${split_chunk}"
+fi
 
 docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_version} ; \
             \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh -a ${target} ${split_chunk_arg} ${test_name_regexp_arg} ${env_vars} ; exit \$?"

.build/run-ci

@@ -198,6 +198,10 @@ def argument_parser() -> argparse.ArgumentParser:
     parser.add_argument("-p", "--profile", choices=['packaging','skinny','pre-commit','pre-commit w/ upgrades','post-commit','custom'], default=DEFAULT_PROFILE, help="CI pipeline profile. Defaults to skinny.")
     parser.add_argument("-e", "--profile-custom-regexp", help="Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: 'stress.*|jvm-dtest.'")
     parser.add_argument("-j", "--jdk", help="Specify JDK version. Defaults to all JDKs the current branch supports.")
+    parser.add_argument("-t", "--repeat-test-regex", help="Test name regexp (csv list) to run repeatedly via the *-repeat stages. Requires -p custom and -e selecting a *-repeat stage. Example: 'HostReplacementTest'")
+    parser.add_argument("-n", "--repeat-count", help="How many times to run the *-repeat stages. Example: 200")
+    parser.add_argument("--repeat-stop-on-failure", action="store_true", help="Stop all workers in a *-repeat stage when any worker reports a failed run (default: run all iterations and report the failure rate)")
+    parser.add_argument("-m", "--repeat-machines", default="1", help="Number of machines that each run the full set of repeated test iterations in parallel (default 1). Example: 4")
     parser.add_argument("-d", "--dtest-repository", default=DEFAULT_DTEST_REPO_URL, help="DTest repository URL.")
     parser.add_argument("-k", "--dtest-branch", default=DEFAULT_DTEST_REPO_BRANCH, help="DTest repository branch.")
     parser.add_argument("-s", "--setup", action="store_true", help="Set up Jenkins before the build.")
@@ -230,6 +234,15 @@ def parse_arguments() -> argparse.Namespace:
     assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified."
     assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified."
     assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp."
+    assert re.fullmatch(r"[1-9][0-9]*", args.repeat_machines or ""), "--repeat-machines must be a positive integer."
+    repeat_options_used = args.repeat_test_regex or args.repeat_count or args.repeat_stop_on_failure or args.repeat_machines != "1"
+    repeat_stages_selected = args.profile == "custom" and any(re.fullmatch(args.profile_custom_regexp, stage)
+                                                               for stage in ("test-repeat", "jvm-dtest-repeat"))
+    repeating_tests = repeat_options_used or repeat_stages_selected
+    assert not (repeating_tests and args.profile != "custom"), "Repeating tests requires --profile custom."
+    assert not (repeating_tests and not repeat_stages_selected), "Repeating tests requires --profile-custom-regexp selecting a *-repeat stage (see `repeatTestSteps()` in .jenkins/Jenkinsfile)."
+    assert not (repeating_tests and not (args.repeat_test_regex and args.repeat_count)), "Repeating tests requires both --repeat-test-regex and --repeat-count."
+    assert not args.repeat_count or re.fullmatch(r"[1-9][0-9]*", args.repeat_count), "--repeat-count must be a positive integer."
     assert not (args.values_override and not (args.setup or args.only_setup)), "--values-override requires --setup or --only-setup."
     assert not (args.values_override and not Path(args.values_override).is_file()), f"No such values override file: {args.values_override}"
 
@@ -1158,6 +1171,10 @@ def main():
         "profile": args.profile,
         "profile_custom_regexp": args.profile_custom_regexp or "",
         "jdk": args.jdk or "",
+        "repeat_test_regex": args.repeat_test_regex or "",
+        "repeated_tests_count": args.repeat_count or "",
+        "repeated_tests_stop_on_failure": "true" if args.repeat_stop_on_failure else "false",
+        "repeated_tests_machines": args.repeat_machines or "1",
         "dtest_repository": args.dtest_repository or "",
         "dtest_branch": args.dtest_branch or ""
     }

.build/run-ci.d/README.md

@@ -2,8 +2,9 @@
 
 ```
 ➤ .build/run-ci --help
-usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH]
-              [-s] [--only-setup] [-v VALUES_OVERRIDE] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
+usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK]
+              [-t REPEAT_TEST_REGEX] [-n REPEAT_COUNT] [--repeat-stop-on-failure] [-m REPEAT_MACHINES] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH] [-s] [--only-setup] [-v VALUES_OVERRIDE]
+              [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS]
 
 Run CI pipeline for Cassandra on K8s using Jenkins.
 
@@ -24,6 +25,14 @@ options:
   -e PROFILE_CUSTOM_REGEXP, --profile-custom-regexp PROFILE_CUSTOM_REGEXP
                         Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: 'stress.*|jvm-dtest.'
   -j JDK, --jdk JDK     Specify JDK version. Defaults to all JDKs the current branch supports.
+  -t REPEAT_TEST_REGEX, --repeat-test-regex REPEAT_TEST_REGEX
+                        Test name regexp (csv list) to run repeatedly via the *-repeat stages. Requires -p custom and -e selecting a *-repeat stage. Example: 'HostReplacementTest'
+  -n REPEAT_COUNT, --repeat-count REPEAT_COUNT
+                        How many times to run the *-repeat stages. Example: 200
+  --repeat-stop-on-failure
+                        Stop all workers in a *-repeat stage when any worker reports a failed run (default: run all iterations and report the failure rate)
+  -m REPEAT_MACHINES, --repeat-machines REPEAT_MACHINES
+                        Number of machines that each run the full set of repeated test iterations in parallel (default 1). Example: 4
   -d DTEST_REPOSITORY, --dtest-repository DTEST_REPOSITORY
                         DTest repository URL.
   -k DTEST_BRANCH, --dtest-branch DTEST_BRANCH
@@ -60,6 +69,21 @@ Run the the specified fork and branch through just the "fqltool-test" tests
 .build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "fqltool-test"
 ```
 
+Run a single test 200 times to hunt for a flake (custom profile, `jvm-dtest-repeat` stage; `test-repeat` for unit tests)
+```
+.build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "jvm-dtest-repeat" -t "HostReplacementTest" -n 200
+```
+
+The same but aborting all repeat workers when any worker reports a failed run
+```
+.build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "jvm-dtest-repeat" -t "HostReplacementTest" -n 200 --repeat-stop-on-failure
+```
+
+The same but on 4 machines, each running all 200 iterations in parallel (4x the samples, same wall-clock time)
+```
+.build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "jvm-dtest-repeat" -t "HostReplacementTest" -n 200 -m 4
+```
+
 Setup/Update Jenkins Helm into your current kubeconfig
 ```
 .build/run-ci --only-setup

.build/run-tests.sh

@@ -270,7 +270,8 @@ _run_testlist() {
           done
 
           if [ "$(_get_env_var 'REPEATED_TESTS_STOP_ON_FAILURE')" == true ]; then
-            error 0 "fail fast, after ${i} successful runs"
+            # A distinct non-zero status lets Jenkins fail-fast cancel the other repeat workers.
+            error 2 "fail fast, after ${i} successful runs"
           fi
           let failures+=1
         fi
@@ -364,9 +365,10 @@ _main() {
     esac
   fi
 
-  # "-repeat" is a reserved suffix on target types
+  # "-repeat" is a reserved suffix on target types.
+  # Splits are allowed and multiply the number of machines: every split chunk runs the full set
+  # of REPEATED_TESTS_COUNT iterations itself (the chunk does not partition the iterations).
   if [[ ${test_target} == *"-repeat" ]] ; then
-    [[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]] && { error 1 "Repeated tests not valid with splits"; }
     if [[ -z "${test_name_regexp}" ]] ; then
       error 1 "Repeated tests requires use of -t option"
     fi