diff --git a/.build/README.md b/.build/README.md index 52a8cc2178da..2d0c4d9a921e 100644 --- a/.build/README.md +++ b/.build/README.md @@ -134,6 +134,11 @@ Fail fast with repeating tests is done with REPEATED_TESTS_STOP_ON_FAILURE .build/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2 -e REPEATED_TESTS_STOP_ON_FAILURE=false +Each split chunk runs the full set of iterations on its own machine, so -c multiplies the number of +machines doing the same REPEATED_TESTS_COUNT runs (the chunk does not partition the iterations): + + .build/run-tests.sh -a jvm-dtest-repeat -t BooleanTest -e REPEATED_TESTS_COUNT=2 -c 1/2 + Running python dtests without docker: .build/run-python-dtests.sh dtest diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index 9b57a867385b..30b5448a3058 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -287,7 +287,12 @@ logfile="${build_dir}/test/logs/docker_attach_${container_name}.log" # Docker commands: # set java to java_version # execute the run_script -[ -n "${test_name_regexp}" ] && test_name_regexp_arg="-t ${test_name_regexp}" || split_chunk_arg="-c ${split_chunk}" +[ -n "${test_name_regexp}" ] && test_name_regexp_arg="-t ${test_name_regexp}" +# Named tests normally disable splitting. For *-repeat targets the chunk instead identifies the +# machine, with each machine running every iteration. +if [ -z "${test_name_regexp}" ] || [[ "${test_target}" == *"-repeat" ]]; then + split_chunk_arg="-c ${split_chunk}" +fi docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_version} ; \ \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh -a ${target} ${split_chunk_arg} ${test_name_regexp_arg} ${env_vars} ; exit \$?" diff --git a/.build/run-ci b/.build/run-ci index 22d30c3603f6..371776a5575e 100755 --- a/.build/run-ci +++ b/.build/run-ci @@ -198,6 +198,10 @@ def argument_parser() -> argparse.ArgumentParser: parser.add_argument("-p", "--profile", choices=['packaging','skinny','pre-commit','pre-commit w/ upgrades','post-commit','custom'], default=DEFAULT_PROFILE, help="CI pipeline profile. Defaults to skinny.") parser.add_argument("-e", "--profile-custom-regexp", help="Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: 'stress.*|jvm-dtest.'") parser.add_argument("-j", "--jdk", help="Specify JDK version. Defaults to all JDKs the current branch supports.") + parser.add_argument("-t", "--repeat-test-regex", help="Test name regexp (csv list) to run repeatedly via the *-repeat stages. Requires -p custom and -e selecting a *-repeat stage. Example: 'HostReplacementTest'") + parser.add_argument("-n", "--repeat-count", help="How many times to run the *-repeat stages. Example: 200") + parser.add_argument("--repeat-stop-on-failure", action="store_true", help="Stop all workers in a *-repeat stage when any worker reports a failed run (default: run all iterations and report the failure rate)") + parser.add_argument("-m", "--repeat-machines", default="1", help="Number of machines that each run the full set of repeated test iterations in parallel (default 1). Example: 4") parser.add_argument("-d", "--dtest-repository", default=DEFAULT_DTEST_REPO_URL, help="DTest repository URL.") parser.add_argument("-k", "--dtest-branch", default=DEFAULT_DTEST_REPO_BRANCH, help="DTest repository branch.") parser.add_argument("-s", "--setup", action="store_true", help="Set up Jenkins before the build.") @@ -230,6 +234,15 @@ def parse_arguments() -> argparse.Namespace: assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified." assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified." assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp." + assert re.fullmatch(r"[1-9][0-9]*", args.repeat_machines or ""), "--repeat-machines must be a positive integer." + repeat_options_used = args.repeat_test_regex or args.repeat_count or args.repeat_stop_on_failure or args.repeat_machines != "1" + repeat_stages_selected = args.profile == "custom" and any(re.fullmatch(args.profile_custom_regexp, stage) + for stage in ("test-repeat", "jvm-dtest-repeat")) + repeating_tests = repeat_options_used or repeat_stages_selected + assert not (repeating_tests and args.profile != "custom"), "Repeating tests requires --profile custom." + assert not (repeating_tests and not repeat_stages_selected), "Repeating tests requires --profile-custom-regexp selecting a *-repeat stage (see `repeatTestSteps()` in .jenkins/Jenkinsfile)." + assert not (repeating_tests and not (args.repeat_test_regex and args.repeat_count)), "Repeating tests requires both --repeat-test-regex and --repeat-count." + assert not args.repeat_count or re.fullmatch(r"[1-9][0-9]*", args.repeat_count), "--repeat-count must be a positive integer." assert not (args.values_override and not (args.setup or args.only_setup)), "--values-override requires --setup or --only-setup." assert not (args.values_override and not Path(args.values_override).is_file()), f"No such values override file: {args.values_override}" @@ -1158,6 +1171,10 @@ def main(): "profile": args.profile, "profile_custom_regexp": args.profile_custom_regexp or "", "jdk": args.jdk or "", + "repeat_test_regex": args.repeat_test_regex or "", + "repeated_tests_count": args.repeat_count or "", + "repeated_tests_stop_on_failure": "true" if args.repeat_stop_on_failure else "false", + "repeated_tests_machines": args.repeat_machines or "1", "dtest_repository": args.dtest_repository or "", "dtest_branch": args.dtest_branch or "" } diff --git a/.build/run-ci.d/README.md b/.build/run-ci.d/README.md index d87e5b137880..d408f9bc7571 100644 --- a/.build/run-ci.d/README.md +++ b/.build/run-ci.d/README.md @@ -2,8 +2,9 @@ ``` ➤ .build/run-ci --help -usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH] - [-s] [--only-setup] [-v VALUES_OVERRIDE] [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS] +usage: run-ci [-h] [-c KUBECONFIG] [-x KUBECONTEXT] [-i URL] [-u USER] [-r REPOSITORY] [-b BRANCH] [-p {packaging,skinny,pre-commit,pre-commit w/ upgrades,post-commit,custom}] [-e PROFILE_CUSTOM_REGEXP] [-j JDK] + [-t REPEAT_TEST_REGEX] [-n REPEAT_COUNT] [--repeat-stop-on-failure] [-m REPEAT_MACHINES] [-d DTEST_REPOSITORY] [-k DTEST_BRANCH] [-s] [--only-setup] [-v VALUES_OVERRIDE] + [--tear-down] [--only-tear-down] [--only-node-cleaner] [-o DOWNLOAD_RESULTS] Run CI pipeline for Cassandra on K8s using Jenkins. @@ -24,6 +25,14 @@ options: -e PROFILE_CUSTOM_REGEXP, --profile-custom-regexp PROFILE_CUSTOM_REGEXP Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: 'stress.*|jvm-dtest.' -j JDK, --jdk JDK Specify JDK version. Defaults to all JDKs the current branch supports. + -t REPEAT_TEST_REGEX, --repeat-test-regex REPEAT_TEST_REGEX + Test name regexp (csv list) to run repeatedly via the *-repeat stages. Requires -p custom and -e selecting a *-repeat stage. Example: 'HostReplacementTest' + -n REPEAT_COUNT, --repeat-count REPEAT_COUNT + How many times to run the *-repeat stages. Example: 200 + --repeat-stop-on-failure + Stop all workers in a *-repeat stage when any worker reports a failed run (default: run all iterations and report the failure rate) + -m REPEAT_MACHINES, --repeat-machines REPEAT_MACHINES + Number of machines that each run the full set of repeated test iterations in parallel (default 1). Example: 4 -d DTEST_REPOSITORY, --dtest-repository DTEST_REPOSITORY DTest repository URL. -k DTEST_BRANCH, --dtest-branch DTEST_BRANCH @@ -60,6 +69,21 @@ Run the the specified fork and branch through just the "fqltool-test" tests .build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "fqltool-test" ``` +Run a single test 200 times to hunt for a flake (custom profile, `jvm-dtest-repeat` stage; `test-repeat` for unit tests) +``` +.build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "jvm-dtest-repeat" -t "HostReplacementTest" -n 200 +``` + +The same but aborting all repeat workers when any worker reports a failed run +``` +.build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "jvm-dtest-repeat" -t "HostReplacementTest" -n 200 --repeat-stop-on-failure +``` + +The same but on 4 machines, each running all 200 iterations in parallel (4x the samples, same wall-clock time) +``` +.build/run-ci -r "https://github.com/jrwest/cassandra.git" -b "jwest/15452-5.0" -p "custom" -e "jvm-dtest-repeat" -t "HostReplacementTest" -n 200 -m 4 +``` + Setup/Update Jenkins Helm into your current kubeconfig ``` .build/run-ci --only-setup diff --git a/.build/run-tests.sh b/.build/run-tests.sh index 470cc85b35fc..4ccf92e1a40f 100755 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@ -270,7 +270,8 @@ _run_testlist() { done if [ "$(_get_env_var 'REPEATED_TESTS_STOP_ON_FAILURE')" == true ]; then - error 0 "fail fast, after ${i} successful runs" + # A distinct non-zero status lets Jenkins fail-fast cancel the other repeat workers. + error 2 "fail fast, after ${i} successful runs" fi let failures+=1 fi @@ -364,9 +365,10 @@ _main() { esac fi - # "-repeat" is a reserved suffix on target types + # "-repeat" is a reserved suffix on target types. + # Splits are allowed and multiply the number of machines: every split chunk runs the full set + # of REPEATED_TESTS_COUNT iterations itself (the chunk does not partition the iterations). if [[ ${test_target} == *"-repeat" ]] ; then - [[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]] && { error 1 "Repeated tests not valid with splits"; } if [[ -z "${test_name_regexp}" ]] ; then error 1 "Repeated tests requires use of -t option" fi diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile index 2f0d786ca6ba..0e0527d19c38 100644 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@ -76,6 +76,10 @@ pipeline { choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.') string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*') + string(name: 'repeat_test_regex', defaultValue: params.repeat_test_regex ?: '', description: 'Test name regexp (csv list) to run repeatedly. Only used by *-repeat stages, see `repeatTestSteps()` in Jenkinsfile. Example: HostReplacementTest') + string(name: 'repeated_tests_count', defaultValue: params.repeated_tests_count ?: '', description: 'How many times to run the tests selected by repeat_test_regex in a *-repeat stage. Example: 200') + booleanParam(name: 'repeated_tests_stop_on_failure', defaultValue: false, description: 'Stop all workers in a *-repeat stage when any worker reports a failed run. Default runs all iterations and reports the failure rate.') + string(name: 'repeated_tests_machines', defaultValue: params.repeated_tests_machines ?: '1', description: 'Number of machines that each run the full set of repeated test iterations in parallel, in *-repeat stages. Example: 4') choice(name: 'architecture', choices: archsSupported + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.') string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)') @@ -89,7 +93,19 @@ pipeline { script { // this helps assure folk their parameters are correct and will be used (despite the earlier output about the configured job coordinates) echo "Printing parameters used for this build" - ["Repository: ${params.repository}", "Branch: ${params.branch}", "Profile: ${params.profile}", "Custom Profile Regexp: ${params.profile_custom_regexp}", "Architecture: ${params.architecture}", "JDK: ${params.jdk}", "DTest Repository: ${params.dtest_repository}", "DTest Branch: ${params.dtest_branch}"].each { println it } + ["Repository: ${params.repository}", "Branch: ${params.branch}", "Profile: ${params.profile}", "Custom Profile Regexp: ${params.profile_custom_regexp}", "Architecture: ${params.architecture}", "JDK: ${params.jdk}", "DTest Repository: ${params.dtest_repository}", "DTest Branch: ${params.dtest_branch}", "Repeat Test Regexp: ${params.repeat_test_regex}", "Repeated Tests Count: ${params.repeated_tests_count}", "Repeated Tests Stop On Failure: ${params.repeated_tests_stop_on_failure}"].each { println it } + def repeatStagesSelected = repeatTestSteps().keySet().findAll { it ==~ (params.profile_custom_regexp ?: '') } + if ("custom" == params.profile && repeatStagesSelected) { + if (!(params.repeat_test_regex?.trim() && params.repeated_tests_count?.trim())) { + error("The custom profile regexp '${params.profile_custom_regexp}' selects the repeat stages ${repeatStagesSelected}, which require the 'repeat_test_regex' and 'repeated_tests_count' parameters") + } + if (!(params.repeated_tests_count?.trim() ==~ /^[1-9][0-9]*$/)) { + error("The 'repeated_tests_count' parameter must be a positive integer, got: '${params.repeated_tests_count}'") + } + if (!(params.repeated_tests_machines?.trim() ==~ /^[1-9][0-9]*$/)) { + error("The 'repeated_tests_machines' parameter must be a positive integer, got: '${params.repeated_tests_machines}'") + } + } } } } @@ -149,6 +165,29 @@ def pipelineProfiles() { ] } +@NonCPS +def repeatTestSteps() { + // stages that re-run the test(s) named by the `repeat_test_regex` parameter `repeated_tests_count` times + // (the REPEATED_TESTS_COUNT/REPEATED_TESTS_STOP_ON_FAILURE options of .build/run-tests.sh). + // Custom profile only (see isStageEnabled). The stage is "split" across `repeated_tests_machines` + // machines, each running every iteration (see splitsFor()). A long timeout: 200 iterations of a + // dtest easily outlasts the default hour. + return [ + 'test-repeat': [splits: 1, size: 'medium', timeout_hours: 24], + 'jvm-dtest-repeat': [splits: 1, size: 'medium', timeout_hours: 24], + ] +} + +// number of machines each running the full set of repeated test iterations in *-repeat stages +def repeatMachines() { + return (params.repeated_tests_machines?.trim() ?: '1').toInteger() +} + +// repeat stages are "split" across machines, each running every iteration; other stages split the test set +def splitsFor(String step, def stepConfig) { + return step.endsWith('-repeat') ? repeatMachines() : stepConfig.splits +} + @NonCPS def pipelineProfileNames(putFirst) { set = pipelineProfiles().keySet() as List @@ -217,6 +256,8 @@ def tasks() { // performance tests need 'cassandra-*large-dedicated' nodes 'microbench': [splits: 4, size: 'large', timeout_hours: 6, benchmark: true], ] + // *-repeat stages: re-run a single test (csv list) N times, see the repeat parameters + testSteps.putAll(repeatTestSteps()) testSteps.each() { it.value.put('type', 'test') if (!it.value['size']) { @@ -245,7 +286,7 @@ def tasks() { python: pythonsSupported, cython: ['yes', 'no'], step: stepsMap.keySet(), - split: (1..testSteps.values().splits.max()).toList() + split: (1..Math.max(testSteps.values().splits.max(), repeatMachines())).toList() ] def List _axes = getMatrixAxes(matrix_axes).findAll { axis -> @@ -257,7 +298,7 @@ def tasks() { !(axis['cython'] == 'yes' && (axis['python'] == '3.12' || axis['python'] == '3.13')) && // Skip cython for Python 3.12+ see CASSANDRA-21482 !(axis['jdk'] != javaVersionDefault && ('cqlsh-test' == axis['step'] || 'simulator-dtest' == axis['step'] || axis['step'].contains('dtest-upgrade'))) && // run cqlsh-test, simulator-dtest, *dtest-upgrade only with jdk11 // Disable splits for all but proper stages - !(axis['split'] > 1 && !stepsMap.findAll { entry -> entry.value.splits >= axis['split'] }.keySet().contains(axis['step'])) && + !(axis['split'] > 1 && !stepsMap.findAll { entry -> splitsFor(entry.key, entry.value) >= axis['split'] }.keySet().contains(axis['step'])) && // run only the build types on non-amd64 !(axis['arch'] != 'amd64' && !stepsMap.findAll { entry -> 'build' == entry.value.type }.keySet().contains(axis['step'])) } @@ -295,7 +336,8 @@ def getStepName(cell, command) { def arch = "amd64" == cell.arch ? "" : " ${cell.arch}" def python = "cqlsh-test" != cell.step ? "" : " python${cell.python}" def cython = "no" == cell.cython ? "" : " cython" - def split = command.splits > 1 ? " ${cell.split}/${command.splits}" : "" + def splits = splitsFor(cell.step, command) + def split = splits > 1 ? " ${cell.split}/${splits}" : "" return "${cell.step}${arch} jdk${cell.jdk}${python}${cython}${split}" } @@ -325,7 +367,9 @@ def isCanonical() { } def isStageEnabled(stage) { - return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp) + return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp && + // *-repeat stages also need their parameters, otherwise run-tests.sh would fail inside the container + (!stage.endsWith('-repeat') || (params.repeat_test_regex?.trim() && params.repeated_tests_count?.trim()))) } def isArchEnabled(arch) { @@ -401,8 +445,10 @@ def build(command, cell) { def test(command, cell) { if (command.containsKey('script')) { error("test commands all use `.build/docker/run-tests.sh`") } - def splits = command.splits ? command.splits : 1 - def maxAttempts = 2 + def splits = splitsFor(cell.step, command) + def stopRepeatedTestsOnFailure = cell.step.endsWith('-repeat') && params.repeated_tests_stop_on_failure + // An intentional repeat-test failure must cancel sibling workers, not retry this worker. + def maxAttempts = stopRepeatedTestsOnFailure ? 1 : 2 def attempt = 0 def nodeExclusion = "" retry(maxAttempts) { @@ -423,16 +469,30 @@ def test(command, cell) { script_vars = "${script_vars} cython=\'${cell.cython}\'" } script_vars = fetchDTestsSource(command, script_vars) + def status = 0 + def repeatedTestFailed = false timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes def timer = System.currentTimeMillis() try { buildJVMDTestJars(cell, script_vars, logfile) script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\"" - def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true + def test_args = ".build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk}" + if (cell.step.endsWith('-repeat')) { + // each "split" machine runs the full set of iterations itself, hence -c plus the repeat options + test_args += " -t '${params.repeat_test_regex}' -e REPEATED_TESTS_COUNT=${params.repeated_tests_count}" + if (params.repeated_tests_stop_on_failure) { + test_args += " -e REPEATED_TESTS_STOP_ON_FAILURE=true" + } + } + status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} ${test_args} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true dir("build") { archiveArtifacts artifacts: "${logfile}", fingerprint: true } - if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } + // run-tests.sh uses status 2 for an intentional repeat-test fail-fast. Preserve its + // results below before failing this branch, which makes parallel(failFast: true) + // interrupt the other workers. Other execution failures retain the normal retry path. + repeatedTestFailed = stopRepeatedTestsOnFailure && 2 == status + if (0 != status && !repeatedTestFailed) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") } } catch (exc) { if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) { def descriptions = [] @@ -462,6 +522,9 @@ def test(command, cell) { archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_")) } + if (repeatedTestFailed) { + error("Stage ${cell.step}${cell_suffix} stopped after a repeated test failure") + } } finally { cleanAgent(cell.step) }