Engage vectorizers, omp simd correctness

.gitignore

@@ -7,6 +7,7 @@ res
 *.swp
 found/old-*
 check_isa
+check_isa.exe
 Test_Makefile
 testing_log_*
 gen_test_makefile_log_*

bugs.rst

@@ -3,6 +3,7 @@ LLVM:
 ====================================
 Version #2
 ---------------
+| `221240 <https://github.com/llvm/llvm-project/issues/221240>`_ - unsigned short >= short compare miscompiled at -O2 - narrowed to signed i16 compare
 | `58616 <https://github.com/llvm/llvm-project/issues/58616>`_ - SLPVectorizer: Assertion \`isVectorLikeInstWithConstOps(FirstInst) && isVectorLikeInstWithConstOps(I) && "Expected vector-like insts only."' failed with -O3 -march=skx
 | `53807 <https://github.com/llvm/llvm-project/issues/53807>`_ - [NewGVN] Assertion \`BeforeCC->isEquivalentTo(AfterCC) && "Value number changed after main loop completed!"' failed
 | `52561 <https://bugs.llvm.org/show_bug.cgi?id=52561>`_ - Assertion failed for sapphirerapids: \`VT.getSizeInBits() == Operand.getValueSizeInBits() && "Cannot BITCAST between types of different sizes!"'
@@ -310,3 +311,15 @@ Version #1
 TCC - Tiny C Compiler:
 ====================================
 | `63816 <https://savannah.nongnu.org/bugs/?63816>`_ - tcc miscompiled test code goes from the middle of an if section into the else section (yarpgen v1)
+
+====================================
+MSVC:
+====================================
+| `11131532 <https://developercommunity.visualstudio.com/t/MSVC-miscompiles-signed-x-lt;lt;-k/11131532>`_ - MSVC miscompiles signed `(x << k) > 0` as `x > 0`
+| `11126250 <https://developercommunity.visualstudio.com/t/MSVC-O1-miscompiles-nested-array-loop/11126250>`_ - MSVC /O1 miscompiles nested array loop, regression from 19.50
+| `11125055 <https://developercommunity.visualstudio.com/t/MSVC-O1-and-O2-miscompiles-signed-modu/11125055>`_ - MSVC /O1 and /O2 miscompiles signed modulo comparison expression
+| `11140883 <https://developercommunity.visualstudio.com/t/MSVC-O2-miscompiles-global-assignment-f/11140883>`_ - MSVC /O2 miscompiles global assignment followed by XOR operation
+| `11141098 <https://developercommunity.visualstudio.com/t/C-optimizer-miscompiles-x-gt;gt;-k/11141098>`_ - C++ optimizer miscompiles (x >> k) >= C when C << k overflows
+| `11126097 <https://developercommunity.visualstudio.com/t/MSVC-miscompiles-unsigned-long-long-expr/11126097>`_ - MSVC miscompiles unsigned long long expression and skips loop in /O1
+| `11127140 <https://developercommunity.visualstudio.com/t/MSVC-O2-miscompiles-short-to-unsigned-w/11127140>`_ - MSVC /O2 miscompiles short-to-unsigned widening and reuses wrong sign extension
+| `11148645 <https://developercommunity.visualstudio.com/t/MSVC-Ot-optimization-causes-super-quadr/11148645>`_ - MSVC /Ot optimization causes super-quadratic compile time for nested loops

scripts/README-msvc-wsl.md

@@ -79,6 +79,32 @@ won't link. The generator therefore gained `--max-array-dims=<n>` (default `0` =
 no explicit cap), and `run_gen.py` sets it to `3` when MSVC is in the pool so the
 shared test fits every compiler. Tune the value if you want larger arrays.
 
+## Targeting MSVC's auto-vectorizer
+
+MSVC's auto-vectorizer is a narrow, pattern-matching one (unlike gcc/clang's):
+it wants a flat, single-statement loop body with unit stride and no nested
+control flow, and yarpgen's default output is far more complex than that, so
+it essentially never fires. Pass `--simple-loops={some,all}` to `run_gen.py`
+to forward the generator's `--simple-loops` flag, which constrains loops to
+that narrow shape:
+
+```sh
+run_gen.py --std c++ --config-file "$YARPGEN_HOME/scripts/test_sets_msvc.txt" \
+           --simple-loops all -o testing --target "gcc msvc" -t 60
+```
+
+- `some` (default policy's own "vectorizable" loops also get the stricter
+  simple-loop treatment) or `all` (every loop does). `none` (the default)
+  omits the flag entirely, i.e. today's behavior.
+- This is a distinct, opt-in mode, not something to leave on for general bug
+  hunting: it trades away the operator/control-flow diversity that finds
+  non-vectorization bugs (this is how the existing 6 MSVC bugs were found,
+  all unrelated to vectorization) in exchange for shapes MSVC's vectorizer
+  will actually attempt. Run it as its own separate session.
+- Even with `all`, only a majority of the resulting loops actually get
+  vectorized (checked via `/Qvec-report:2`) — MSVC still declines some for
+  reasons like too little in-loop computation or too few iterations.
+
 ## Why `/permissive-`
 
 `cl.exe` keeps alive some language extensions and known bugs to allow for backwards

scripts/run_gen.py

@@ -199,6 +199,18 @@ class Test(object):
     # Static variables
     # Don't save anything other than log-file if compile time expires
     ignore_comp_time_exp = True
+    # Forwarded to the generator's --simple-loops flag ("none", "some", "all").
+    # "none" means the flag is omitted entirely, matching the generator's own default.
+    simple_loops_mode = "none"
+    # Forwarded to the generator's --vectorizer-target flag ("msvc", "gcc-clang").
+    # Only meaningful when simple_loops_mode != "none".
+    vectorizer_target = "gcc-clang"
+    # Forwarded to the generator's --omp-simd flag ("on", "off"). "on" means the
+    # flag is omitted entirely, matching the generator's own default.
+    omp_simd_pragma = "on"
+    # Forwarded to the generator's --max-array-dims flag. 0 means the flag is
+    # omitted entirely, matching the generator's own default (unlimited).
+    max_array_dims = 0
 
     # Generate new test
     # stat is statistics object
@@ -212,12 +224,20 @@ def __init__(self, stat, seed="", proc_num=-1, blame=False, creduce_makefile=Non
             yarpgen_run_list += ["-s", seed]
         # MSVC (driven via cl-proxy) has no large-memory model and a 2GB image
         # limit, so the default multi-GB arrays won't link. When an MSVC target
-        # is selected, cap the array dimensions so the shared test fits.
+        # is selected and the user hasn't set --max-array-dims explicitly, cap
+        # the array dimensions so the shared test fits.
         msvc_selected = any(
             getattr(t.specs, "comp_cxx_name", "") == "cl-proxy"
             for t in gen_test_makefile.CompilerTarget.all_targets)
-        if msvc_selected:
+        if Test.max_array_dims != 0:
+            yarpgen_run_list.append("--max-array-dims=" + str(Test.max_array_dims))
+        elif msvc_selected:
             yarpgen_run_list.append("--max-array-dims=3")
+        if Test.simple_loops_mode != "none":
+            yarpgen_run_list.append("--simple-loops=" + Test.simple_loops_mode)
+            yarpgen_run_list.append("--vectorizer-target=" + Test.vectorizer_target)
+        if Test.omp_simd_pragma != "on":
+            yarpgen_run_list.append("--omp-simd=" + Test.omp_simd_pragma)
         self.yarpgen_cmd = " ".join(str(p) for p in yarpgen_run_list)
         self.ret_code, self.stdout, self.stderr, self.is_time_expired, self.elapsed_time = \
             common.run_cmd(yarpgen_run_list, yarpgen_timeout, proc_num, yarpgen_mem_limit)
@@ -1707,8 +1727,10 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescri
     # Worker processes rely on inheriting global state (logger,
     # CompilerSpecs.all_comp_specs, ...) set up here in the parent, which
     # only works with 'fork' semantics. Python 3.14 changed the default
-    # start method on Linux to 'forkserver', which breaks that.
-    multiprocessing.set_start_method("fork")
+    # start method on Linux to 'forkserver', which breaks that. Windows has
+    # no 'fork' at all, so it keeps its default 'spawn'.
+    if "fork" in multiprocessing.get_all_start_methods():
+        multiprocessing.set_start_method("fork")
 
     if os.environ.get("YARPGEN_HOME") is None:
         sys.stderr.write("\nWarning: please set YARPGEN_HOME environment variable to point to yarpgen's directory,"
@@ -1766,6 +1788,26 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescri
                         help="List of testing sets for statistics collection")
     parser.add_argument("--ignore-comp-time-exp", dest="ignore_comp_time_exp", default=True, action="store_true",
                         help="Don't save files (except log-file) when compile time expires")
+    parser.add_argument("--simple-loops", dest="simple_loops", default="none", choices=["none", "some", "all"],
+                        type=str,
+                        help="Forwarded to the generator's --simple-loops: constrain loops to shapes that "
+                             "narrow vectorizers (e.g. MSVC's) can recognize. 'none' omits the flag "
+                             "(generator default).")
+    parser.add_argument("--vectorizer-target", dest="vectorizer_target", default="gcc-clang",
+                        choices=["msvc", "gcc-clang"], type=str,
+                        help="Forwarded to the generator's --vectorizer-target: which vectorizer's "
+                             "recognizable loop shapes --simple-loops should constrain to. Only meaningful "
+                             "when --simple-loops != none.")
+    parser.add_argument("--omp-simd", dest="omp_simd_pragma", default="on",
+                        choices=["on", "off"], type=str,
+                        help="Forwarded to the generator's --omp-simd: whether \"#pragma omp simd\" "
+                             "(and its reduction clauses) may be generated. Use 'off' to keep OpenMP "
+                             "semantics out of the generated tests entirely.")
+    parser.add_argument("--max-array-dims", dest="max_array_dims", default=0, type=int,
+                        help="Forwarded to the generator's --max-array-dims: caps the number of array "
+                             "dimensions. 0 omits the flag (generator default, unlimited), except when an "
+                             "MSVC target is selected, in which case it's auto-capped at 3 to fit MSVC's "
+                             "2GB image limit.")
     args = parser.parse_args()
 
     log_level = logging.DEBUG if args.verbose else logging.INFO
@@ -1791,6 +1833,10 @@ class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescri
     targets = re.split(' |,', args.target)
 
     Test.ignore_comp_time_exp = args.ignore_comp_time_exp
+    Test.simple_loops_mode = args.simple_loops
+    Test.vectorizer_target = args.vectorizer_target
+    Test.omp_simd_pragma = args.omp_simd_pragma
+    Test.max_array_dims = args.max_array_dims
     prepare_env_and_start_testing(os.path.abspath(args.out_dir), args.timeout, targets, args.num_jobs,
                                   args.config_file, args.seeds_option_value, args.blame, args.creduce,
                                   args.no_tmp_cleaner, args.collect_stat)

scripts/test_sets.txt

@@ -30,7 +30,7 @@ Compiler specs:
 # Arch prefix - prefix for specifying different architectures (it will be concatenated with Compiler arch value)
 
 # Spec name | C++ executable name | C executable name | Common arguments                                            | Arch prefix
-gcc         | g++                 | gcc               | -fPIC -mcmodel=large -w -fpermissive                                             | -march=
+gcc         | g++                 | gcc               | -fPIC -mcmodel=large -w -fpermissive -fopenmp-simd                               | -march=
 clang       | clang++             | clang             | -fPIC -mcmodel=large -w -fopenmp-simd                                            | -march=
 polly       | clang++             | clang             | -fPIC -mcmodel=large -w -fopenmp-simd -mllvm -polly -mllvm -polly-vectorizer=stripmine | -march=
 # Ubsan is clang or gcc with sanitizer options. It is used for generator check.
@@ -39,8 +39,8 @@ polly       | clang++             | clang             | -fPIC -mcmodel=large -w
 # See https://bugs.llvm.org//show_bug.cgi?id=16404 for more information
 # Note that -fpermissive option for gcc is required to allow reduction of ubsan_gcc fails.
 # Otherwise result of reduction is empty program.
-ubsan_clang | clang++             | clang             | -fPIC -mcmodel=large -fsanitize=undefined -fno-sanitize-recover=undefined -Werror=uninitialized -Werror=implicitly-unsigned-literal | -march=
-ubsan_gcc   | g++                 | gcc               | -fPIC -mcmodel=large -fsanitize=undefined -fno-sanitize-recover=undefined -Werror=uninitialized -fpermissive | -march=
+ubsan_clang | clang++             | clang             | -fPIC -mcmodel=large -fsanitize=undefined -fno-sanitize-recover=undefined -Werror=uninitialized -Werror=implicitly-unsigned-literal -fopenmp-simd | -march=
+ubsan_gcc   | g++                 | gcc               | -fPIC -mcmodel=large -fsanitize=undefined -fno-sanitize-recover=undefined -Werror=uninitialized -fpermissive -fopenmp-simd | -march=
 icc         | icpc                | icc               | -fPIC -mcmodel=large -w                                                          | -x
 icx         | icpx                | icx               | -fPIC -mcmodel=large -w -fopenmp-simd -mllvm -vec-threshold=0                    | -march=
 dpcpp       | clang++             | clang             | -fsycl -w                                                   | -march=

src/context.cpp

@@ -41,6 +41,7 @@ PopulateCtx::PopulateCtx(std::shared_ptr<PopulateCtx> _par_ctx)
         inside_foreach = par_ctx->isInsideForeach();
         inside_mutation = par_ctx->isInsideMutation();
         inside_omp_simd = par_ctx->inside_omp_simd;
+        omp_reduction_vars = par_ctx->omp_reduction_vars;
         dims = par_ctx->dims;
         in_stencil = par_ctx->in_stencil;
         mul_vals_iter = par_ctx->mul_vals_iter;
@@ -57,6 +58,7 @@ PopulateCtx::PopulateCtx() {
     taken = true;
     inside_mutation = false;
     inside_omp_simd = false;
+    omp_reduction_vars = nullptr;
     in_stencil = false;
     mul_vals_iter = nullptr;
     allow_mul_vals = false;

src/context.h

@@ -30,6 +30,17 @@ limitations under the License.
 
 namespace yarpgen {
 
+// A scalar variable that a "#pragma omp simd" loop's body updates via a
+// reduction operation (e.g. "x *= ..."), together with the OpenMP reduction
+// identifier ("+", "*", "min", ...) it needs to be listed under. The pragma
+// text is generated before the loop's body exists, so these are collected
+// while populating the body and consumed afterwards, when we know what (if
+// anything) the loop actually reduces.
+struct OmpReductionVar {
+    std::shared_ptr<Expr> var;
+    std::string op;
+};
+
 // Class that is used to determine the evaluation context.
 // It allows us to evaluate the same arithmetic tree with different input
 // values.
@@ -195,6 +206,16 @@ class PopulateCtx : public GenCtx {
     void setInsideOMPSimd(bool val) { inside_omp_simd = val; }
     bool isInsideOMPSimd() { return inside_omp_simd; }
 
+    // Shared (not deep-copied) across a "#pragma omp simd" loop's whole
+    // body, so that nested scopes/loops all register into the same list,
+    // which the loop that actually owns the pragma later reads back.
+    void setOmpReductionVars(std::shared_ptr<std::vector<OmpReductionVar>> v) {
+        omp_reduction_vars = std::move(v);
+    }
+    std::shared_ptr<std::vector<OmpReductionVar>> getOmpReductionVars() {
+        return omp_reduction_vars;
+    }
+
     size_t generateNumberOfDims(ArrayDimsUseKind dims_use_kind) const;
     void addDimension(size_t dim) { dims.push_back(dim); }
     std::vector<size_t> getDimensions() { return dims; }
@@ -226,6 +247,7 @@ class PopulateCtx : public GenCtx {
     // As of now, the pragma omp simd is attached to a loop and can't be nested.
     // TODO: we need to think about pragma omp ordered simd
     bool inside_omp_simd;
+    std::shared_ptr<std::vector<OmpReductionVar>> omp_reduction_vars;
 
     // Each loop header has a limit that any iterator should respect
     // For the simplicity, we assume that the limit is the same for all

src/data.cpp

@@ -21,6 +21,7 @@ limitations under the License.
 #include "context.h"
 #include "expr.h"
 
+#include <algorithm>
 #include <utility>
 
 using namespace yarpgen;
@@ -158,17 +159,21 @@ void Iterator::setParameters(std::shared_ptr<Expr> _start,
 }
 
 std::shared_ptr<Iterator> Iterator::create(std::shared_ptr<PopulateCtx> ctx,
-                                           size_t _end_val, bool is_uniform) {
+                                           size_t _end_val, bool is_uniform,
+                                           bool force_omp_canonical) {
     // TODO: this function is full of magic constants and weird hacks to cut
     //  some corners for ISPC and overflows
     auto gen_pol = ctx->getGenPolicy();
 
     IntTypeID type_id = rand_val_gen->getRandId(gen_pol->int_type_distr);
 
-    // TODO: It looks like integral promotion rules for bool in ISPC are
-    // broken, so we have to do them manually
     Options &options = Options::getInstance();
-    if (options.isISPC() && type_id == IntTypeID::BOOL)
+    // A bool induction variable is fine in a plain loop, but:
+    //   - ISPC's integral promotion rules for bool are broken so we have to do them manually
+    //   - under "#pragma omp simd" the loop must be in OpenMP canonical form, which includes
+    //     int induction var
+    if (type_id == IntTypeID::BOOL &&
+        (options.isISPC() || force_omp_canonical || ctx->isInsideOMPSimd()))
         type_id = IntTypeID::INT;
 
     std::shared_ptr<Type> type = IntegralType::init(type_id);
@@ -239,6 +244,15 @@ std::shared_ptr<Iterator> Iterator::create(std::shared_ptr<PopulateCtx> ctx,
         nh.getIterName(), type, start, left_span, end, right_span, step,
         end_val == left_span, total_iters_num);
 
+    // Under "#pragma omp simd", and whenever "force_omp_canonical" asks for it,
+    // Iterator::create() is constrained to an integer type that is not bool and
+    // populate() keeps the step call-free, so the loop stays in OpenMP's
+    // canonical form. "force_omp_canonical" has to count here too: such an
+    // iterator heads a same-iter-space span and is canonical in fact even
+    // though it was built outside a simd region, and LoopSeqStmt::populate()
+    // consults this flag before cloning it into the loops that follow.
+    iter->setOmpCanonical(ctx->isInsideOMPSimd() || force_omp_canonical);
+
     bool supports_mul_vals = step_val % 2 != Options::main_val_idx ||
                              left_span % 2 != Options::main_val_idx;
     if (supports_mul_vals) {
@@ -327,7 +341,8 @@ static std::shared_ptr<Expr> adjustIterExprValue(std::shared_ptr<Expr> expr,
     return ret;
 }
 
-void Iterator::populate(std::shared_ptr<PopulateCtx> ctx) {
+void Iterator::populate(std::shared_ptr<PopulateCtx> ctx,
+                        bool force_omp_canonical) {
     auto gen_pol = ctx->getGenPolicy();
 
     Options &options = Options::getInstance();
@@ -409,6 +424,22 @@ void Iterator::populate(std::shared_ptr<PopulateCtx> ctx) {
 
     start = populate_impl(type, start);
     end = populate_impl(type, end);
+
+    // OpenMP canonical loop form restricts the increment to a simple loop-invariant
+    // integer expression. A function call (say std::min/std::max) isn't one.
+    if (force_omp_canonical || ctx->isInsideOMPSimd()) {
+        auto step_gen_pol = std::make_shared<GenPolicy>(*gen_pol);
+        auto &step_node_distr = step_gen_pol->arith_node_distr;
+        step_node_distr.erase(
+            std::remove_if(step_node_distr.begin(), step_node_distr.end(),
+                           [](Probability<IRNodeKind> &p) {
+                               return p.getId() == IRNodeKind::CALL;
+                           }),
+            step_node_distr.end());
+        new_ctx->setGenPolicy(step_gen_pol);
+        gen_pol = step_gen_pol;
+    }
+
     step = populate_impl(type, step);
 }