fix: separate denormal modes for inputs and outputs

ir/instr.cpp

@@ -849,10 +849,13 @@ static StateValue fm_poison(State &s, expr a, const expr &ap, expr b,
   }
 
   if (!bitwise && val.isFloat()) {
+    const FloatType &ty = to_ty ? *to_ty->getAsFloatType() : fpty;
+    // the denormal mode is per-type, so the flushing of the result is governed
+    // by the mode of the result's type rather than that of the operands; these
+    // differ for casts between floating-point types
     val = handle_subnormal(s,
-                           s.getFn().getFnAttrs().getFPDenormal(from_ty).output,
+                           s.getFn().getFnAttrs().getFPDenormal(ty).output,
                            std::move(val));
-    const FloatType &ty = to_ty ? *to_ty->getAsFloatType() : fpty;
     val = ty.fromFloat(s, val, fpty, nary, a, b, c);
   }
 

tests/alive-tv/fp/denormal-fptrunc-result-type-2.srctgt.ll

@@ -0,0 +1,13 @@
+; Converse of denormal-fptrunc-result-type.srctgt.ll: the f32 result of this
+; fptrunc is governed by the "float:" override even though the operand is f64,
+; so the subnormal result may be flushed to +0.0.
+
+define float @src() denormal_fpenv(float: positivezero) {
+; 0x3730000000000000 is 0x1p-140: a normal double, and a subnormal float
+  %v = fptrunc double 0x3730000000000000 to float
+  ret float %v
+}
+
+define float @tgt() denormal_fpenv(float: positivezero) {
+  ret float 0.000000e+00
+}

tests/alive-tv/fp/denormal-fptrunc-result-type-3.srctgt.ll

@@ -0,0 +1,14 @@
+; ERROR: Value mismatch
+
+; As denormal-fptrunc-result-type-2.srctgt.ll, but "float: ieee" overrides the
+; base positivezero for the f32 result type, so the result is not flushed.
+
+define float @src() denormal_fpenv(positivezero, float: ieee) {
+; 0x3730000000000000 is 0x1p-140: a normal double, and a subnormal float
+  %v = fptrunc double 0x3730000000000000 to float
+  ret float %v
+}
+
+define float @tgt() denormal_fpenv(positivezero, float: ieee) {
+  ret float 0.000000e+00
+}

tests/alive-tv/fp/denormal-fptrunc-result-type.srctgt.ll

@@ -0,0 +1,14 @@
+; The denormal mode is per-type, and the flushing of an operation's result is
+; governed by the mode of the *result* type, not of its operands.  Here the
+; "float:" override applies only to f32, so the f16 result of this fptrunc
+; follows the base ieee mode and must not be flushed.
+
+define half @src() denormal_fpenv(float: preservesign) {
+  ret half 0xH0001
+}
+
+define half @tgt() denormal_fpenv(float: preservesign) {
+; 0x3E70000000000000 is 0x1p-24: a normal float, and the smallest subnormal half
+  %v = fptrunc float 0x3E70000000000000 to half
+  ret half %v
+}