fix whisper .bin suppress tokens by tokenizer

src/arch/whisper/bin_load.cpp

@@ -20,6 +20,7 @@
 #include <cmath>
 #include <cstdio>
 #include <cstring>
+#include <iterator>
 #include <memory>
 #include <string>
 #include <unordered_map>
@@ -118,20 +119,28 @@ WhisperSpecials compute_specials(int n_vocab) {
     return s;
 }
 
-// HF generation_config.suppress_tokens for whisper (~88 ids never emitted in
-// transcripts). Hardcoded here because the .bin doesn't store it; source is HF
-// generation_config.json (identical across variants; suppressing ids absent in
-// the smaller .en vocab is a no-op).
-const int32_t k_whisper_suppress_tokens[] = {
-    1,     2,     7,     8,     9,     10,    14,    25,    26,    27,    28,    29,    31,    58,    59,
-    60,    61,    62,    63,    90,    91,    92,    93,    359,   503,   522,   542,   873,   893,   902,
-    918,   922,   931,   1350,  1853,  1982,  2460,  2627,  3246,  3253,  3268,  3536,  3846,  3961,  4183,
-    4667,  6585,  6647,  7273,  9061,  9383,  10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635,
-    15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650,
-    32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50358, 50359, 50360, 50361, 50362,
+// OpenAI Whisper's tokenizer-derived non-speech ids, excluding the six
+// control tokens appended by synthesize_bin_suppress_tokens(). The legacy
+// format stores the vocabulary but not generation_config; its two tokenizer
+// families assign different meanings to most ids above the shared punctuation
+// prefix.
+const int32_t k_whisper_english_non_speech_tokens[] = {
+    1,     2,     7,     8,     9,     10,    14,    25,    26,    27,    28,    29,    31,    58,
+    59,    60,    61,    62,    63,    90,    91,    92,    93,    357,   366,   438,   532,   685,
+    705,   796,   930,   1058,  1220,  1267,  1279,  1303,  1343,  1377,  1391,  1635,  1782,  1875,
+    2162,  2361,  2488,  3467,  4008,  4211,  4600,  4808,  5299,  5855,  6329,  7203,  9609,  9959,
+    10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203,
+    19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146,
+};
+
+const int32_t k_whisper_multilingual_non_speech_tokens[] = {
+    1,     2,     7,     8,     9,     10,    14,    25,    26,    27,    28,    29,    31,    58,
+    59,    60,    61,    62,    63,    90,    91,    92,    93,    359,   503,   522,   542,   873,
+    893,   902,   918,   922,   931,   1350,  1853,  1982,  2460,  2627,  3246,  3253,  3268,  3536,
+    3846,  3961,  4183,  4667,  6585,  6647,  7273,  9061,  9383,  10428, 10929, 11938, 12033, 12331,
+    12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130,
+    26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254,
 };
-constexpr int k_whisper_n_suppress =
-    static_cast<int>(sizeof(k_whisper_suppress_tokens) / sizeof(k_whisper_suppress_tokens[0]));
 
 // Tensor rename rules: legacy whisper.cpp name -> canonical transcribe.cpp
 // name. collapse_to_1d drops the size-1 dim whisper.cpp uses for conv biases.
@@ -259,12 +268,8 @@ void fill_whisper_hparams(const transcribe::bin_loader::WhisperBinModel & bm, Wh
     hp.translate_token_id     = bm.is_multilingual ? sp.translate : -1;
     hp.prev_sot_token_id      = sp.prev;
 
-    // Suppression list — same across all multilingual whisper.cpp .bin
-    // variants. For .en, suppressing ids that don't exist in the
-    // smaller vocab is a no-op; the runtime check `id < vocab_size`
-    // gates application.
-    hp.suppress_tokens.assign(k_whisper_suppress_tokens, k_whisper_suppress_tokens + k_whisper_n_suppress);
-    hp.begin_suppress_tokens.assign({ 220, sp.eot });
+    hp.suppress_tokens       = synthesize_bin_suppress_tokens(bm.is_multilingual, bm.hp.n_vocab);
+    hp.begin_suppress_tokens = { 220, sp.eot };
 
     // Frontend (fixed across whisper variants — no .bin field carries
     // these so we use the canonical preprocessor_config.json values).
@@ -528,6 +533,20 @@ transcribe_status install_tokenizer(WhisperModel & m, const transcribe::bin_load
 
 }  // namespace
 
+std::vector<int32_t> synthesize_bin_suppress_tokens(bool is_multilingual, int n_vocab) {
+    std::vector<int32_t> result;
+    if (is_multilingual) {
+        result.assign(std::begin(k_whisper_multilingual_non_speech_tokens),
+                      std::end(k_whisper_multilingual_non_speech_tokens));
+    } else {
+        result.assign(std::begin(k_whisper_english_non_speech_tokens), std::end(k_whisper_english_non_speech_tokens));
+    }
+
+    const WhisperSpecials sp = compute_specials(n_vocab);
+    result.insert(result.end(), { sp.sot, sp.translate, sp.transcribe, sp.solm, sp.prev, sp.nosp });
+    return result;
+}
+
 transcribe_status load_from_bin(const char *                                path,
                                 const struct transcribe_model_load_params * params,
                                 struct transcribe_model **                  out_model) {

src/arch/whisper/bin_load.h

@@ -12,8 +12,17 @@
 
 #include "transcribe.h"
 
+#include <cstdint>
+#include <vector>
+
 namespace transcribe::whisper {
 
+// Synthesize the generation-time suppression list omitted by the legacy
+// format. English-only and multilingual .bin files carry different tokenizers,
+// so their non-speech token ids differ. Special-token ids additionally shift
+// with the multilingual language count.
+std::vector<int32_t> synthesize_bin_suppress_tokens(bool is_multilingual, int n_vocab);
+
 // Load a legacy whisper.cpp `.bin` (single magic 0x67676d6c) and
 // produce a populated transcribe_model* compatible with the rest of
 // the Whisper runtime.

tests/CMakeLists.txt

@@ -1216,6 +1216,23 @@ transcribe_apply_warnings(transcribe_tokenizer_decode_only_unit)
 add_test(NAME transcribe_tokenizer_decode_only_unit
     COMMAND transcribe_tokenizer_decode_only_unit)
 
+# -----------------------------------------------------------------------------
+# Whisper .bin synthesized suppression metadata
+# -----------------------------------------------------------------------------
+
+add_executable(transcribe_whisper_bin_suppress_unit
+    whisper_bin_suppress_unit.cpp)
+
+target_link_libraries(transcribe_whisper_bin_suppress_unit PRIVATE transcribe)
+
+target_include_directories(transcribe_whisper_bin_suppress_unit PRIVATE
+    ${CMAKE_SOURCE_DIR}/src)
+
+transcribe_apply_warnings(transcribe_whisper_bin_suppress_unit)
+
+add_test(NAME transcribe_whisper_bin_suppress_unit
+    COMMAND transcribe_whisper_bin_suppress_unit)
+
 # -----------------------------------------------------------------------------
 # Whisper .bin parser unit test
 # -----------------------------------------------------------------------------

tests/whisper_bin_e2e_smoke.cpp

@@ -19,6 +19,8 @@
 //   - caps.languages == ["en"] and caps.n_languages == 1
 //   - JFK transcribes without language hint (auto-detect short-
 //     circuits to "en" since the model has no language tokens)
+//   - affected English pieces such as " computers" and " graduate" remain
+//     available during whole-earth.wav transcription
 //   - --language en accepted; --language de rejected with a non-OK
 //     status
 //   - translate task rejected (no <|translate|> token)
@@ -298,8 +300,9 @@ void test_multilingual(const char * model_path) {
 
 void test_english_only(const char * model_path) {
     std::vector<float> jfk;
-    if (!load_wav("jfk.wav", jfk)) {
-        std::fprintf(stderr, "FAIL: could not load jfk.wav\n");
+    std::vector<float> whole_earth;
+    if (!load_wav("jfk.wav", jfk) || !load_wav("whole-earth.wav", whole_earth)) {
+        std::fprintf(stderr, "FAIL: could not load English sample wavs\n");
         ++g_failures;
         return;
     }
@@ -361,6 +364,19 @@ void test_english_only(const char * model_path) {
         CHECK(contains(transcribe_full_text(ctx), "country"));
     }
 
+    // Regression for English .bin suppression metadata. The multilingual
+    // suppression ids map to ordinary English pieces including " computers"
+    // and " graduate", producing "computer" / "graduates" on this sample.
+    {
+        transcribe_run_params rp;
+        transcribe_run_params_init(&rp);
+        rp.language = "en";
+        st          = transcribe_run(ctx, whole_earth.data(), static_cast<int>(whole_earth.size()), &rp);
+        CHECK(st == TRANSCRIBE_OK);
+        CHECK(contains(transcribe_full_text(ctx), "personal computers"));
+        CHECK(contains(transcribe_full_text(ctx), "graduate to begin anew"));
+    }
+
     // Non-English language — rejected with a non-OK status.
     {
         transcribe_run_params rp;

tests/whisper_bin_suppress_unit.cpp

@@ -0,0 +1,96 @@
+// Regression coverage for legacy Whisper .bin suppression metadata.
+// The format omits generation_config, so the adapter must choose ids for the
+// tokenizer family and compute control-token shifts from the vocabulary size.
+
+#include "arch/whisper/bin_load.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <vector>
+
+namespace {
+
+int g_failures = 0;
+
+#define CHECK(cond)                                                              \
+    do {                                                                         \
+        if (!(cond)) {                                                           \
+            std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+            ++g_failures;                                                        \
+        }                                                                        \
+    } while (0)
+
+const int32_t k_english_expected[] = {
+    1,     2,     7,     8,     9,     10,    14,    25,    26,    27,    28,    29,    31,    58,    59,
+    60,    61,    62,    63,    90,    91,    92,    93,    357,   366,   438,   532,   685,   705,   796,
+    930,   1058,  1220,  1267,  1279,  1303,  1343,  1377,  1391,  1635,  1782,  1875,  2162,  2361,  2488,
+    3467,  4008,  4211,  4600,  4808,  5299,  5855,  6329,  7203,  9609,  9959,  10563, 10786, 11420, 11709,
+    11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007,
+    30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50357, 50358, 50359, 50360, 50361,
+};
+
+const int32_t k_multilingual_expected[] = {
+    1,     2,     7,     8,     9,     10,    14,    25,    26,    27,    28,    29,    31,    58,    59,
+    60,    61,    62,    63,    90,    91,    92,    93,    359,   503,   522,   542,   873,   893,   902,
+    918,   922,   931,   1350,  1853,  1982,  2460,  2627,  3246,  3253,  3268,  3536,  3846,  3961,  4183,
+    4667,  6585,  6647,  7273,  9061,  9383,  10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635,
+    15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650,
+    32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50358, 50359, 50360, 50361, 50362,
+};
+
+template <size_t N> void check_exact(const std::vector<int32_t> & actual, const int32_t (&expected)[N]) {
+    CHECK(actual.size() == N);
+    if (actual.size() != N) {
+        return;
+    }
+    for (size_t i = 0; i < N; ++i) {
+        if (actual[i] != expected[i]) {
+            std::fprintf(stderr, "FAIL suppression id %zu: got %d, expected %d\n", i, actual[i], expected[i]);
+            ++g_failures;
+        }
+    }
+}
+
+void test_english() {
+    const std::vector<int32_t> actual = transcribe::whisper::synthesize_bin_suppress_tokens(false, 51864);
+    check_exact(actual, k_english_expected);
+
+    // These ids are ordinary English word pieces under the multilingual list.
+    CHECK(std::find(actual.begin(), actual.end(), 922) == actual.end());   // " good"
+    CHECK(std::find(actual.begin(), actual.end(), 2627) == actual.end());  // " became"
+    CHECK(std::find(actual.begin(), actual.end(), 357) != actual.end());
+}
+
+void test_multilingual() {
+    const std::vector<int32_t> actual = transcribe::whisper::synthesize_bin_suppress_tokens(true, 51865);
+    check_exact(actual, k_multilingual_expected);
+}
+
+void test_large_v3_special_shift() {
+    const std::vector<int32_t> actual = transcribe::whisper::synthesize_bin_suppress_tokens(true, 51866);
+    CHECK(actual.size() == 88);
+
+    const int32_t expected_tail[] = { 50258, 50359, 50360, 50361, 50362, 50363 };
+    CHECK(actual.size() >= 6);
+    if (actual.size() >= 6) {
+        for (size_t i = 0; i < 6; ++i) {
+            CHECK(actual[actual.size() - 6 + i] == expected_tail[i]);
+        }
+    }
+}
+
+}  // namespace
+
+int main() {
+    test_english();
+    test_multilingual();
+    test_large_v3_special_shift();
+
+    if (g_failures != 0) {
+        std::fprintf(stderr, "FAILED: %d check(s)\n", g_failures);
+        return 1;
+    }
+    std::fprintf(stderr, "OK\n");
+    return 0;
+}