diff --git a/src/arch/whisper/bin_load.cpp b/src/arch/whisper/bin_load.cpp index a7ac6da1..1e1221e8 100644 --- a/src/arch/whisper/bin_load.cpp +++ b/src/arch/whisper/bin_load.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -118,20 +119,28 @@ WhisperSpecials compute_specials(int n_vocab) { return s; } -// HF generation_config.suppress_tokens for whisper (~88 ids never emitted in -// transcripts). Hardcoded here because the .bin doesn't store it; source is HF -// generation_config.json (identical across variants; suppressing ids absent in -// the smaller .en vocab is a no-op). -const int32_t k_whisper_suppress_tokens[] = { - 1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, - 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, - 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, - 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, - 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, - 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50358, 50359, 50360, 50361, 50362, +// OpenAI Whisper's tokenizer-derived non-speech ids, excluding the six +// control tokens appended by synthesize_bin_suppress_tokens(). The legacy +// format stores the vocabulary but not generation_config; its two tokenizer +// families assign different meanings to most ids above the shared punctuation +// prefix. +const int32_t k_whisper_english_non_speech_tokens[] = { + 1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, + 59, 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, + 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, + 2162, 2361, 2488, 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, + 10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, + 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146, +}; + +const int32_t k_whisper_multilingual_non_speech_tokens[] = { + 1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, + 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, + 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, + 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, + 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, + 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254, }; -constexpr int k_whisper_n_suppress = - static_cast(sizeof(k_whisper_suppress_tokens) / sizeof(k_whisper_suppress_tokens[0])); // Tensor rename rules: legacy whisper.cpp name -> canonical transcribe.cpp // name. collapse_to_1d drops the size-1 dim whisper.cpp uses for conv biases. @@ -259,12 +268,8 @@ void fill_whisper_hparams(const transcribe::bin_loader::WhisperBinModel & bm, Wh hp.translate_token_id = bm.is_multilingual ? sp.translate : -1; hp.prev_sot_token_id = sp.prev; - // Suppression list — same across all multilingual whisper.cpp .bin - // variants. For .en, suppressing ids that don't exist in the - // smaller vocab is a no-op; the runtime check `id < vocab_size` - // gates application. - hp.suppress_tokens.assign(k_whisper_suppress_tokens, k_whisper_suppress_tokens + k_whisper_n_suppress); - hp.begin_suppress_tokens.assign({ 220, sp.eot }); + hp.suppress_tokens = synthesize_bin_suppress_tokens(bm.is_multilingual, bm.hp.n_vocab); + hp.begin_suppress_tokens = { 220, sp.eot }; // Frontend (fixed across whisper variants — no .bin field carries // these so we use the canonical preprocessor_config.json values). @@ -528,6 +533,20 @@ transcribe_status install_tokenizer(WhisperModel & m, const transcribe::bin_load } // namespace +std::vector synthesize_bin_suppress_tokens(bool is_multilingual, int n_vocab) { + std::vector result; + if (is_multilingual) { + result.assign(std::begin(k_whisper_multilingual_non_speech_tokens), + std::end(k_whisper_multilingual_non_speech_tokens)); + } else { + result.assign(std::begin(k_whisper_english_non_speech_tokens), std::end(k_whisper_english_non_speech_tokens)); + } + + const WhisperSpecials sp = compute_specials(n_vocab); + result.insert(result.end(), { sp.sot, sp.translate, sp.transcribe, sp.solm, sp.prev, sp.nosp }); + return result; +} + transcribe_status load_from_bin(const char * path, const struct transcribe_model_load_params * params, struct transcribe_model ** out_model) { diff --git a/src/arch/whisper/bin_load.h b/src/arch/whisper/bin_load.h index a08573fc..46d93bee 100644 --- a/src/arch/whisper/bin_load.h +++ b/src/arch/whisper/bin_load.h @@ -12,8 +12,17 @@ #include "transcribe.h" +#include +#include + namespace transcribe::whisper { +// Synthesize the generation-time suppression list omitted by the legacy +// format. English-only and multilingual .bin files carry different tokenizers, +// so their non-speech token ids differ. Special-token ids additionally shift +// with the multilingual language count. +std::vector synthesize_bin_suppress_tokens(bool is_multilingual, int n_vocab); + // Load a legacy whisper.cpp `.bin` (single magic 0x67676d6c) and // produce a populated transcribe_model* compatible with the rest of // the Whisper runtime. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 92e9a0d2..a8dfdb07 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1216,6 +1216,23 @@ transcribe_apply_warnings(transcribe_tokenizer_decode_only_unit) add_test(NAME transcribe_tokenizer_decode_only_unit COMMAND transcribe_tokenizer_decode_only_unit) +# ----------------------------------------------------------------------------- +# Whisper .bin synthesized suppression metadata +# ----------------------------------------------------------------------------- + +add_executable(transcribe_whisper_bin_suppress_unit + whisper_bin_suppress_unit.cpp) + +target_link_libraries(transcribe_whisper_bin_suppress_unit PRIVATE transcribe) + +target_include_directories(transcribe_whisper_bin_suppress_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_whisper_bin_suppress_unit) + +add_test(NAME transcribe_whisper_bin_suppress_unit + COMMAND transcribe_whisper_bin_suppress_unit) + # ----------------------------------------------------------------------------- # Whisper .bin parser unit test # ----------------------------------------------------------------------------- diff --git a/tests/whisper_bin_e2e_smoke.cpp b/tests/whisper_bin_e2e_smoke.cpp index 4323fb5b..06cd8c7e 100644 --- a/tests/whisper_bin_e2e_smoke.cpp +++ b/tests/whisper_bin_e2e_smoke.cpp @@ -19,6 +19,8 @@ // - caps.languages == ["en"] and caps.n_languages == 1 // - JFK transcribes without language hint (auto-detect short- // circuits to "en" since the model has no language tokens) +// - affected English pieces such as " computers" and " graduate" remain +// available during whole-earth.wav transcription // - --language en accepted; --language de rejected with a non-OK // status // - translate task rejected (no <|translate|> token) @@ -298,8 +300,9 @@ void test_multilingual(const char * model_path) { void test_english_only(const char * model_path) { std::vector jfk; - if (!load_wav("jfk.wav", jfk)) { - std::fprintf(stderr, "FAIL: could not load jfk.wav\n"); + std::vector whole_earth; + if (!load_wav("jfk.wav", jfk) || !load_wav("whole-earth.wav", whole_earth)) { + std::fprintf(stderr, "FAIL: could not load English sample wavs\n"); ++g_failures; return; } @@ -361,6 +364,19 @@ void test_english_only(const char * model_path) { CHECK(contains(transcribe_full_text(ctx), "country")); } + // Regression for English .bin suppression metadata. The multilingual + // suppression ids map to ordinary English pieces including " computers" + // and " graduate", producing "computer" / "graduates" on this sample. + { + transcribe_run_params rp; + transcribe_run_params_init(&rp); + rp.language = "en"; + st = transcribe_run(ctx, whole_earth.data(), static_cast(whole_earth.size()), &rp); + CHECK(st == TRANSCRIBE_OK); + CHECK(contains(transcribe_full_text(ctx), "personal computers")); + CHECK(contains(transcribe_full_text(ctx), "graduate to begin anew")); + } + // Non-English language — rejected with a non-OK status. { transcribe_run_params rp; diff --git a/tests/whisper_bin_suppress_unit.cpp b/tests/whisper_bin_suppress_unit.cpp new file mode 100644 index 00000000..f440c9c1 --- /dev/null +++ b/tests/whisper_bin_suppress_unit.cpp @@ -0,0 +1,96 @@ +// Regression coverage for legacy Whisper .bin suppression metadata. +// The format omits generation_config, so the adapter must choose ids for the +// tokenizer family and compute control-token shifts from the vocabulary size. + +#include "arch/whisper/bin_load.h" + +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +const int32_t k_english_expected[] = { + 1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, + 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, 705, 796, + 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, 2162, 2361, 2488, + 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, 11420, 11709, + 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007, + 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50357, 50358, 50359, 50360, 50361, +}; + +const int32_t k_multilingual_expected[] = { + 1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, + 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, + 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, + 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, + 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, + 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50358, 50359, 50360, 50361, 50362, +}; + +template void check_exact(const std::vector & actual, const int32_t (&expected)[N]) { + CHECK(actual.size() == N); + if (actual.size() != N) { + return; + } + for (size_t i = 0; i < N; ++i) { + if (actual[i] != expected[i]) { + std::fprintf(stderr, "FAIL suppression id %zu: got %d, expected %d\n", i, actual[i], expected[i]); + ++g_failures; + } + } +} + +void test_english() { + const std::vector actual = transcribe::whisper::synthesize_bin_suppress_tokens(false, 51864); + check_exact(actual, k_english_expected); + + // These ids are ordinary English word pieces under the multilingual list. + CHECK(std::find(actual.begin(), actual.end(), 922) == actual.end()); // " good" + CHECK(std::find(actual.begin(), actual.end(), 2627) == actual.end()); // " became" + CHECK(std::find(actual.begin(), actual.end(), 357) != actual.end()); +} + +void test_multilingual() { + const std::vector actual = transcribe::whisper::synthesize_bin_suppress_tokens(true, 51865); + check_exact(actual, k_multilingual_expected); +} + +void test_large_v3_special_shift() { + const std::vector actual = transcribe::whisper::synthesize_bin_suppress_tokens(true, 51866); + CHECK(actual.size() == 88); + + const int32_t expected_tail[] = { 50258, 50359, 50360, 50361, 50362, 50363 }; + CHECK(actual.size() >= 6); + if (actual.size() >= 6) { + for (size_t i = 0; i < 6; ++i) { + CHECK(actual[actual.size() - 6 + i] == expected_tail[i]); + } + } +} + +} // namespace + +int main() { + test_english(); + test_multilingual(); + test_large_v3_special_shift(); + + if (g_failures != 0) { + std::fprintf(stderr, "FAILED: %d check(s)\n", g_failures); + return 1; + } + std::fprintf(stderr, "OK\n"); + return 0; +}