From 7e02fe36fca81e2eec1b92228b3431cbe2b8943c Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Sat, 17 Oct 2026 16:32:56 +0100 Subject: [PATCH 01/25] IQ FIFO and 12-bit IQ conversion kernels for DEMOD_2400X --- Makefile | 3 +- convert.c | 58 +++++ dsp/generated/benchmark.c | 27 +++ dsp/generated/dispatcher.c | 297 +++++++++++++++++++++++++ dsp/generated/starch.h | 113 +++++++--- dsp/impl/magnitude_power_uc8.generic.c | 35 +++ dsp/impl/magnitude_sc16.generic.c | 29 +++ dsp/impl/magnitude_sc16q11.generic.c | 35 +++ dsp/starchgen.py | 11 +- dump1090.c | 12 +- fifo.c | 15 +- fifo.h | 4 +- 12 files changed, 600 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 0006c9744..b09a75853 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,8 @@ DUMP1090_VERSION ?= unknown CFLAGS ?= -O3 -g DUMP1090_CFLAGS := -std=c11 -fno-common -Wall -Wmissing-declarations -Werror -Wformat-signedness -W DUMP1090_CPPFLAGS := -I. -D_POSIX_C_SOURCE=200112L -DMODES_DUMP1090_VERSION=\"$(DUMP1090_VERSION)\" -DMODES_DUMP1090_VARIANT=\"dump1090-fa\" - +#DUMP1090_CPPFLAGS += -DDEMOD_2400X + LIBS = -lpthread -lm SDR_OBJ = cpu.o sdr.o fifo.o sdr_ifile.o dsp/helpers/tables.o diff --git a/convert.c b/convert.c index 5f9b31111..0e91b7f0f 100644 --- a/convert.c +++ b/convert.c @@ -19,6 +19,61 @@ #include "dump1090.h" +#ifdef DEMOD_2400X + +// +// DEMOD_2400X build: every converter outputs interleaved signed I,Q at +// 12-bit scale (-2048..+2047) into the FIFO buffer (stored as uint16_t), +// instead of magnitudes. The names match the stock converters so that +// init_converter() below is the same for both builds. +// +// The iqdata_conv_* kernels accept NULL for the level/power outputs. +// + +static void convert_uc8(void *iq_data, + uint16_t *mag_data, + unsigned nsamples, + struct converter_state *state, + double *out_mean_level, + double *out_mean_power) +{ + MODES_NOTUSED(state); + + const uc8_t *in = (const uc8_t *) iq_data; + + starch_iqdata_conv_uc8(in, mag_data, nsamples, out_mean_level, out_mean_power); +} + +static void convert_sc16(void *iq_data, + uint16_t *mag_data, + unsigned nsamples, + struct converter_state *state, + double *out_mean_level, + double *out_mean_power) +{ + MODES_NOTUSED(state); + + const sc16_t *in = (const sc16_t *) iq_data; + + starch_iqdata_conv_sc16(in, mag_data, nsamples, out_mean_level, out_mean_power); +} + +static void convert_sc16q11(void *iq_data, + uint16_t *mag_data, + unsigned nsamples, + struct converter_state *state, + double *out_mean_level, + double *out_mean_power) +{ + MODES_NOTUSED(state); + + const sc16_t *in = (const sc16_t *) iq_data; + + starch_iqdata_conv_sc16q11(in, mag_data, nsamples, out_mean_level, out_mean_power); +} + +#else /* !DEMOD_2400X */ + static void convert_uc8(void *iq_data, uint16_t *mag_data, unsigned nsamples, @@ -73,6 +128,8 @@ static void convert_sc16q11(void *iq_data, } } +#endif /* DEMOD_2400X */ + iq_convert_fn init_converter(input_format_t format, double sample_rate, int filter_dc, @@ -103,3 +160,4 @@ void cleanup_converter(struct converter_state *state) { MODES_NOTUSED(state); } + diff --git a/dsp/generated/benchmark.c b/dsp/generated/benchmark.c index deae7840f..ea75f83f7 100644 --- a/dsp/generated/benchmark.c +++ b/dsp/generated/benchmark.c @@ -867,6 +867,9 @@ static void starch_benchmark_usage(const char *argv0) fprintf(stderr, "\nFunctions: " "count_above_u16 " + "iqdata_conv_sc16 " + "iqdata_conv_sc16q11 " + "iqdata_conv_uc8 " "magnitude_power_uc8 " "magnitude_sc16 " "magnitude_sc16q11 " @@ -962,6 +965,24 @@ int main(int argc, char **argv) starch_benchmark_all_count_above_u16(); continue; } + if (!strcmp(argv[i], "iqdata_conv_sc16")) { + specific = 1; + fprintf(stderr, "=== iqdata_conv_sc16 ===\n"); + fprintf(stderr, " (no benchmark support defined)\n"); + continue; + } + if (!strcmp(argv[i], "iqdata_conv_sc16q11")) { + specific = 1; + fprintf(stderr, "=== iqdata_conv_sc16q11 ===\n"); + fprintf(stderr, " (no benchmark support defined)\n"); + continue; + } + if (!strcmp(argv[i], "iqdata_conv_uc8")) { + specific = 1; + fprintf(stderr, "=== iqdata_conv_uc8 ===\n"); + fprintf(stderr, " (no benchmark support defined)\n"); + continue; + } if (!strcmp(argv[i], "magnitude_power_uc8")) { specific = 1; starch_benchmark_all_magnitude_power_uc8(); @@ -994,6 +1015,12 @@ int main(int argc, char **argv) if (!specific) { starch_benchmark_all_count_above_u16(); + fprintf(stderr, "=== iqdata_conv_sc16 ===\n"); + fprintf(stderr, " (no benchmark support defined)\n"); + fprintf(stderr, "=== iqdata_conv_sc16q11 ===\n"); + fprintf(stderr, " (no benchmark support defined)\n"); + fprintf(stderr, "=== iqdata_conv_uc8 ===\n"); + fprintf(stderr, " (no benchmark support defined)\n"); starch_benchmark_all_magnitude_power_uc8(); starch_benchmark_all_magnitude_sc16(); starch_benchmark_all_magnitude_sc16q11(); diff --git a/dsp/generated/dispatcher.c b/dsp/generated/dispatcher.c index 6f8b18423..c4b8d10ba 100644 --- a/dsp/generated/dispatcher.c +++ b/dsp/generated/dispatcher.c @@ -96,6 +96,231 @@ starch_count_above_u16_regentry starch_count_above_u16_registry[] = { { 0, NULL, NULL, NULL, NULL } }; +/* dispatcher / registry for iqdata_conv_sc16 */ + +starch_iqdata_conv_sc16_regentry * starch_iqdata_conv_sc16_select() { + for (starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_registry; + entry->name; + ++entry) + { + if (entry->flavor_supported && !(entry->flavor_supported())) + continue; + return entry; + } + return NULL; +} + +static void starch_iqdata_conv_sc16_dispatch ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ) { + starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_select(); + if (!entry) + abort(); + + starch_iqdata_conv_sc16 = entry->callable; + starch_iqdata_conv_sc16 ( arg0, arg1, arg2, arg3, arg4 ); +} + +starch_iqdata_conv_sc16_ptr starch_iqdata_conv_sc16 = starch_iqdata_conv_sc16_dispatch; + +void starch_iqdata_conv_sc16_set_wisdom (const char * const * received_wisdom) +{ + /* re-rank the registry based on received wisdom */ + starch_iqdata_conv_sc16_regentry *entry; + for (entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { + const char * const *search; + for (search = received_wisdom; *search; ++search) { + if (!strcmp(*search, entry->name)) { + break; + } + } + if (*search) { + /* matches an entry in the wisdom list, order by position in the list */ + entry->rank = search - received_wisdom; + } else { + /* no match, rank after all possible matches, retaining existing order */ + entry->rank = (search - received_wisdom) + (entry - starch_iqdata_conv_sc16_registry); + } + } + + /* re-sort based on the new ranking */ + qsort(starch_iqdata_conv_sc16_registry, entry - starch_iqdata_conv_sc16_registry, sizeof(starch_iqdata_conv_sc16_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_iqdata_conv_sc16 = starch_iqdata_conv_sc16_dispatch; +} + +starch_iqdata_conv_sc16_regentry starch_iqdata_conv_sc16_registry[] = { + +#ifdef STARCH_MIX_AARCH64 + { 0, "arith_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_arith_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, +#endif /* STARCH_MIX_AARCH64 */ + +#ifdef STARCH_MIX_ARM + { 0, "arith_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_arith_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, +#endif /* STARCH_MIX_ARM */ + +#ifdef STARCH_MIX_GENERIC + { 0, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, +#endif /* STARCH_MIX_GENERIC */ + +#ifdef STARCH_MIX_X86 + { 0, "arith_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_arith_x86_avx2, cpu_supports_avx2 }, + { 1, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, +#endif /* STARCH_MIX_X86 */ + { 0, NULL, NULL, NULL, NULL } +}; + +/* dispatcher / registry for iqdata_conv_sc16q11 */ + +starch_iqdata_conv_sc16q11_regentry * starch_iqdata_conv_sc16q11_select() { + for (starch_iqdata_conv_sc16q11_regentry *entry = starch_iqdata_conv_sc16q11_registry; + entry->name; + ++entry) + { + if (entry->flavor_supported && !(entry->flavor_supported())) + continue; + return entry; + } + return NULL; +} + +static void starch_iqdata_conv_sc16q11_dispatch ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ) { + starch_iqdata_conv_sc16q11_regentry *entry = starch_iqdata_conv_sc16q11_select(); + if (!entry) + abort(); + + starch_iqdata_conv_sc16q11 = entry->callable; + starch_iqdata_conv_sc16q11 ( arg0, arg1, arg2, arg3, arg4 ); +} + +starch_iqdata_conv_sc16q11_ptr starch_iqdata_conv_sc16q11 = starch_iqdata_conv_sc16q11_dispatch; + +void starch_iqdata_conv_sc16q11_set_wisdom (const char * const * received_wisdom) +{ + /* re-rank the registry based on received wisdom */ + starch_iqdata_conv_sc16q11_regentry *entry; + for (entry = starch_iqdata_conv_sc16q11_registry; entry->name; ++entry) { + const char * const *search; + for (search = received_wisdom; *search; ++search) { + if (!strcmp(*search, entry->name)) { + break; + } + } + if (*search) { + /* matches an entry in the wisdom list, order by position in the list */ + entry->rank = search - received_wisdom; + } else { + /* no match, rank after all possible matches, retaining existing order */ + entry->rank = (search - received_wisdom) + (entry - starch_iqdata_conv_sc16q11_registry); + } + } + + /* re-sort based on the new ranking */ + qsort(starch_iqdata_conv_sc16q11_registry, entry - starch_iqdata_conv_sc16q11_registry, sizeof(starch_iqdata_conv_sc16q11_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_iqdata_conv_sc16q11 = starch_iqdata_conv_sc16q11_dispatch; +} + +starch_iqdata_conv_sc16q11_regentry starch_iqdata_conv_sc16q11_registry[] = { + +#ifdef STARCH_MIX_AARCH64 + { 0, "arith_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_arith_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, +#endif /* STARCH_MIX_AARCH64 */ + +#ifdef STARCH_MIX_ARM + { 0, "arith_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_arith_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, +#endif /* STARCH_MIX_ARM */ + +#ifdef STARCH_MIX_GENERIC + { 0, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, +#endif /* STARCH_MIX_GENERIC */ + +#ifdef STARCH_MIX_X86 + { 0, "arith_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_arith_x86_avx2, cpu_supports_avx2 }, + { 1, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, +#endif /* STARCH_MIX_X86 */ + { 0, NULL, NULL, NULL, NULL } +}; + +/* dispatcher / registry for iqdata_conv_uc8 */ + +starch_iqdata_conv_uc8_regentry * starch_iqdata_conv_uc8_select() { + for (starch_iqdata_conv_uc8_regentry *entry = starch_iqdata_conv_uc8_registry; + entry->name; + ++entry) + { + if (entry->flavor_supported && !(entry->flavor_supported())) + continue; + return entry; + } + return NULL; +} + +static void starch_iqdata_conv_uc8_dispatch ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ) { + starch_iqdata_conv_uc8_regentry *entry = starch_iqdata_conv_uc8_select(); + if (!entry) + abort(); + + starch_iqdata_conv_uc8 = entry->callable; + starch_iqdata_conv_uc8 ( arg0, arg1, arg2, arg3, arg4 ); +} + +starch_iqdata_conv_uc8_ptr starch_iqdata_conv_uc8 = starch_iqdata_conv_uc8_dispatch; + +void starch_iqdata_conv_uc8_set_wisdom (const char * const * received_wisdom) +{ + /* re-rank the registry based on received wisdom */ + starch_iqdata_conv_uc8_regentry *entry; + for (entry = starch_iqdata_conv_uc8_registry; entry->name; ++entry) { + const char * const *search; + for (search = received_wisdom; *search; ++search) { + if (!strcmp(*search, entry->name)) { + break; + } + } + if (*search) { + /* matches an entry in the wisdom list, order by position in the list */ + entry->rank = search - received_wisdom; + } else { + /* no match, rank after all possible matches, retaining existing order */ + entry->rank = (search - received_wisdom) + (entry - starch_iqdata_conv_uc8_registry); + } + } + + /* re-sort based on the new ranking */ + qsort(starch_iqdata_conv_uc8_registry, entry - starch_iqdata_conv_uc8_registry, sizeof(starch_iqdata_conv_uc8_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_iqdata_conv_uc8 = starch_iqdata_conv_uc8_dispatch; +} + +starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[] = { + +#ifdef STARCH_MIX_AARCH64 + { 0, "arith_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_arith_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, +#endif /* STARCH_MIX_AARCH64 */ + +#ifdef STARCH_MIX_ARM + { 0, "arith_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_arith_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, +#endif /* STARCH_MIX_ARM */ + +#ifdef STARCH_MIX_GENERIC + { 0, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, +#endif /* STARCH_MIX_GENERIC */ + +#ifdef STARCH_MIX_X86 + { 0, "arith_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_arith_x86_avx2, cpu_supports_avx2 }, + { 1, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, +#endif /* STARCH_MIX_X86 */ + { 0, NULL, NULL, NULL, NULL } +}; + /* dispatcher / registry for magnitude_power_uc8 */ starch_magnitude_power_uc8_regentry * starch_magnitude_power_uc8_select() { @@ -563,6 +788,18 @@ int starch_read_wisdom (const char * path) for (starch_count_above_u16_regentry *entry = starch_count_above_u16_registry; entry->name; ++entry) { entry->rank = 0; } + int rank_iqdata_conv_sc16 = 0; + for (starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { + entry->rank = 0; + } + int rank_iqdata_conv_sc16q11 = 0; + for (starch_iqdata_conv_sc16q11_regentry *entry = starch_iqdata_conv_sc16q11_registry; entry->name; ++entry) { + entry->rank = 0; + } + int rank_iqdata_conv_uc8 = 0; + for (starch_iqdata_conv_uc8_regentry *entry = starch_iqdata_conv_uc8_registry; entry->name; ++entry) { + entry->rank = 0; + } int rank_magnitude_power_uc8 = 0; for (starch_magnitude_power_uc8_regentry *entry = starch_magnitude_power_uc8_registry; entry->name; ++entry) { entry->rank = 0; @@ -625,6 +862,33 @@ int starch_read_wisdom (const char * path) } continue; } + if (!strcmp(name, "iqdata_conv_sc16")) { + for (starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { + if (!strcmp(impl, entry->name)) { + entry->rank = ++rank_iqdata_conv_sc16; + break; + } + } + continue; + } + if (!strcmp(name, "iqdata_conv_sc16q11")) { + for (starch_iqdata_conv_sc16q11_regentry *entry = starch_iqdata_conv_sc16q11_registry; entry->name; ++entry) { + if (!strcmp(impl, entry->name)) { + entry->rank = ++rank_iqdata_conv_sc16q11; + break; + } + } + continue; + } + if (!strcmp(name, "iqdata_conv_uc8")) { + for (starch_iqdata_conv_uc8_regentry *entry = starch_iqdata_conv_uc8_registry; entry->name; ++entry) { + if (!strcmp(impl, entry->name)) { + entry->rank = ++rank_iqdata_conv_uc8; + break; + } + } + continue; + } if (!strcmp(name, "magnitude_power_uc8")) { for (starch_magnitude_power_uc8_regentry *entry = starch_magnitude_power_uc8_registry; entry->name; ++entry) { if (!strcmp(impl, entry->name)) { @@ -691,6 +955,39 @@ int starch_read_wisdom (const char * path) /* reset the implementation pointer so the next call will re-select */ starch_count_above_u16 = starch_count_above_u16_dispatch; } + { + starch_iqdata_conv_sc16_regentry *entry; + for (entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { + if (!entry->rank) + entry->rank = ++rank_iqdata_conv_sc16; + } + qsort(starch_iqdata_conv_sc16_registry, entry - starch_iqdata_conv_sc16_registry, sizeof(starch_iqdata_conv_sc16_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_iqdata_conv_sc16 = starch_iqdata_conv_sc16_dispatch; + } + { + starch_iqdata_conv_sc16q11_regentry *entry; + for (entry = starch_iqdata_conv_sc16q11_registry; entry->name; ++entry) { + if (!entry->rank) + entry->rank = ++rank_iqdata_conv_sc16q11; + } + qsort(starch_iqdata_conv_sc16q11_registry, entry - starch_iqdata_conv_sc16q11_registry, sizeof(starch_iqdata_conv_sc16q11_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_iqdata_conv_sc16q11 = starch_iqdata_conv_sc16q11_dispatch; + } + { + starch_iqdata_conv_uc8_regentry *entry; + for (entry = starch_iqdata_conv_uc8_registry; entry->name; ++entry) { + if (!entry->rank) + entry->rank = ++rank_iqdata_conv_uc8; + } + qsort(starch_iqdata_conv_uc8_registry, entry - starch_iqdata_conv_uc8_registry, sizeof(starch_iqdata_conv_uc8_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_iqdata_conv_uc8 = starch_iqdata_conv_uc8_dispatch; + } { starch_magnitude_power_uc8_regentry *entry; for (entry = starch_magnitude_power_uc8_registry; entry->name; ++entry) { diff --git a/dsp/generated/starch.h b/dsp/generated/starch.h index ceae4c2e9..fa2db8dbd 100644 --- a/dsp/generated/starch.h +++ b/dsp/generated/starch.h @@ -138,104 +138,161 @@ extern starch_count_above_u16_regentry starch_count_above_u16_registry[]; starch_count_above_u16_regentry * starch_count_above_u16_select(); void starch_count_above_u16_set_wisdom( const char * const * received_wisdom ); +typedef void (* starch_iqdata_conv_uc8_ptr) ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +extern starch_iqdata_conv_uc8_ptr starch_iqdata_conv_uc8; + +typedef struct { + int rank; + const char *name; + const char *flavor; + starch_iqdata_conv_uc8_ptr callable; + int (*flavor_supported)(); +} starch_iqdata_conv_uc8_regentry; + +extern starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[]; +starch_iqdata_conv_uc8_regentry * starch_iqdata_conv_uc8_select(); +void starch_iqdata_conv_uc8_set_wisdom( const char * const * received_wisdom ); + +typedef void (* starch_iqdata_conv_sc16_ptr) ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +extern starch_iqdata_conv_sc16_ptr starch_iqdata_conv_sc16; + +typedef struct { + int rank; + const char *name; + const char *flavor; + starch_iqdata_conv_sc16_ptr callable; + int (*flavor_supported)(); +} starch_iqdata_conv_sc16_regentry; + +extern starch_iqdata_conv_sc16_regentry starch_iqdata_conv_sc16_registry[]; +starch_iqdata_conv_sc16_regentry * starch_iqdata_conv_sc16_select(); +void starch_iqdata_conv_sc16_set_wisdom( const char * const * received_wisdom ); + +typedef void (* starch_iqdata_conv_sc16q11_ptr) ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +extern starch_iqdata_conv_sc16q11_ptr starch_iqdata_conv_sc16q11; + +typedef struct { + int rank; + const char *name; + const char *flavor; + starch_iqdata_conv_sc16q11_ptr callable; + int (*flavor_supported)(); +} starch_iqdata_conv_sc16q11_regentry; + +extern starch_iqdata_conv_sc16q11_regentry starch_iqdata_conv_sc16q11_registry[]; +starch_iqdata_conv_sc16q11_regentry * starch_iqdata_conv_sc16q11_select(); +void starch_iqdata_conv_sc16q11_set_wisdom( const char * const * received_wisdom ); + /* flavors and prototypes */ #ifdef STARCH_FLAVOR_ARMV7A_NEON_VFPV4 int cpu_supports_armv7_neon_vfpv4 (void); -void starch_count_above_u16_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); +void starch_magnitude_power_uc8_twopass_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_unroll_4_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_iqdata_conv_uc8_arith_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_sc16_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_magnitude_sc16_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16_arith_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16q11_arith_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); +void starch_count_above_u16_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); void starch_magnitude_uc8_lookup_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_lookup_unroll_4_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_exact_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_power_uc8_twopass_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_unroll_4_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_sc16_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_sc16_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_mean_power_u16_neon_float_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_magnitude_uc8_neon_vrsqrte_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_sc16q11_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_count_above_u16_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); -void starch_mean_power_u16_neon_float_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_magnitude_power_uc8_neon_vrsqrte_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_magnitude_sc16q11_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); #endif /* STARCH_FLAVOR_ARMV7A_NEON_VFPV4 */ int starch_read_wisdom (const char * path); #ifdef STARCH_FLAVOR_ARMV8_NEON_SIMD int cpu_supports_armv8_simd (void); -void starch_count_above_u16_generic_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); +void starch_magnitude_power_uc8_twopass_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_unroll_4_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_iqdata_conv_uc8_arith_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_sc16_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_magnitude_sc16_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16_arith_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16q11_arith_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); +void starch_count_above_u16_generic_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); void starch_magnitude_uc8_lookup_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_lookup_unroll_4_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_exact_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_power_uc8_twopass_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_unroll_4_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_sc16_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_sc16_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_mean_power_u16_neon_float_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_magnitude_uc8_neon_vrsqrte_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_sc16q11_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_count_above_u16_neon_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); -void starch_mean_power_u16_neon_float_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_magnitude_power_uc8_neon_vrsqrte_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_magnitude_sc16q11_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); #endif /* STARCH_FLAVOR_ARMV8_NEON_SIMD */ int starch_read_wisdom (const char * path); #ifdef STARCH_FLAVOR_GENERIC -void starch_count_above_u16_generic_generic ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); +void starch_magnitude_power_uc8_twopass_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_unroll_4_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_iqdata_conv_uc8_arith_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_sc16_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_magnitude_sc16_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16_arith_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16q11_arith_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_generic ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_generic ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_generic ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); +void starch_count_above_u16_generic_generic ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); void starch_magnitude_uc8_lookup_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_lookup_unroll_4_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_exact_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_power_uc8_twopass_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_unroll_4_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_sc16_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_sc16_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); #endif /* STARCH_FLAVOR_GENERIC */ int starch_read_wisdom (const char * path); #ifdef STARCH_FLAVOR_X86_AVX2 int cpu_supports_avx2 (void); -void starch_count_above_u16_generic_x86_avx2 ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); +void starch_magnitude_power_uc8_twopass_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_power_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_iqdata_conv_uc8_arith_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +void starch_magnitude_sc16_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_magnitude_sc16_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16_arith_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +void starch_iqdata_conv_sc16q11_arith_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_x86_avx2 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_x86_avx2 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_x86_avx2 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); +void starch_count_above_u16_generic_x86_avx2 ( const uint16_t * arg0, unsigned arg1, uint16_t arg2, unsigned * arg3 ); void starch_magnitude_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_exact_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_power_uc8_twopass_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_power_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_magnitude_sc16_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_magnitude_sc16_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); #endif /* STARCH_FLAVOR_X86_AVX2 */ int starch_read_wisdom (const char * path); diff --git a/dsp/impl/magnitude_power_uc8.generic.c b/dsp/impl/magnitude_power_uc8.generic.c index a6f4c66df..76908bd1f 100644 --- a/dsp/impl/magnitude_power_uc8.generic.c +++ b/dsp/impl/magnitude_power_uc8.generic.c @@ -83,3 +83,38 @@ void STARCH_IMPL(magnitude_power_uc8, lookup_unroll_4) (const uc8_t * restrict i *out_level = sum_level / 65536.0 / len; *out_power = sum_power / 65536.0 / 65536.0 / len; } + +/* UC8 (8-bit offset binary, centre 127.5) -> 12-bit signed interleaved I,Q, + * -2040..+2040, plus mean level and power normalised to 12-bit full scale */ + +void STARCH_IMPL(iqdata_conv_uc8, arith) (const uc8_t * restrict in, uint16_t * restrict out, unsigned len, double *out_level, double *out_power) +{ + // 8-bit offset binary (centre 127.5) -> 12-bit signed, -2040..+2040, interleaved I,Q + for (unsigned n = 0; n < len; ++n) { + out[2 * n] = (int16_t) ((2 * in[n].I - 255) * 8); + out[2 * n + 1] = (int16_t) ((2 * in[n].Q - 255) * 8); + } + + // Same normalisation as the stock stats: magnitude scaled so full scale (2048) = 1.0 + double sum_level = 0; + uint64_t sum_power = 0; + + for (unsigned n = 0; n < len; ++n) { + int32_t I = (2 * in[n].I - 255) * 8; + int32_t Q = (2 * in[n].Q - 255) * 8; + + out[2 * n] = (int16_t) I; + out[2 * n + 1] = (int16_t) Q; + + uint32_t magsq = (uint32_t) (I * I + Q * Q); + sum_power += magsq; + sum_level += sqrtf((float) magsq); + } + + if (out_level != NULL) + *out_level = sum_level / 2048.0 / len; + + if (out_power != NULL) + *out_power = (double) sum_power / 2048.0 / 2048.0 / len; +} + diff --git a/dsp/impl/magnitude_sc16.generic.c b/dsp/impl/magnitude_sc16.generic.c index cbf14cdd5..d9314868c 100644 --- a/dsp/impl/magnitude_sc16.generic.c +++ b/dsp/impl/magnitude_sc16.generic.c @@ -37,3 +37,32 @@ void STARCH_IMPL(magnitude_sc16, exact_float) (const sc16_t * restrict in, uint1 in += 1; } } + +/* SC16 (full-scale 16-bit, little-endian) -> 12-bit signed interleaved I,Q, + * plus mean level and power normalised to 12-bit full scale */ + +void STARCH_IMPL(iqdata_conv_sc16, arith) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, double *out_level, double *out_power) +{ + int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer + + double sum_level = 0; + uint64_t sum_power = 0; + + for (unsigned n = 0; n < len; ++n) { + int32_t I = (int16_t) le16toh(in[n].I) >> 4; + int32_t Q = (int16_t) le16toh(in[n].Q) >> 4; + + out[2 * n] = (int16_t) I; + out[2 * n + 1] = (int16_t) Q; + + uint32_t magsq = (uint32_t) (I * I + Q * Q); + sum_power += magsq; + sum_level += sqrtf((float) magsq); + } + + if (out_level != NULL) + *out_level = sum_level / 2048.0 / len; + + if (out_power != NULL) + *out_power = (double) sum_power / 2048.0 / 2048.0 / len; +} diff --git a/dsp/impl/magnitude_sc16q11.generic.c b/dsp/impl/magnitude_sc16q11.generic.c index ca99bd950..924b5c6e5 100644 --- a/dsp/impl/magnitude_sc16q11.generic.c +++ b/dsp/impl/magnitude_sc16q11.generic.c @@ -70,3 +70,38 @@ void STARCH_IMPL(magnitude_sc16q11, 12bit_table) (const sc16_t * restrict in, ui out += 1; } } + + +/* SC16Q11 (already 12-bit, little-endian) -> 12-bit signed interleaved I,Q, + * plus mean level and power normalised to 12-bit full scale */ + +void STARCH_IMPL(iqdata_conv_sc16q11, arith) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, double *out_level, double *out_power) +{ + int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer + + double sum_level = 0; + uint64_t sum_power = 0; + + for (unsigned n = 0; n < len; ++n) { + int32_t I = (int16_t) le16toh(in[n].I); + int32_t Q = (int16_t) le16toh(in[n].Q); + + // keep to 12-bit like the stock 11-bit table path + if (I > 2047) I = 2047; else if (I < -2048) I = -2048; + if (Q > 2047) Q = 2047; else if (Q < -2048) Q = -2048; + + out[2 * n] = (int16_t) I; + out[2 * n + 1] = (int16_t) Q; + + uint32_t magsq = (uint32_t) (I * I + Q * Q); + sum_power += magsq; + sum_level += sqrtf((float) magsq); + } + + if (out_level != NULL) + *out_level = sum_level / 2048.0 / len; + + if (out_power != NULL) + *out_power = (double) sum_power / 2048.0 / 2048.0 / len; +} + diff --git a/dsp/starchgen.py b/dsp/starchgen.py index 99b3a5548..9b7018f52 100755 --- a/dsp/starchgen.py +++ b/dsp/starchgen.py @@ -33,7 +33,16 @@ gen.add_function( name = 'count_above_u16', argtypes = ['const uint16_t *', 'unsigned', 'uint16_t', 'unsigned *']) - +gen.add_function( + name = 'iqdata_conv_uc8', + argtypes = ['const uc8_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) +gen.add_function( + name = 'iqdata_conv_sc16', + argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) +gen.add_function( + name = 'iqdata_conv_sc16q11', + argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) + gen.add_flavor( name = 'generic', description = 'Generic build, default compiler options', diff --git a/dump1090.c b/dump1090.c index bcc9a9e87..f01c34ebd 100644 --- a/dump1090.c +++ b/dump1090.c @@ -119,7 +119,11 @@ static void modesInitConfig(void) { Modes.json_location_accuracy = 1; Modes.maxRange = 1852 * 300; // 300NM default max range Modes.mode_ac_auto = 1; - + #ifdef DEMOD_2400X + Modes.mode_ac_auto = 0; + #else + Modes.mode_ac_auto = 1; + #endif Modes.net_heartbeat_interval = MODES_NET_HEARTBEAT_INTERVAL; Modes.net_output_flush_size = 1300; Modes.net_output_flush_interval = 500; @@ -164,7 +168,11 @@ static void modesInit(void) { exit(1); } - if (!fifo_create(MODES_MAG_BUFFERS, MODES_MAG_BUF_SAMPLES + Modes.trailing_samples, Modes.trailing_samples)) { +#ifdef DEMOD_2400X + if (!fifo_create(MODES_MAG_BUFFERS, MODES_MAG_BUF_SAMPLES + Modes.trailing_samples, 1, Modes.trailing_samples)) { +#else + if (!fifo_create(MODES_MAG_BUFFERS, MODES_MAG_BUF_SAMPLES + Modes.trailing_samples, 0, Modes.trailing_samples)) { +#endif fprintf(stderr, "Out of memory allocating FIFO\n"); exit(1); } diff --git a/fifo.c b/fifo.c index 73ddd5b14..efa4094e2 100644 --- a/fifo.c +++ b/fifo.c @@ -40,9 +40,10 @@ static unsigned overlap_length; // desired overlap size in samples (size of static uint16_t *overlap_buffer; // buffer used to save overlapping data // Create the queue structures. Not threadsafe. -bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned overlap) +bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned datatype_iq, unsigned overlap) { - if (!(overlap_buffer = calloc(overlap, sizeof(overlap_buffer[0])))) + unsigned esize = datatype_iq?2:1; + if (!(overlap_buffer = calloc(overlap, sizeof(overlap_buffer[0])*esize))) goto nomem; overlap_length = overlap; @@ -53,13 +54,14 @@ bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned overlap) goto nomem; } - if (!(newbuf->data = calloc(buffer_size, sizeof(newbuf->data[0])))) { + if (!(newbuf->data = calloc(buffer_size, sizeof(newbuf->data[0])*esize))) { free(newbuf); goto nomem; } newbuf->totalLength = buffer_size; newbuf->next = fifo_freelist; + newbuf->datatype_iq = datatype_iq; fifo_freelist = newbuf; } @@ -176,6 +178,7 @@ void fifo_enqueue(struct mag_buf *buf) assert(buf->validLength >= overlap_length); pthread_mutex_lock(&fifo_mutex); + unsigned esize = buf->datatype_iq?2:1; if (fifo_halted) { // Shutting down, just return the buffer to the freelist. @@ -187,13 +190,13 @@ void fifo_enqueue(struct mag_buf *buf) // Populate the overlap region if (buf->flags & MAGBUF_DISCONTINUOUS) { // This buffer is discontinuous to the previous, so the overlap region is not valid; zero it out - memset(buf->data, 0, overlap_length * sizeof(buf->data[0])); + memset(buf->data, 0, overlap_length * esize * sizeof(buf->data[0])); } else { - memcpy(buf->data, overlap_buffer, overlap_length * sizeof(buf->data[0])); + memcpy(buf->data, overlap_buffer, overlap_length * esize * sizeof(buf->data[0])); } // Save the tail of the buffer for next time - memcpy(overlap_buffer, &buf->data[buf->validLength - overlap_length], overlap_length * sizeof(overlap_buffer[0])); + memcpy(overlap_buffer, &buf->data[(buf->validLength - overlap_length) * esize], overlap_length * esize * sizeof(overlap_buffer[0])); // enqueue and tell the main thread buf->next = NULL; diff --git a/fifo.h b/fifo.h index e7003c1eb..1dcac464b 100644 --- a/fifo.h +++ b/fifo.h @@ -57,6 +57,7 @@ struct mag_buf { unsigned validLength; // Number of valid samples in "data", including overlap samples unsigned overlap; // Number of leading overlap samples at the start of "data"; // also the number of trailing samples that will be preserved for next time + unsigned datatype_iq; // 0 is power samples, 1 = IQ samples uint64_t sampleTimestamp; // Clock timestamp of the start of this block, 12MHz clock uint64_t sysTimestamp; // Estimated system time at start of block @@ -73,8 +74,9 @@ struct mag_buf { // // buffer_count - the number of buffers to preallocate // buffer_size - the size of each magnitude buffer, in samples, including overlap +// data_type - 0 is power samples, 1 = IQ samples // overlap - the number of samples to overlap between adjacent buffers -bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned overlap); +bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned datatype_iq, unsigned overlap); // Destroy the fifo structures allocated in magbuf_fifo_create. Not threadsafe; ensure all FIFO users // are done before calling. From fc57da1b96c9cedf92e80712f17373c64b795aed Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Sat, 17 Oct 2026 19:44:17 +0100 Subject: [PATCH 02/25] added dsp part of demod --- Makefile | 2 +- demod_2400.c | 36 ++++ demod_2400x.h | 26 +++ demod_dsp.c | 549 ++++++++++++++++++++++++++++++++++++++++++++++++++ dump1090.c | 6 +- fifo.c | 10 +- fifo.h | 6 +- 7 files changed, 621 insertions(+), 14 deletions(-) create mode 100644 demod_2400x.h create mode 100644 demod_dsp.c diff --git a/Makefile b/Makefile index b09a75853..f04942bd8 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ showconfig: %.o: %.c *.h $(CC) $(ALL_CCFLAGS) -c $< -o $@ -dump1090: dump1090.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o net_io.o crc.o demod_2400.o stats.o cpr.o icao_filter.o track.o util.o convert.o ais_charset.o adaptive.o $(SDR_OBJ) $(COMPAT) $(STARCH_OBJS) +dump1090: dump1090.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o net_io.o crc.o demod_2400.o demod_dsp.o stats.o cpr.o icao_filter.o track.o util.o convert.o ais_charset.o adaptive.o $(SDR_OBJ) $(COMPAT) $(STARCH_OBJS) $(CC) -g -o $@ $^ $(LDFLAGS) $(LIBS) $(LIBS_SDR) $(LIBS_CURSES) view1090: view1090.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o net_io.o crc.o stats.o cpr.o icao_filter.o track.o util.o ais_charset.o sdr_stub.o $(COMPAT) diff --git a/demod_2400.c b/demod_2400.c index d6d90e036..5ae7efe71 100644 --- a/demod_2400.c +++ b/demod_2400.c @@ -536,10 +536,46 @@ static void draw_modeac(uint16_t *m, unsigned modeac, unsigned f1_clock, unsigne // // one 2.4MHz sample = 25 cycles +#ifdef DEMOD_2400X + +// Convert the IQ FIFO buffer (interleaved 12-bit I/Q = SC16Q11 layout) into +// magnitudes for the stock demodulators. Returns NULL if out of memory. +static uint16_t *getMagnitude(struct mag_buf *mag) +{ + static uint16_t *mptr = NULL; + static unsigned msize = 0; + + if (mag->totalLength > msize) { + + uint16_t *nptr; + if ((nptr = realloc(mptr, mag->totalLength * sizeof(*nptr))) == NULL) { + + free(mptr); + mptr = NULL; + msize = 0; + return NULL; + } + msize = mag->totalLength; + mptr = nptr; + } + + // whole buffer, including the leading overlap the demodulator looks back into + starch_magnitude_sc16q11((const sc16_t *) mag->data, mptr, mag->validLength); + + return mptr; +} +#endif + + void demodulate2400AC(struct mag_buf *mag) { struct modesMessage mm; +#ifdef DEMOD_2400X + uint16_t *m; + if ((m = getMagnitude(mag)) == NULL) return; +#else uint16_t *m = mag->data; +#endif uint32_t mlen = mag->validLength - mag->overlap; unsigned f1_sample; diff --git a/demod_2400x.h b/demod_2400x.h new file mode 100644 index 000000000..db4adeab6 --- /dev/null +++ b/demod_2400x.h @@ -0,0 +1,26 @@ +#ifndef DEMOD_2400X_H +#define DEMOD_2400X_H + +#include "fifo.h" + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) < (b) ? (b) : (a)) + +#define NU_SAMP(t) (5*((t)-7)) +#define NU_STRIDE(t) ((((t)+5)/6+7)&~7) +#define METRIC_SIZE(t) ((t)-15-12*LONG_FRAME) + +#define LONG_FRAME 112 +#define SHORT_FRAME 56 + +#define PREAMBLE_KK 2 +#define PREAMBLE_MIN_PULSE + +#define OVERLAP_WINDOW \ + (7 + ((15 + 1 + 2*LONG_FRAME + 4) / 5)) + +extern int upsample12(const int16_t * restrict in, int len, int shift, uint16_t * restrict oo); +extern int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm); +extern void demodulate2400x(struct mag_buf *mbuf); + +#endif diff --git a/demod_dsp.c b/demod_dsp.c new file mode 100644 index 000000000..5b1277cea --- /dev/null +++ b/demod_dsp.c @@ -0,0 +1,549 @@ +#include +#include +#include "demod_2400x.h" + +#if defined(__aarch64__) + #define TARGET_NEON + #define TARGET_SIMD +#elif defined(__arm__) && defined(__ARM_NEON) + #define TARGET_NEON +#elif defined(__x86_64__) && defined(__AVX2__) + #define TARGET_AVX2 +#else + #define TARGET_GENERIC +#endif + +static const int8_t coeffs[5][8] = { + { -1, 4, -10, 125, 14, -6, 3, -1 }, // phase=0 sum=128 energy=15984 gd=3.0938 + { -3, 8, -21, 109, 47, -16, 7, -3 }, // phase=1 sum=128 energy=14918 gd=3.2969 + { -4, 10, -23, 81, 81, -23, 10, -4 }, // phase=2 sum=128 energy=14412 gd=3.5000 + { -3, 7, -16, 47, 109, -21, 8, -3 }, // phase=3 sum=128 energy=14918 gd=3.7031 + { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 +}; + + +#ifdef TARGET_NEON +#include + +// Horizontal add +static inline __attribute__((always_inline)) int hsum_s16x8(const int16x8_t v) +{ +#ifdef TARGET_SIMD + return vaddvq_s16(v); +#else + int16x4_t s = vadd_s16(vget_low_s16(v), + vget_high_s16(v)); + s = vpadd_s16(s, s); + s = vpadd_s16(s, s); + return vget_lane_s16(s, 0); +#endif +} + +// Horizontal add +static inline __attribute__((always_inline)) +uint32_t hsum_u32x4(const uint32x4_t v) +{ +#ifdef TARGET_SIMD + return vaddvq_u32(v); +#else + uint32x2_t s = vadd_u32(vget_low_u32(v), + vget_high_u32(v)); + s = vpadd_u32(s, s); + return vget_lane_u32(s, 0); +#endif +} + +// Horizontal add of two vectors at once: returns [sum of a, sum of b] +static inline __attribute__((always_inline)) +int32x2_t hsum2_s32x4(const int32x4_t a, const int32x4_t b) +{ +#ifdef TARGET_SIMD + int32x4_t ab = vpaddq_s32(a, b); // [a0+a1, a2+a3, b0+b1, b2+b3] + return vpadd_s32(vget_low_s32(ab), vget_high_s32(ab)); +#else + return vpadd_s32(vadd_s32(vget_low_s32(a), vget_high_s32(a)), + vadd_s32(vget_low_s32(b), vget_high_s32(b))); +#endif +} + + +static inline __attribute__((always_inline)) unsigned getpow(const int8x8_t cv8, const int16x8x2_t iq, int shift) +{ + const int16x8_t cv = vmovl_s8(cv8); // widen coefficients to 16-bit + int32x4_t si, sq; + + si = vmull_s16(vget_low_s16(iq.val[0]), vget_low_s16(cv)); + si = vmlal_s16(si, vget_high_s16(iq.val[0]), vget_high_s16(cv)); + + sq = vmull_s16(vget_low_s16(iq.val[1]), vget_low_s16(cv)); + sq = vmlal_s16(sq, vget_high_s16(iq.val[1]), vget_high_s16(cv)); + + int32x2_t res = hsum2_s32x4(si, sq); // [resi, resq] + res = vrshr_n_s32(res, 5); // (x+16)>>5 on both + + uint32x2_t sqr = vreinterpret_u32_s32(vmul_s32(res, res)); // [resi^2, resq^2] + uint32_t pwr = vget_lane_u32(sqr, 0) + vget_lane_u32(sqr, 1); + + pwr >>= shift; // Set gain + if (pwr > 65535) pwr = 65535; + + return pwr; +} + +// Upsample +int upsample12(const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) { + + if (len < 8) return 0; + + int8x8_t c0 = vld1_s8(coeffs[0]); + int8x8_t c1 = vld1_s8(coeffs[1]); + int8x8_t c2 = vld1_s8(coeffs[2]); + int8x8_t c3 = vld1_s8(coeffs[3]); + int8x8_t c4 = vld1_s8(coeffs[4]); + + int16x8x2_t iq = vld2q_s16(in); + + int nt = NU_SAMP(len); // Total samples + int stride = NU_STRIDE(nt); // Stride + int stride6 = 6*stride; + int bpos = 0; + + in += 16; + len -= 8; + while (1) { + + // Rearrage output + oo[bpos] = getpow(c0, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c1, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c2, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c3, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c4, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + if (len == 0) break; + len--; + + // Shift + iq.val[0] = vextq_s16(iq.val[0], iq.val[0], 1); + iq.val[1] = vextq_s16(iq.val[1], iq.val[1], 1); + + iq.val[0] = vsetq_lane_s16(in[0], iq.val[0], 7); + iq.val[1] = vsetq_lane_s16(in[1], iq.val[1], 7); + in += 2; + } + + return nt; +} + + +// Locate preamble +int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm) { + + int stride = NU_STRIDE(nt); // Stride + const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; + + uint16x8_t gm1 = vld1q_u16((const uint16_t *)mask_g+0); + uint16x8_t gm2 = vld1q_u16((const uint16_t *)mask_g+8); + + for (int j = 0; j < 6; j++) { // Look at each phase in turn + + const uint16_t * restrict p = &pp[j*stride]; + int plen = (nt + 5 - j) / 6; // Length of each phase + + plen -= 2 * LONG_FRAME; // No point in searching at end + if (plen < 16) continue; + + uint16x8_t p1 = vld1q_u16(p + 0); + uint16x8_t p2 = vld1q_u16(p + 8); + + plen -= 16; + p += 16; + uint8_t * restrict mp = &mm[j]; + + while (1) { + + // Preamble detection + // P G P G, G G G P, G P G G, G G G G + + // Extract gaps and sum together + uint32x4_t g1 = vpaddlq_u16(vandq_u16(p1, gm1)); + uint32x4_t g2 = vpaddlq_u16(vandq_u16(p2, gm2)); + uint32x4_t gs = vaddq_u32(g1, g2); + uint32_t gap = hsum_u32x4(gs); + + // Extract pulses + uint16x8_t dm1 = vbicq_u16(p1, gm1); + uint16x8_t dm2 = vbicq_u16(p2, gm2); + + uint16x4_t d0 = vget_low_u16(dm1); + uint16x4_t d1 = vget_high_u16(dm1); + uint16x4_t d2 = vget_low_u16(dm2); + + uint16x4_t px = vorr_u16(vorr_u16(d0, d1), d2); + +#ifdef PREAMBLE_MIN_PULSE + uint16x4_t m = vpmin_u16(px, px); + m = vpmin_u16(m, m); + unsigned minp = vget_lane_u16(m, 0); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32x2_t s = vpaddl_u16(px); + s = vpadd_u32(s, s); + uint32_t minp = vget_lane_u32(s, 0); +#endif + // Metric + *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); + mp += 6; + + if (plen == 0) break; + plen--; + + unsigned nv = *p++; + p1 = vextq_u16(p1, p2, 1); + p2 = vextq_u16(p2, p2, 1); + p2 = vsetq_lane_u16(nv, p2, 7); + } + } + return METRIC_SIZE(nt); +} + +#endif + + +#ifdef TARGET_GENERIC + +// Get power +static inline unsigned getpow( + const int8_t *cv, + const int16_t *ii, + const int16_t *qq, + int shift) +{ + int32_t si = + cv[0] * ii[0] + + cv[1] * ii[1] + + cv[2] * ii[2] + + cv[3] * ii[3] + + cv[4] * ii[4] + + cv[5] * ii[5] + + cv[6] * ii[6] + + cv[7] * ii[7]; + + int32_t sq = + cv[0] * qq[0] + + cv[1] * qq[1] + + cv[2] * qq[2] + + cv[3] * qq[3] + + cv[4] * qq[4] + + cv[5] * qq[5] + + cv[6] * qq[6] + + cv[7] * qq[7]; + + si = (si+16)>>5; + sq = (sq+16)>>5; + uint32_t pwr = (uint32_t)(si*si) + (uint32_t)(sq*sq); + + pwr >>= shift; // Set gain + if (pwr > 65535) pwr = 65535; + + return pwr; +} + + +// Upsample +int upsample12( + const int16_t * restrict in, + int len, int shift, + uint16_t * restrict oo) +{ + if (len < 8) + return 0; + + int16_t ii[8]; + int16_t qq[8]; + + int nt = NU_SAMP(len); + int stride = NU_STRIDE(nt); + int stride6 = 6 * stride; + int bpos = 0; + + /* Initial 8-sample window */ + for (int i = 0; i < 8; i++) { + + ii[i] = in[0]; + qq[i] = in[1]; + in += 2; + } + len -= 8; + + while (1) { + + oo[bpos] = getpow(coeffs[0], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow(coeffs[1], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow(coeffs[2], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow(coeffs[3], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow(coeffs[4], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + if (len == 0) + break; + + len--; + + /* Shift window left by one sample */ + for (int i = 0; i < 7; i++) { + ii[i] = ii[i + 1]; + qq[i] = qq[i + 1]; + } + + ii[7] = in[0]; + qq[7] = in[1]; + in += 2; + } + + return nt; +} + + +// Locate preamble +int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm) { + + int stride = NU_STRIDE(nt); + + for (int j = 0; j < 6; j++) { + + const uint16_t *restrict p = pp + j * stride; + uint8_t *restrict mp = mm + j; + + int plen = (nt + 5 - j) / 6; + + plen -= 2 * LONG_FRAME; + + for (int i = 0; i < plen - 15; i++) { + +#ifdef PREAMBLE_MIN_PULSE + uint32_t minp = + MIN( + MIN(p[0], p[2]), + MIN(p[7], p[9]) + ); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32_t minp = p[0] + p[2] + p[7] + p[9]; +#endif + uint32_t gap = + p[ 1] + + p[ 3] + + p[ 4] + + p[ 5] + + p[ 6] + + p[ 8] + + p[10] + + p[11] + + p[12] + + p[13] + + p[14] + + p[15]; + + *mp = (256 * PREAMBLE_KK * minp) + / (PREAMBLE_KK * minp + gap + 1); + + p++; + mp += 6; + } + } + + return METRIC_SIZE(nt); + +} +#endif + +#ifdef TARGET_AVX2 +#include + +// Two vectors of 8 x int16: val[0] = I, val[1] = Q (same layout as NEON's int16x8x2_t) +typedef struct { + __m128i val[2]; +} iq16x8x2_t; + +// Horizontal add +static inline __attribute__((always_inline)) int32_t hsum_s32x4(const __m128i v) +{ + __m128i s = _mm_add_epi32(v, _mm_shuffle_epi32(v, _MM_SHUFFLE(1, 0, 3, 2))); + s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(2, 3, 0, 1))); + return _mm_cvtsi128_si32(s); +} + +// Horizontal add +static inline __attribute__((always_inline)) uint32_t hsum_u32x4(const __m128i v) +{ + return (uint32_t) hsum_s32x4(v); +} + +// Load 8 interleaved I/Q samples and deinterleave them (like NEON's vld2q_s16) +static inline __attribute__((always_inline)) iq16x8x2_t load2_s16(const int16_t *in) +{ + const __m128i deint = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + + __m128i a = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) (in + 0)), deint); // I0-3 Q0-3 + __m128i b = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) (in + 8)), deint); // I4-7 Q4-7 + + iq16x8x2_t iq; + iq.val[0] = _mm_unpacklo_epi64(a, b); // I0-7 + iq.val[1] = _mm_unpackhi_epi64(a, b); // Q0-7 + return iq; +} + + +static inline __attribute__((always_inline)) unsigned getpow(const __m128i cv8, const iq16x8x2_t iq, int shift) +{ + const __m128i cv = _mm_cvtepi8_epi16(cv8); // widen coefficients to 16-bit + + // madd: multiply 8 pairs and add neighbours -> 4 x int32 partial sums + int32_t resi = (hsum_s32x4(_mm_madd_epi16(iq.val[0], cv)) + 16) >> 5; + int32_t resq = (hsum_s32x4(_mm_madd_epi16(iq.val[1], cv)) + 16) >> 5; + + uint32_t pwr = (uint32_t)(resi*resi) + (uint32_t)(resq*resq); + + pwr >>= shift; // Set gain + if (pwr > 65535) pwr = 65535; + + return pwr; +} + + +// Upsample +int upsample12(const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) { + + if (len < 8) return 0; + + __m128i c0 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[0]); // 8 x int8 + __m128i c1 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[1]); // 8 x int8 + __m128i c2 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[2]); // 8 x int8 + __m128i c3 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[3]); // 8 x int8 + __m128i c4 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[4]); // 8 x int8 + + iq16x8x2_t iq = load2_s16(in); + + int nt = NU_SAMP(len); // Total samples + int stride = NU_STRIDE(nt); // Stride + int stride6 = 6*stride; + int bpos = 0; + + in += 16; + len -= 8; + while (1) { + + // Rearrage output + oo[bpos] = getpow(c0, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c1, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c2, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c3, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow(c4, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + if (len == 0) break; + len--; + + // Shift + iq.val[0] = _mm_insert_epi16(_mm_srli_si128(iq.val[0], 2), in[0], 7); + iq.val[1] = _mm_insert_epi16(_mm_srli_si128(iq.val[1], 2), in[1], 7); + in += 2; + } + + return nt; +} + + +// Locate preamble +int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm) { + + int stride = NU_STRIDE(nt); // Stride + const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; + + const __m128i gm1 = _mm_loadu_si128((const __m128i *) (mask_g + 0)); + const __m128i gm2 = _mm_loadu_si128((const __m128i *) (mask_g + 8)); + const __m128i zero = _mm_setzero_si128(); + + for (int j = 0; j < 6; j++) { // Look at each phase in turn + + const uint16_t * restrict p = &pp[j*stride]; + int plen = (nt + 5 - j) / 6; // Length of each phase + + plen -= 2 * LONG_FRAME; // No point in searching at end + if (plen < 16) continue; + + __m128i p1 = _mm_loadu_si128((const __m128i *) (p + 0)); + __m128i p2 = _mm_loadu_si128((const __m128i *) (p + 8)); + + plen -= 16; + p += 16; + uint8_t * restrict mp = &mm[j]; + + while (1) { + + // Preamble detection + // P G P G, G G G P, G P G G, G G G G + + // Extract gaps and sum together (widened to 32-bit, values are unsigned) + __m128i g1 = _mm_and_si128(p1, gm1); + __m128i g2 = _mm_and_si128(p2, gm2); + __m128i gs = _mm_add_epi32(_mm_add_epi32(_mm_unpacklo_epi16(g1, zero), _mm_unpackhi_epi16(g1, zero)), + _mm_add_epi32(_mm_unpacklo_epi16(g2, zero), _mm_unpackhi_epi16(g2, zero))); + uint32_t gap = hsum_u32x4(gs); + + // Extract pulses: low 4 lanes of px = p[0], p[9], p[2], p[7] + __m128i dm1 = _mm_andnot_si128(gm1, p1); + __m128i dm2 = _mm_andnot_si128(gm2, p2); + + __m128i px = _mm_or_si128(_mm_or_si128(dm1, _mm_srli_si128(dm1, 8)), dm2); + +#ifdef PREAMBLE_MIN_PULSE + __m128i m = _mm_min_epu16(px, _mm_srli_si128(px, 4)); + m = _mm_min_epu16(m, _mm_srli_si128(m, 2)); + unsigned minp = (unsigned) _mm_extract_epi16(m, 0); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32_t minp = hsum_u32x4(_mm_cvtepu16_epi32(px)); +#endif + // Metric + *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); + mp += 6; + + if (plen == 0) break; + plen--; + + unsigned nv = *p++; + p1 = _mm_alignr_epi8(p2, p1, 2); + p2 = _mm_insert_epi16(_mm_srli_si128(p2, 2), (int) nv, 7); + } + } + return METRIC_SIZE(nt); +} + +#endif + diff --git a/dump1090.c b/dump1090.c index f01c34ebd..2540790fe 100644 --- a/dump1090.c +++ b/dump1090.c @@ -119,11 +119,7 @@ static void modesInitConfig(void) { Modes.json_location_accuracy = 1; Modes.maxRange = 1852 * 300; // 300NM default max range Modes.mode_ac_auto = 1; - #ifdef DEMOD_2400X - Modes.mode_ac_auto = 0; - #else - Modes.mode_ac_auto = 1; - #endif + Modes.net_heartbeat_interval = MODES_NET_HEARTBEAT_INTERVAL; Modes.net_output_flush_size = 1300; Modes.net_output_flush_interval = 500; diff --git a/fifo.c b/fifo.c index efa4094e2..2fe0d51e6 100644 --- a/fifo.c +++ b/fifo.c @@ -38,11 +38,13 @@ static bool fifo_halted; // true if queue has been halted static unsigned overlap_length; // desired overlap size in samples (size of overlap_buffer) static uint16_t *overlap_buffer; // buffer used to save overlapping data +static unsigned datatype_iq; // 0 is power samples, 1 = IQ samples + // Create the queue structures. Not threadsafe. -bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned datatype_iq, unsigned overlap) +bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned dtype_iq, unsigned overlap) { - unsigned esize = datatype_iq?2:1; + unsigned esize = dtype_iq?2:1; if (!(overlap_buffer = calloc(overlap, sizeof(overlap_buffer[0])*esize))) goto nomem; @@ -61,7 +63,7 @@ bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned datatype_ newbuf->totalLength = buffer_size; newbuf->next = fifo_freelist; - newbuf->datatype_iq = datatype_iq; + datatype_iq = dtype_iq; fifo_freelist = newbuf; } @@ -178,7 +180,7 @@ void fifo_enqueue(struct mag_buf *buf) assert(buf->validLength >= overlap_length); pthread_mutex_lock(&fifo_mutex); - unsigned esize = buf->datatype_iq?2:1; + unsigned esize = datatype_iq?2:1; if (fifo_halted) { // Shutting down, just return the buffer to the freelist. diff --git a/fifo.h b/fifo.h index 1dcac464b..05b79ea6d 100644 --- a/fifo.h +++ b/fifo.h @@ -57,8 +57,6 @@ struct mag_buf { unsigned validLength; // Number of valid samples in "data", including overlap samples unsigned overlap; // Number of leading overlap samples at the start of "data"; // also the number of trailing samples that will be preserved for next time - unsigned datatype_iq; // 0 is power samples, 1 = IQ samples - uint64_t sampleTimestamp; // Clock timestamp of the start of this block, 12MHz clock uint64_t sysTimestamp; // Estimated system time at start of block @@ -74,9 +72,9 @@ struct mag_buf { // // buffer_count - the number of buffers to preallocate // buffer_size - the size of each magnitude buffer, in samples, including overlap -// data_type - 0 is power samples, 1 = IQ samples +// dtype_iq - 0 is power samples, 1 = IQ samples // overlap - the number of samples to overlap between adjacent buffers -bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned datatype_iq, unsigned overlap); +bool fifo_create(unsigned buffer_count, unsigned buffer_size, unsigned dtype_iq, unsigned overlap); // Destroy the fifo structures allocated in magbuf_fifo_create. Not threadsafe; ensure all FIFO users // are done before calling. From 06b5839545a667cb515ba3b2840a8c754a8b1002 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Sun, 18 Oct 2026 15:12:05 +0100 Subject: [PATCH 03/25] Split demod DSP into starch kernels (generic/NEON/AVX2) with benchmark and wisdom --- Makefile | 4 +- demod_2400x.c | 25 ++ demod_2400x.h | 20 - demod_dsp.c | 549 ----------------------- dsp/generated/benchmark.c | 251 +++++++++++ dsp/generated/dispatcher.c | 204 +++++++++ dsp/generated/flavor.armv7a_neon_vfpv4.c | 2 + dsp/generated/flavor.armv8_neon_simd.c | 2 + dsp/generated/flavor.generic.c | 1 + dsp/generated/flavor.x86_avx2.c | 2 + dsp/generated/makefile.aarch64 | 6 +- dsp/generated/makefile.arm | 6 +- dsp/generated/makefile.generic | 4 +- dsp/generated/makefile.x86 | 6 +- dsp/generated/starch.h | 44 ++ dsp/helpers/demod2400x_dsp.h | 45 ++ dsp/impl/demod2400x.avx2.c | 185 ++++++++ dsp/impl/demod2400x.benchmark.c | 123 +++++ dsp/impl/demod2400x.generic.c | 160 +++++++ dsp/impl/demod2400x.neon.c | 193 ++++++++ dsp/starchgen.py | 11 +- dump1090.h | 1 + wisdom.aarch64 | 10 + wisdom.arm | 10 + wisdom.x86 | 9 + 25 files changed, 1290 insertions(+), 583 deletions(-) create mode 100644 demod_2400x.c delete mode 100644 demod_dsp.c create mode 100644 dsp/helpers/demod2400x_dsp.h create mode 100644 dsp/impl/demod2400x.avx2.c create mode 100644 dsp/impl/demod2400x.benchmark.c create mode 100644 dsp/impl/demod2400x.generic.c create mode 100644 dsp/impl/demod2400x.neon.c diff --git a/Makefile b/Makefile index f04942bd8..ceabba9f3 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ DUMP1090_VERSION ?= unknown CFLAGS ?= -O3 -g DUMP1090_CFLAGS := -std=c11 -fno-common -Wall -Wmissing-declarations -Werror -Wformat-signedness -W DUMP1090_CPPFLAGS := -I. -D_POSIX_C_SOURCE=200112L -DMODES_DUMP1090_VERSION=\"$(DUMP1090_VERSION)\" -DMODES_DUMP1090_VARIANT=\"dump1090-fa\" -#DUMP1090_CPPFLAGS += -DDEMOD_2400X +DUMP1090_CPPFLAGS += -DDEMOD_2400X LIBS = -lpthread -lm SDR_OBJ = cpu.o sdr.o fifo.o sdr_ifile.o dsp/helpers/tables.o @@ -206,7 +206,7 @@ showconfig: %.o: %.c *.h $(CC) $(ALL_CCFLAGS) -c $< -o $@ -dump1090: dump1090.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o net_io.o crc.o demod_2400.o demod_dsp.o stats.o cpr.o icao_filter.o track.o util.o convert.o ais_charset.o adaptive.o $(SDR_OBJ) $(COMPAT) $(STARCH_OBJS) +dump1090: dump1090.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o net_io.o crc.o demod_2400.o demod_2400x.o stats.o cpr.o icao_filter.o track.o util.o convert.o ais_charset.o adaptive.o $(SDR_OBJ) $(COMPAT) $(STARCH_OBJS) $(CC) -g -o $@ $^ $(LDFLAGS) $(LIBS) $(LIBS_SDR) $(LIBS_CURSES) view1090: view1090.o anet.o interactive.o mode_ac.o mode_s.o comm_b.o net_io.o crc.o stats.o cpr.o icao_filter.o track.o util.o ais_charset.o sdr_stub.o $(COMPAT) diff --git a/demod_2400x.c b/demod_2400x.c new file mode 100644 index 000000000..b20e05e4b --- /dev/null +++ b/demod_2400x.c @@ -0,0 +1,25 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod_2400.c: 2.4MHz Mode S demodulator. +// +// Copyright (c) 2014,2015 Oliver Jowett +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. +// +// This file is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include "dump1090.h" + +void demodulate2400x(struct mag_buf *mag) +{ + MODES_NOTUSED(mag); // remove once the body uses mag +} diff --git a/demod_2400x.h b/demod_2400x.h index db4adeab6..0ba7d3727 100644 --- a/demod_2400x.h +++ b/demod_2400x.h @@ -1,26 +1,6 @@ #ifndef DEMOD_2400X_H #define DEMOD_2400X_H -#include "fifo.h" - -#define MIN(a,b) ((a) < (b) ? (a) : (b)) -#define MAX(a,b) ((a) < (b) ? (b) : (a)) - -#define NU_SAMP(t) (5*((t)-7)) -#define NU_STRIDE(t) ((((t)+5)/6+7)&~7) -#define METRIC_SIZE(t) ((t)-15-12*LONG_FRAME) - -#define LONG_FRAME 112 -#define SHORT_FRAME 56 - -#define PREAMBLE_KK 2 -#define PREAMBLE_MIN_PULSE - -#define OVERLAP_WINDOW \ - (7 + ((15 + 1 + 2*LONG_FRAME + 4) / 5)) - -extern int upsample12(const int16_t * restrict in, int len, int shift, uint16_t * restrict oo); -extern int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm); extern void demodulate2400x(struct mag_buf *mbuf); #endif diff --git a/demod_dsp.c b/demod_dsp.c deleted file mode 100644 index 5b1277cea..000000000 --- a/demod_dsp.c +++ /dev/null @@ -1,549 +0,0 @@ -#include -#include -#include "demod_2400x.h" - -#if defined(__aarch64__) - #define TARGET_NEON - #define TARGET_SIMD -#elif defined(__arm__) && defined(__ARM_NEON) - #define TARGET_NEON -#elif defined(__x86_64__) && defined(__AVX2__) - #define TARGET_AVX2 -#else - #define TARGET_GENERIC -#endif - -static const int8_t coeffs[5][8] = { - { -1, 4, -10, 125, 14, -6, 3, -1 }, // phase=0 sum=128 energy=15984 gd=3.0938 - { -3, 8, -21, 109, 47, -16, 7, -3 }, // phase=1 sum=128 energy=14918 gd=3.2969 - { -4, 10, -23, 81, 81, -23, 10, -4 }, // phase=2 sum=128 energy=14412 gd=3.5000 - { -3, 7, -16, 47, 109, -21, 8, -3 }, // phase=3 sum=128 energy=14918 gd=3.7031 - { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 -}; - - -#ifdef TARGET_NEON -#include - -// Horizontal add -static inline __attribute__((always_inline)) int hsum_s16x8(const int16x8_t v) -{ -#ifdef TARGET_SIMD - return vaddvq_s16(v); -#else - int16x4_t s = vadd_s16(vget_low_s16(v), - vget_high_s16(v)); - s = vpadd_s16(s, s); - s = vpadd_s16(s, s); - return vget_lane_s16(s, 0); -#endif -} - -// Horizontal add -static inline __attribute__((always_inline)) -uint32_t hsum_u32x4(const uint32x4_t v) -{ -#ifdef TARGET_SIMD - return vaddvq_u32(v); -#else - uint32x2_t s = vadd_u32(vget_low_u32(v), - vget_high_u32(v)); - s = vpadd_u32(s, s); - return vget_lane_u32(s, 0); -#endif -} - -// Horizontal add of two vectors at once: returns [sum of a, sum of b] -static inline __attribute__((always_inline)) -int32x2_t hsum2_s32x4(const int32x4_t a, const int32x4_t b) -{ -#ifdef TARGET_SIMD - int32x4_t ab = vpaddq_s32(a, b); // [a0+a1, a2+a3, b0+b1, b2+b3] - return vpadd_s32(vget_low_s32(ab), vget_high_s32(ab)); -#else - return vpadd_s32(vadd_s32(vget_low_s32(a), vget_high_s32(a)), - vadd_s32(vget_low_s32(b), vget_high_s32(b))); -#endif -} - - -static inline __attribute__((always_inline)) unsigned getpow(const int8x8_t cv8, const int16x8x2_t iq, int shift) -{ - const int16x8_t cv = vmovl_s8(cv8); // widen coefficients to 16-bit - int32x4_t si, sq; - - si = vmull_s16(vget_low_s16(iq.val[0]), vget_low_s16(cv)); - si = vmlal_s16(si, vget_high_s16(iq.val[0]), vget_high_s16(cv)); - - sq = vmull_s16(vget_low_s16(iq.val[1]), vget_low_s16(cv)); - sq = vmlal_s16(sq, vget_high_s16(iq.val[1]), vget_high_s16(cv)); - - int32x2_t res = hsum2_s32x4(si, sq); // [resi, resq] - res = vrshr_n_s32(res, 5); // (x+16)>>5 on both - - uint32x2_t sqr = vreinterpret_u32_s32(vmul_s32(res, res)); // [resi^2, resq^2] - uint32_t pwr = vget_lane_u32(sqr, 0) + vget_lane_u32(sqr, 1); - - pwr >>= shift; // Set gain - if (pwr > 65535) pwr = 65535; - - return pwr; -} - -// Upsample -int upsample12(const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) { - - if (len < 8) return 0; - - int8x8_t c0 = vld1_s8(coeffs[0]); - int8x8_t c1 = vld1_s8(coeffs[1]); - int8x8_t c2 = vld1_s8(coeffs[2]); - int8x8_t c3 = vld1_s8(coeffs[3]); - int8x8_t c4 = vld1_s8(coeffs[4]); - - int16x8x2_t iq = vld2q_s16(in); - - int nt = NU_SAMP(len); // Total samples - int stride = NU_STRIDE(nt); // Stride - int stride6 = 6*stride; - int bpos = 0; - - in += 16; - len -= 8; - while (1) { - - // Rearrage output - oo[bpos] = getpow(c0, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c1, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c2, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c3, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c4, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - if (len == 0) break; - len--; - - // Shift - iq.val[0] = vextq_s16(iq.val[0], iq.val[0], 1); - iq.val[1] = vextq_s16(iq.val[1], iq.val[1], 1); - - iq.val[0] = vsetq_lane_s16(in[0], iq.val[0], 7); - iq.val[1] = vsetq_lane_s16(in[1], iq.val[1], 7); - in += 2; - } - - return nt; -} - - -// Locate preamble -int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm) { - - int stride = NU_STRIDE(nt); // Stride - const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; - - uint16x8_t gm1 = vld1q_u16((const uint16_t *)mask_g+0); - uint16x8_t gm2 = vld1q_u16((const uint16_t *)mask_g+8); - - for (int j = 0; j < 6; j++) { // Look at each phase in turn - - const uint16_t * restrict p = &pp[j*stride]; - int plen = (nt + 5 - j) / 6; // Length of each phase - - plen -= 2 * LONG_FRAME; // No point in searching at end - if (plen < 16) continue; - - uint16x8_t p1 = vld1q_u16(p + 0); - uint16x8_t p2 = vld1q_u16(p + 8); - - plen -= 16; - p += 16; - uint8_t * restrict mp = &mm[j]; - - while (1) { - - // Preamble detection - // P G P G, G G G P, G P G G, G G G G - - // Extract gaps and sum together - uint32x4_t g1 = vpaddlq_u16(vandq_u16(p1, gm1)); - uint32x4_t g2 = vpaddlq_u16(vandq_u16(p2, gm2)); - uint32x4_t gs = vaddq_u32(g1, g2); - uint32_t gap = hsum_u32x4(gs); - - // Extract pulses - uint16x8_t dm1 = vbicq_u16(p1, gm1); - uint16x8_t dm2 = vbicq_u16(p2, gm2); - - uint16x4_t d0 = vget_low_u16(dm1); - uint16x4_t d1 = vget_high_u16(dm1); - uint16x4_t d2 = vget_low_u16(dm2); - - uint16x4_t px = vorr_u16(vorr_u16(d0, d1), d2); - -#ifdef PREAMBLE_MIN_PULSE - uint16x4_t m = vpmin_u16(px, px); - m = vpmin_u16(m, m); - unsigned minp = vget_lane_u16(m, 0); -#endif -#ifdef PREAMBLE_SUM_PULSE - uint32x2_t s = vpaddl_u16(px); - s = vpadd_u32(s, s); - uint32_t minp = vget_lane_u32(s, 0); -#endif - // Metric - *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); - mp += 6; - - if (plen == 0) break; - plen--; - - unsigned nv = *p++; - p1 = vextq_u16(p1, p2, 1); - p2 = vextq_u16(p2, p2, 1); - p2 = vsetq_lane_u16(nv, p2, 7); - } - } - return METRIC_SIZE(nt); -} - -#endif - - -#ifdef TARGET_GENERIC - -// Get power -static inline unsigned getpow( - const int8_t *cv, - const int16_t *ii, - const int16_t *qq, - int shift) -{ - int32_t si = - cv[0] * ii[0] - + cv[1] * ii[1] - + cv[2] * ii[2] - + cv[3] * ii[3] - + cv[4] * ii[4] - + cv[5] * ii[5] - + cv[6] * ii[6] - + cv[7] * ii[7]; - - int32_t sq = - cv[0] * qq[0] - + cv[1] * qq[1] - + cv[2] * qq[2] - + cv[3] * qq[3] - + cv[4] * qq[4] - + cv[5] * qq[5] - + cv[6] * qq[6] - + cv[7] * qq[7]; - - si = (si+16)>>5; - sq = (sq+16)>>5; - uint32_t pwr = (uint32_t)(si*si) + (uint32_t)(sq*sq); - - pwr >>= shift; // Set gain - if (pwr > 65535) pwr = 65535; - - return pwr; -} - - -// Upsample -int upsample12( - const int16_t * restrict in, - int len, int shift, - uint16_t * restrict oo) -{ - if (len < 8) - return 0; - - int16_t ii[8]; - int16_t qq[8]; - - int nt = NU_SAMP(len); - int stride = NU_STRIDE(nt); - int stride6 = 6 * stride; - int bpos = 0; - - /* Initial 8-sample window */ - for (int i = 0; i < 8; i++) { - - ii[i] = in[0]; - qq[i] = in[1]; - in += 2; - } - len -= 8; - - while (1) { - - oo[bpos] = getpow(coeffs[0], ii, qq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6 - 1; - - oo[bpos] = getpow(coeffs[1], ii, qq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6 - 1; - - oo[bpos] = getpow(coeffs[2], ii, qq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6 - 1; - - oo[bpos] = getpow(coeffs[3], ii, qq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6 - 1; - - oo[bpos] = getpow(coeffs[4], ii, qq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6 - 1; - - if (len == 0) - break; - - len--; - - /* Shift window left by one sample */ - for (int i = 0; i < 7; i++) { - ii[i] = ii[i + 1]; - qq[i] = qq[i + 1]; - } - - ii[7] = in[0]; - qq[7] = in[1]; - in += 2; - } - - return nt; -} - - -// Locate preamble -int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm) { - - int stride = NU_STRIDE(nt); - - for (int j = 0; j < 6; j++) { - - const uint16_t *restrict p = pp + j * stride; - uint8_t *restrict mp = mm + j; - - int plen = (nt + 5 - j) / 6; - - plen -= 2 * LONG_FRAME; - - for (int i = 0; i < plen - 15; i++) { - -#ifdef PREAMBLE_MIN_PULSE - uint32_t minp = - MIN( - MIN(p[0], p[2]), - MIN(p[7], p[9]) - ); -#endif -#ifdef PREAMBLE_SUM_PULSE - uint32_t minp = p[0] + p[2] + p[7] + p[9]; -#endif - uint32_t gap = - p[ 1] + - p[ 3] + - p[ 4] + - p[ 5] + - p[ 6] + - p[ 8] + - p[10] + - p[11] + - p[12] + - p[13] + - p[14] + - p[15]; - - *mp = (256 * PREAMBLE_KK * minp) - / (PREAMBLE_KK * minp + gap + 1); - - p++; - mp += 6; - } - } - - return METRIC_SIZE(nt); - -} -#endif - -#ifdef TARGET_AVX2 -#include - -// Two vectors of 8 x int16: val[0] = I, val[1] = Q (same layout as NEON's int16x8x2_t) -typedef struct { - __m128i val[2]; -} iq16x8x2_t; - -// Horizontal add -static inline __attribute__((always_inline)) int32_t hsum_s32x4(const __m128i v) -{ - __m128i s = _mm_add_epi32(v, _mm_shuffle_epi32(v, _MM_SHUFFLE(1, 0, 3, 2))); - s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(2, 3, 0, 1))); - return _mm_cvtsi128_si32(s); -} - -// Horizontal add -static inline __attribute__((always_inline)) uint32_t hsum_u32x4(const __m128i v) -{ - return (uint32_t) hsum_s32x4(v); -} - -// Load 8 interleaved I/Q samples and deinterleave them (like NEON's vld2q_s16) -static inline __attribute__((always_inline)) iq16x8x2_t load2_s16(const int16_t *in) -{ - const __m128i deint = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - - __m128i a = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) (in + 0)), deint); // I0-3 Q0-3 - __m128i b = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) (in + 8)), deint); // I4-7 Q4-7 - - iq16x8x2_t iq; - iq.val[0] = _mm_unpacklo_epi64(a, b); // I0-7 - iq.val[1] = _mm_unpackhi_epi64(a, b); // Q0-7 - return iq; -} - - -static inline __attribute__((always_inline)) unsigned getpow(const __m128i cv8, const iq16x8x2_t iq, int shift) -{ - const __m128i cv = _mm_cvtepi8_epi16(cv8); // widen coefficients to 16-bit - - // madd: multiply 8 pairs and add neighbours -> 4 x int32 partial sums - int32_t resi = (hsum_s32x4(_mm_madd_epi16(iq.val[0], cv)) + 16) >> 5; - int32_t resq = (hsum_s32x4(_mm_madd_epi16(iq.val[1], cv)) + 16) >> 5; - - uint32_t pwr = (uint32_t)(resi*resi) + (uint32_t)(resq*resq); - - pwr >>= shift; // Set gain - if (pwr > 65535) pwr = 65535; - - return pwr; -} - - -// Upsample -int upsample12(const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) { - - if (len < 8) return 0; - - __m128i c0 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[0]); // 8 x int8 - __m128i c1 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[1]); // 8 x int8 - __m128i c2 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[2]); // 8 x int8 - __m128i c3 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[3]); // 8 x int8 - __m128i c4 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[4]); // 8 x int8 - - iq16x8x2_t iq = load2_s16(in); - - int nt = NU_SAMP(len); // Total samples - int stride = NU_STRIDE(nt); // Stride - int stride6 = 6*stride; - int bpos = 0; - - in += 16; - len -= 8; - while (1) { - - // Rearrage output - oo[bpos] = getpow(c0, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c1, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c2, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c3, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow(c4, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - if (len == 0) break; - len--; - - // Shift - iq.val[0] = _mm_insert_epi16(_mm_srli_si128(iq.val[0], 2), in[0], 7); - iq.val[1] = _mm_insert_epi16(_mm_srli_si128(iq.val[1], 2), in[1], 7); - in += 2; - } - - return nt; -} - - -// Locate preamble -int locate_preamble(const uint16_t * restrict pp, int nt, uint8_t * restrict mm) { - - int stride = NU_STRIDE(nt); // Stride - const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; - - const __m128i gm1 = _mm_loadu_si128((const __m128i *) (mask_g + 0)); - const __m128i gm2 = _mm_loadu_si128((const __m128i *) (mask_g + 8)); - const __m128i zero = _mm_setzero_si128(); - - for (int j = 0; j < 6; j++) { // Look at each phase in turn - - const uint16_t * restrict p = &pp[j*stride]; - int plen = (nt + 5 - j) / 6; // Length of each phase - - plen -= 2 * LONG_FRAME; // No point in searching at end - if (plen < 16) continue; - - __m128i p1 = _mm_loadu_si128((const __m128i *) (p + 0)); - __m128i p2 = _mm_loadu_si128((const __m128i *) (p + 8)); - - plen -= 16; - p += 16; - uint8_t * restrict mp = &mm[j]; - - while (1) { - - // Preamble detection - // P G P G, G G G P, G P G G, G G G G - - // Extract gaps and sum together (widened to 32-bit, values are unsigned) - __m128i g1 = _mm_and_si128(p1, gm1); - __m128i g2 = _mm_and_si128(p2, gm2); - __m128i gs = _mm_add_epi32(_mm_add_epi32(_mm_unpacklo_epi16(g1, zero), _mm_unpackhi_epi16(g1, zero)), - _mm_add_epi32(_mm_unpacklo_epi16(g2, zero), _mm_unpackhi_epi16(g2, zero))); - uint32_t gap = hsum_u32x4(gs); - - // Extract pulses: low 4 lanes of px = p[0], p[9], p[2], p[7] - __m128i dm1 = _mm_andnot_si128(gm1, p1); - __m128i dm2 = _mm_andnot_si128(gm2, p2); - - __m128i px = _mm_or_si128(_mm_or_si128(dm1, _mm_srli_si128(dm1, 8)), dm2); - -#ifdef PREAMBLE_MIN_PULSE - __m128i m = _mm_min_epu16(px, _mm_srli_si128(px, 4)); - m = _mm_min_epu16(m, _mm_srli_si128(m, 2)); - unsigned minp = (unsigned) _mm_extract_epi16(m, 0); -#endif -#ifdef PREAMBLE_SUM_PULSE - uint32_t minp = hsum_u32x4(_mm_cvtepu16_epi32(px)); -#endif - // Metric - *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); - mp += 6; - - if (plen == 0) break; - plen--; - - unsigned nv = *p++; - p1 = _mm_alignr_epi8(p2, p1, 2); - p2 = _mm_insert_epi16(_mm_srli_si128(p2, 2), (int) nv, 7); - } - } - return METRIC_SIZE(nt); -} - -#endif - diff --git a/dsp/generated/benchmark.c b/dsp/generated/benchmark.c index ea75f83f7..e25c50d28 100644 --- a/dsp/generated/benchmark.c +++ b/dsp/generated/benchmark.c @@ -184,6 +184,119 @@ static void starch_benchmark_run_count_above_u16( const uint16_t * arg0, unsigne } } +/* prototypes for benchmark helpers provided by user code */ +void starch_locate_preamble_benchmark (void); +bool starch_locate_preamble_benchmark_verify ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); + +/* prototype the benchmarking function so that we can build with -Wmissing-declarations */ +void starch_locate_preamble_benchmark(void); + +static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentry * _entry, const uint16_t * arg0, int arg1, uint8_t * arg2 ) +{ + fprintf(stderr, " %-40s ", _entry->name); + + /* test for support */ + if (_entry->flavor_supported && !(_entry->flavor_supported())) { + fprintf(stderr, "unsupported\n"); + return; + } + + if (starch_benchmark_flavor_whitelist && !starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_whitelist)) { + fprintf(stderr, "skipped (not whitelisted)\n"); + return; + } + + if (starch_benchmark_flavor_blacklist && starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_blacklist)) { + fprintf(stderr, "skipped (blacklisted)\n"); + return; + } + + if (starch_benchmark_list_only) { + fprintf(stderr, "supported\n"); + return; + } + + /* initial warmup */ + for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) + _entry->callable ( arg0, arg1, arg2 ); + + /* verify correctness of the output */ + if (! starch_locate_preamble_benchmark_verify ( arg0, arg1, arg2 )) { + fprintf(stderr, "skipped (verification failed)\n"); + starch_benchmark_validation_failed = true; + return; + } + if (starch_benchmark_validate_only) { + fprintf(stderr, "validation ok\n"); + return; + } + + /* pre-benchmark, find a loop count that takes at least 100ms */ + starch_benchmark_time _start, _end; + uint64_t _elapsed = 0; + uint64_t _loops = 127; + while (_elapsed < 100000000) { + _loops *= 2; + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2 ); + starch_benchmark_get_time(&_end); + _elapsed = starch_benchmark_elapsed(&_start, &_end); + } + + /* real benchmark, run for approx 1 second */ + _loops = _loops * 1000000000 / _elapsed; + + _elapsed = 0; + uint64_t _elapsed_min = UINT64_MAX; + uint64_t _elapsed_max = 0; + for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2 ); + starch_benchmark_get_time(&_end); + uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); + if (_elapsed_one < _elapsed_min) + _elapsed_min = _elapsed_one; + if (_elapsed_one > _elapsed_max) + _elapsed_max = _elapsed_one; + _elapsed += _elapsed_one; + } + + uint64_t _per_loop; + if (starch_benchmark_iterations > 2) + _per_loop = (_elapsed - _elapsed_min - _elapsed_max) / _loops / (starch_benchmark_iterations - 2); + else + _per_loop = _elapsed / _loops / starch_benchmark_iterations; + + fprintf(stderr, "%" PRIu64 " ns/call\n", _per_loop); + + if (starch_benchmark_result_count >= starch_benchmark_result_size) { + if (!starch_benchmark_result_size) + starch_benchmark_result_size = 64; + else + starch_benchmark_result_size *= 2; + starch_benchmark_results = realloc(starch_benchmark_results, starch_benchmark_result_size * sizeof(*starch_benchmark_results)); + if (!starch_benchmark_results) { + fprintf(stderr, "realloc: %s\n", strerror(errno)); + exit(1); + } + } + + starch_benchmark_results[starch_benchmark_result_count].name = "locate_preamble"; + starch_benchmark_results[starch_benchmark_result_count].impl = _entry->name; + starch_benchmark_results[starch_benchmark_result_count].ns = _per_loop; + starch_benchmark_results[starch_benchmark_result_count].flavor = _entry->flavor; + ++starch_benchmark_result_count; +} + +static void starch_benchmark_run_locate_preamble( const uint16_t * arg0, int arg1, uint8_t * arg2 ) +{ + for (starch_locate_preamble_regentry *_entry = starch_locate_preamble_registry; _entry->name; ++_entry) { + starch_benchmark_one_locate_preamble( _entry, arg0, arg1, arg2 ); + } +} + /* prototypes for benchmark helpers provided by user code */ void starch_magnitude_power_uc8_benchmark (void); bool starch_magnitude_power_uc8_benchmark_verify ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); @@ -749,6 +862,119 @@ static void starch_benchmark_run_mean_power_u16( const uint16_t * arg0, unsigned } } +/* prototypes for benchmark helpers provided by user code */ +void starch_upsample12_benchmark (void); +bool starch_upsample12_benchmark_verify ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); + +/* prototype the benchmarking function so that we can build with -Wmissing-declarations */ +void starch_upsample12_benchmark(void); + +static void starch_benchmark_one_upsample12( starch_upsample12_regentry * _entry, const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ) +{ + fprintf(stderr, " %-40s ", _entry->name); + + /* test for support */ + if (_entry->flavor_supported && !(_entry->flavor_supported())) { + fprintf(stderr, "unsupported\n"); + return; + } + + if (starch_benchmark_flavor_whitelist && !starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_whitelist)) { + fprintf(stderr, "skipped (not whitelisted)\n"); + return; + } + + if (starch_benchmark_flavor_blacklist && starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_blacklist)) { + fprintf(stderr, "skipped (blacklisted)\n"); + return; + } + + if (starch_benchmark_list_only) { + fprintf(stderr, "supported\n"); + return; + } + + /* initial warmup */ + for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3 ); + + /* verify correctness of the output */ + if (! starch_upsample12_benchmark_verify ( arg0, arg1, arg2, arg3 )) { + fprintf(stderr, "skipped (verification failed)\n"); + starch_benchmark_validation_failed = true; + return; + } + if (starch_benchmark_validate_only) { + fprintf(stderr, "validation ok\n"); + return; + } + + /* pre-benchmark, find a loop count that takes at least 100ms */ + starch_benchmark_time _start, _end; + uint64_t _elapsed = 0; + uint64_t _loops = 127; + while (_elapsed < 100000000) { + _loops *= 2; + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3 ); + starch_benchmark_get_time(&_end); + _elapsed = starch_benchmark_elapsed(&_start, &_end); + } + + /* real benchmark, run for approx 1 second */ + _loops = _loops * 1000000000 / _elapsed; + + _elapsed = 0; + uint64_t _elapsed_min = UINT64_MAX; + uint64_t _elapsed_max = 0; + for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3 ); + starch_benchmark_get_time(&_end); + uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); + if (_elapsed_one < _elapsed_min) + _elapsed_min = _elapsed_one; + if (_elapsed_one > _elapsed_max) + _elapsed_max = _elapsed_one; + _elapsed += _elapsed_one; + } + + uint64_t _per_loop; + if (starch_benchmark_iterations > 2) + _per_loop = (_elapsed - _elapsed_min - _elapsed_max) / _loops / (starch_benchmark_iterations - 2); + else + _per_loop = _elapsed / _loops / starch_benchmark_iterations; + + fprintf(stderr, "%" PRIu64 " ns/call\n", _per_loop); + + if (starch_benchmark_result_count >= starch_benchmark_result_size) { + if (!starch_benchmark_result_size) + starch_benchmark_result_size = 64; + else + starch_benchmark_result_size *= 2; + starch_benchmark_results = realloc(starch_benchmark_results, starch_benchmark_result_size * sizeof(*starch_benchmark_results)); + if (!starch_benchmark_results) { + fprintf(stderr, "realloc: %s\n", strerror(errno)); + exit(1); + } + } + + starch_benchmark_results[starch_benchmark_result_count].name = "upsample12"; + starch_benchmark_results[starch_benchmark_result_count].impl = _entry->name; + starch_benchmark_results[starch_benchmark_result_count].ns = _per_loop; + starch_benchmark_results[starch_benchmark_result_count].flavor = _entry->flavor; + ++starch_benchmark_result_count; +} + +static void starch_benchmark_run_upsample12( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ) +{ + for (starch_upsample12_regentry *_entry = starch_upsample12_registry; _entry->name; ++_entry) { + starch_benchmark_one_upsample12( _entry, arg0, arg1, arg2, arg3 ); + } +} + #define STARCH_SYMBOL(_name) starch_ ## _name ## _benchmark_sym #define STARCH_BENCHMARK(_function) starch_ ## _function ## _benchmark @@ -756,6 +982,7 @@ static void starch_benchmark_run_mean_power_u16( const uint16_t * arg0, unsigned #define STARCH_BENCHMARK_RUN(_function, ...) starch_benchmark_run_ ## _function ( __VA_ARGS__ ) #include "../impl/count_above_u16.benchmark.c" +#include "../impl/demod2400x.benchmark.c" #include "../impl/magnitude_power_uc8.benchmark.c" #include "../impl/magnitude_sc16.benchmark.c" #include "../impl/magnitude_sc16q11.benchmark.c" @@ -772,6 +999,11 @@ static void starch_benchmark_all_count_above_u16(void) fprintf(stderr, "==== count_above_u16 ===\n"); starch_count_above_u16_benchmark (); } +static void starch_benchmark_all_locate_preamble(void) +{ + fprintf(stderr, "==== locate_preamble ===\n"); + starch_locate_preamble_benchmark (); +} static void starch_benchmark_all_magnitude_power_uc8(void) { fprintf(stderr, "==== magnitude_power_uc8 ===\n"); @@ -797,6 +1029,11 @@ static void starch_benchmark_all_mean_power_u16(void) fprintf(stderr, "==== mean_power_u16 ===\n"); starch_mean_power_u16_benchmark (); } +static void starch_benchmark_all_upsample12(void) +{ + fprintf(stderr, "==== upsample12 ===\n"); + starch_upsample12_benchmark (); +} static int starch_benchmark_compare_result(const void *a, const void *b) { @@ -870,11 +1107,13 @@ static void starch_benchmark_usage(const char *argv0) "iqdata_conv_sc16 " "iqdata_conv_sc16q11 " "iqdata_conv_uc8 " + "locate_preamble " "magnitude_power_uc8 " "magnitude_sc16 " "magnitude_sc16q11 " "magnitude_uc8 " "mean_power_u16 " + "upsample12 " "\n"); } @@ -983,6 +1222,11 @@ int main(int argc, char **argv) fprintf(stderr, " (no benchmark support defined)\n"); continue; } + if (!strcmp(argv[i], "locate_preamble")) { + specific = 1; + starch_benchmark_all_locate_preamble(); + continue; + } if (!strcmp(argv[i], "magnitude_power_uc8")) { specific = 1; starch_benchmark_all_magnitude_power_uc8(); @@ -1008,6 +1252,11 @@ int main(int argc, char **argv) starch_benchmark_all_mean_power_u16(); continue; } + if (!strcmp(argv[i], "upsample12")) { + specific = 1; + starch_benchmark_all_upsample12(); + continue; + } fprintf(stderr, "%s: unrecognized function name: %s\n", argv[0], argv[i]); return 2; @@ -1021,11 +1270,13 @@ int main(int argc, char **argv) fprintf(stderr, " (no benchmark support defined)\n"); fprintf(stderr, "=== iqdata_conv_uc8 ===\n"); fprintf(stderr, " (no benchmark support defined)\n"); + starch_benchmark_all_locate_preamble(); starch_benchmark_all_magnitude_power_uc8(); starch_benchmark_all_magnitude_sc16(); starch_benchmark_all_magnitude_sc16q11(); starch_benchmark_all_magnitude_uc8(); starch_benchmark_all_mean_power_u16(); + starch_benchmark_all_upsample12(); } if (output_path) { diff --git a/dsp/generated/dispatcher.c b/dsp/generated/dispatcher.c index c4b8d10ba..52803bfc9 100644 --- a/dsp/generated/dispatcher.c +++ b/dsp/generated/dispatcher.c @@ -321,6 +321,84 @@ starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[] = { { 0, NULL, NULL, NULL, NULL } }; +/* dispatcher / registry for locate_preamble */ + +starch_locate_preamble_regentry * starch_locate_preamble_select() { + for (starch_locate_preamble_regentry *entry = starch_locate_preamble_registry; + entry->name; + ++entry) + { + if (entry->flavor_supported && !(entry->flavor_supported())) + continue; + return entry; + } + return NULL; +} + +static int starch_locate_preamble_dispatch ( const uint16_t * arg0, int arg1, uint8_t * arg2 ) { + starch_locate_preamble_regentry *entry = starch_locate_preamble_select(); + if (!entry) + abort(); + + starch_locate_preamble = entry->callable; + return starch_locate_preamble ( arg0, arg1, arg2 ); +} + +starch_locate_preamble_ptr starch_locate_preamble = starch_locate_preamble_dispatch; + +void starch_locate_preamble_set_wisdom (const char * const * received_wisdom) +{ + /* re-rank the registry based on received wisdom */ + starch_locate_preamble_regentry *entry; + for (entry = starch_locate_preamble_registry; entry->name; ++entry) { + const char * const *search; + for (search = received_wisdom; *search; ++search) { + if (!strcmp(*search, entry->name)) { + break; + } + } + if (*search) { + /* matches an entry in the wisdom list, order by position in the list */ + entry->rank = search - received_wisdom; + } else { + /* no match, rank after all possible matches, retaining existing order */ + entry->rank = (search - received_wisdom) + (entry - starch_locate_preamble_registry); + } + } + + /* re-sort based on the new ranking */ + qsort(starch_locate_preamble_registry, entry - starch_locate_preamble_registry, sizeof(starch_locate_preamble_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_locate_preamble = starch_locate_preamble_dispatch; +} + +starch_locate_preamble_regentry starch_locate_preamble_registry[] = { + +#ifdef STARCH_MIX_AARCH64 + { 0, "neon_armv8_neon_simd", "armv8_neon_simd", starch_locate_preamble_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "generic_armv8_neon_simd", "armv8_neon_simd", starch_locate_preamble_generic_armv8_neon_simd, cpu_supports_armv8_simd }, + { 2, "generic_generic", "generic", starch_locate_preamble_generic_generic, NULL }, +#endif /* STARCH_MIX_AARCH64 */ + +#ifdef STARCH_MIX_ARM + { 0, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_locate_preamble_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_locate_preamble_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 2, "generic_generic", "generic", starch_locate_preamble_generic_generic, NULL }, +#endif /* STARCH_MIX_ARM */ + +#ifdef STARCH_MIX_GENERIC + { 0, "generic_generic", "generic", starch_locate_preamble_generic_generic, NULL }, +#endif /* STARCH_MIX_GENERIC */ + +#ifdef STARCH_MIX_X86 + { 0, "generic_x86_avx2", "x86_avx2", starch_locate_preamble_generic_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_generic", "generic", starch_locate_preamble_generic_generic, NULL }, + { 2, "avx2_x86_avx2", "x86_avx2", starch_locate_preamble_avx2_x86_avx2, cpu_supports_avx2 }, +#endif /* STARCH_MIX_X86 */ + { 0, NULL, NULL, NULL, NULL } +}; + /* dispatcher / registry for magnitude_power_uc8 */ starch_magnitude_power_uc8_regentry * starch_magnitude_power_uc8_select() { @@ -776,6 +854,84 @@ starch_mean_power_u16_regentry starch_mean_power_u16_registry[] = { { 0, NULL, NULL, NULL, NULL } }; +/* dispatcher / registry for upsample12 */ + +starch_upsample12_regentry * starch_upsample12_select() { + for (starch_upsample12_regentry *entry = starch_upsample12_registry; + entry->name; + ++entry) + { + if (entry->flavor_supported && !(entry->flavor_supported())) + continue; + return entry; + } + return NULL; +} + +static int starch_upsample12_dispatch ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ) { + starch_upsample12_regentry *entry = starch_upsample12_select(); + if (!entry) + abort(); + + starch_upsample12 = entry->callable; + return starch_upsample12 ( arg0, arg1, arg2, arg3 ); +} + +starch_upsample12_ptr starch_upsample12 = starch_upsample12_dispatch; + +void starch_upsample12_set_wisdom (const char * const * received_wisdom) +{ + /* re-rank the registry based on received wisdom */ + starch_upsample12_regentry *entry; + for (entry = starch_upsample12_registry; entry->name; ++entry) { + const char * const *search; + for (search = received_wisdom; *search; ++search) { + if (!strcmp(*search, entry->name)) { + break; + } + } + if (*search) { + /* matches an entry in the wisdom list, order by position in the list */ + entry->rank = search - received_wisdom; + } else { + /* no match, rank after all possible matches, retaining existing order */ + entry->rank = (search - received_wisdom) + (entry - starch_upsample12_registry); + } + } + + /* re-sort based on the new ranking */ + qsort(starch_upsample12_registry, entry - starch_upsample12_registry, sizeof(starch_upsample12_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_upsample12 = starch_upsample12_dispatch; +} + +starch_upsample12_regentry starch_upsample12_registry[] = { + +#ifdef STARCH_MIX_AARCH64 + { 0, "neon_armv8_neon_simd", "armv8_neon_simd", starch_upsample12_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "generic_armv8_neon_simd", "armv8_neon_simd", starch_upsample12_generic_armv8_neon_simd, cpu_supports_armv8_simd }, + { 2, "generic_generic", "generic", starch_upsample12_generic_generic, NULL }, +#endif /* STARCH_MIX_AARCH64 */ + +#ifdef STARCH_MIX_ARM + { 0, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_upsample12_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_upsample12_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 2, "generic_generic", "generic", starch_upsample12_generic_generic, NULL }, +#endif /* STARCH_MIX_ARM */ + +#ifdef STARCH_MIX_GENERIC + { 0, "generic_generic", "generic", starch_upsample12_generic_generic, NULL }, +#endif /* STARCH_MIX_GENERIC */ + +#ifdef STARCH_MIX_X86 + { 0, "avx2_x86_avx2", "x86_avx2", starch_upsample12_avx2_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_generic", "generic", starch_upsample12_generic_generic, NULL }, + { 2, "generic_x86_avx2", "x86_avx2", starch_upsample12_generic_x86_avx2, cpu_supports_avx2 }, +#endif /* STARCH_MIX_X86 */ + { 0, NULL, NULL, NULL, NULL } +}; + int starch_read_wisdom (const char * path) { @@ -800,6 +956,10 @@ int starch_read_wisdom (const char * path) for (starch_iqdata_conv_uc8_regentry *entry = starch_iqdata_conv_uc8_registry; entry->name; ++entry) { entry->rank = 0; } + int rank_locate_preamble = 0; + for (starch_locate_preamble_regentry *entry = starch_locate_preamble_registry; entry->name; ++entry) { + entry->rank = 0; + } int rank_magnitude_power_uc8 = 0; for (starch_magnitude_power_uc8_regentry *entry = starch_magnitude_power_uc8_registry; entry->name; ++entry) { entry->rank = 0; @@ -820,6 +980,10 @@ int starch_read_wisdom (const char * path) for (starch_mean_power_u16_regentry *entry = starch_mean_power_u16_registry; entry->name; ++entry) { entry->rank = 0; } + int rank_upsample12 = 0; + for (starch_upsample12_regentry *entry = starch_upsample12_registry; entry->name; ++entry) { + entry->rank = 0; + } char linebuf[512]; while (fgets(linebuf, sizeof(linebuf), fp)) { @@ -889,6 +1053,15 @@ int starch_read_wisdom (const char * path) } continue; } + if (!strcmp(name, "locate_preamble")) { + for (starch_locate_preamble_regentry *entry = starch_locate_preamble_registry; entry->name; ++entry) { + if (!strcmp(impl, entry->name)) { + entry->rank = ++rank_locate_preamble; + break; + } + } + continue; + } if (!strcmp(name, "magnitude_power_uc8")) { for (starch_magnitude_power_uc8_regentry *entry = starch_magnitude_power_uc8_registry; entry->name; ++entry) { if (!strcmp(impl, entry->name)) { @@ -934,6 +1107,15 @@ int starch_read_wisdom (const char * path) } continue; } + if (!strcmp(name, "upsample12")) { + for (starch_upsample12_regentry *entry = starch_upsample12_registry; entry->name; ++entry) { + if (!strcmp(impl, entry->name)) { + entry->rank = ++rank_upsample12; + break; + } + } + continue; + } } if (ferror(fp)) { @@ -988,6 +1170,17 @@ int starch_read_wisdom (const char * path) /* reset the implementation pointer so the next call will re-select */ starch_iqdata_conv_uc8 = starch_iqdata_conv_uc8_dispatch; } + { + starch_locate_preamble_regentry *entry; + for (entry = starch_locate_preamble_registry; entry->name; ++entry) { + if (!entry->rank) + entry->rank = ++rank_locate_preamble; + } + qsort(starch_locate_preamble_registry, entry - starch_locate_preamble_registry, sizeof(starch_locate_preamble_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_locate_preamble = starch_locate_preamble_dispatch; + } { starch_magnitude_power_uc8_regentry *entry; for (entry = starch_magnitude_power_uc8_registry; entry->name; ++entry) { @@ -1043,6 +1236,17 @@ int starch_read_wisdom (const char * path) /* reset the implementation pointer so the next call will re-select */ starch_mean_power_u16 = starch_mean_power_u16_dispatch; } + { + starch_upsample12_regentry *entry; + for (entry = starch_upsample12_registry; entry->name; ++entry) { + if (!entry->rank) + entry->rank = ++rank_upsample12; + } + qsort(starch_upsample12_registry, entry - starch_upsample12_registry, sizeof(starch_upsample12_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_upsample12 = starch_upsample12_dispatch; + } return 0; } diff --git a/dsp/generated/flavor.armv7a_neon_vfpv4.c b/dsp/generated/flavor.armv7a_neon_vfpv4.c index fd52b9a78..ae00df6ef 100644 --- a/dsp/generated/flavor.armv7a_neon_vfpv4.c +++ b/dsp/generated/flavor.armv7a_neon_vfpv4.c @@ -10,6 +10,8 @@ #include "../impl/count_above_u16.generic.c" #include "../impl/count_above_u16.neon.c" +#include "../impl/demod2400x.generic.c" +#include "../impl/demod2400x.neon.c" #include "../impl/magnitude_power_uc8.generic.c" #include "../impl/magnitude_power_uc8.neon.c" #include "../impl/magnitude_sc16.generic.c" diff --git a/dsp/generated/flavor.armv8_neon_simd.c b/dsp/generated/flavor.armv8_neon_simd.c index f58ce71e3..b946cdeae 100644 --- a/dsp/generated/flavor.armv8_neon_simd.c +++ b/dsp/generated/flavor.armv8_neon_simd.c @@ -10,6 +10,8 @@ #include "../impl/count_above_u16.generic.c" #include "../impl/count_above_u16.neon.c" +#include "../impl/demod2400x.generic.c" +#include "../impl/demod2400x.neon.c" #include "../impl/magnitude_power_uc8.generic.c" #include "../impl/magnitude_power_uc8.neon.c" #include "../impl/magnitude_sc16.generic.c" diff --git a/dsp/generated/flavor.generic.c b/dsp/generated/flavor.generic.c index fff5fbcb0..760888c81 100644 --- a/dsp/generated/flavor.generic.c +++ b/dsp/generated/flavor.generic.c @@ -9,6 +9,7 @@ #define STARCH_IMPL(_function,_impl) starch_ ## _function ## _ ## _impl ## _ ## generic #include "../impl/count_above_u16.generic.c" +#include "../impl/demod2400x.generic.c" #include "../impl/magnitude_power_uc8.generic.c" #include "../impl/magnitude_sc16.generic.c" #include "../impl/magnitude_sc16q11.generic.c" diff --git a/dsp/generated/flavor.x86_avx2.c b/dsp/generated/flavor.x86_avx2.c index 56a56029a..730c083c7 100644 --- a/dsp/generated/flavor.x86_avx2.c +++ b/dsp/generated/flavor.x86_avx2.c @@ -9,6 +9,8 @@ #define STARCH_IMPL(_function,_impl) starch_ ## _function ## _ ## _impl ## _ ## x86_avx2 #include "../impl/count_above_u16.generic.c" +#include "../impl/demod2400x.avx2.c" +#include "../impl/demod2400x.generic.c" #include "../impl/magnitude_power_uc8.generic.c" #include "../impl/magnitude_sc16.generic.c" #include "../impl/magnitude_sc16q11.generic.c" diff --git a/dsp/generated/makefile.aarch64 b/dsp/generated/makefile.aarch64 index c0dad13cd..745321a21 100644 --- a/dsp/generated/makefile.aarch64 +++ b/dsp/generated/makefile.aarch64 @@ -24,11 +24,11 @@ STARCH_CFLAGS := -DSTARCH_MIX_AARCH64 -dsp/generated/flavor.armv8_neon_simd.o: dsp/generated/flavor.armv8_neon_simd.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/count_above_u16.neon.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_power_uc8.neon.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16.neon.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_sc16q11.neon.c dsp/impl/magnitude_uc8.generic.c dsp/impl/magnitude_uc8.neon.c dsp/impl/mean_power_u16.generic.c dsp/impl/mean_power_u16.neon.c +dsp/generated/flavor.armv8_neon_simd.o: dsp/generated/flavor.armv8_neon_simd.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/count_above_u16.neon.c dsp/impl/demod2400x.generic.c dsp/impl/demod2400x.neon.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_power_uc8.neon.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16.neon.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_sc16q11.neon.c dsp/impl/magnitude_uc8.generic.c dsp/impl/magnitude_uc8.neon.c dsp/impl/mean_power_u16.generic.c dsp/impl/mean_power_u16.neon.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.armv8_neon_simd.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) -march=armv8-a+simd -ffast-math dsp/generated/flavor.armv8_neon_simd.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.armv8_neon_simd.o -dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c +dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/demod2400x.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/flavor.generic.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o @@ -40,7 +40,7 @@ dsp/generated/dispatcher.o: dsp/generated/dispatcher.c dsp/generated/starch.h STARCH_OBJS := dsp/generated/flavor.armv8_neon_simd.o dsp/generated/flavor.generic.o dsp/generated/dispatcher.o -dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c +dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/demod2400x.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/benchmark.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/benchmark.c -o $(STARCH_OBJ_PATH)dsp/generated/benchmark.o diff --git a/dsp/generated/makefile.arm b/dsp/generated/makefile.arm index 6da5dbbf0..a37feda17 100644 --- a/dsp/generated/makefile.arm +++ b/dsp/generated/makefile.arm @@ -24,11 +24,11 @@ STARCH_CFLAGS := -DSTARCH_MIX_ARM -dsp/generated/flavor.armv7a_neon_vfpv4.o: dsp/generated/flavor.armv7a_neon_vfpv4.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/count_above_u16.neon.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_power_uc8.neon.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16.neon.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_sc16q11.neon.c dsp/impl/magnitude_uc8.generic.c dsp/impl/magnitude_uc8.neon.c dsp/impl/mean_power_u16.generic.c dsp/impl/mean_power_u16.neon.c +dsp/generated/flavor.armv7a_neon_vfpv4.o: dsp/generated/flavor.armv7a_neon_vfpv4.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/count_above_u16.neon.c dsp/impl/demod2400x.generic.c dsp/impl/demod2400x.neon.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_power_uc8.neon.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16.neon.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_sc16q11.neon.c dsp/impl/magnitude_uc8.generic.c dsp/impl/magnitude_uc8.neon.c dsp/impl/mean_power_u16.generic.c dsp/impl/mean_power_u16.neon.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.armv7a_neon_vfpv4.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) -march=armv7-a+neon-vfpv4 -mfpu=neon-vfpv4 -ffast-math dsp/generated/flavor.armv7a_neon_vfpv4.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.armv7a_neon_vfpv4.o -dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c +dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/demod2400x.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/flavor.generic.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o @@ -40,7 +40,7 @@ dsp/generated/dispatcher.o: dsp/generated/dispatcher.c dsp/generated/starch.h STARCH_OBJS := dsp/generated/flavor.armv7a_neon_vfpv4.o dsp/generated/flavor.generic.o dsp/generated/dispatcher.o -dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c +dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/demod2400x.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/benchmark.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/benchmark.c -o $(STARCH_OBJ_PATH)dsp/generated/benchmark.o diff --git a/dsp/generated/makefile.generic b/dsp/generated/makefile.generic index 775641e8b..4e135046e 100644 --- a/dsp/generated/makefile.generic +++ b/dsp/generated/makefile.generic @@ -24,7 +24,7 @@ STARCH_CFLAGS := -DSTARCH_MIX_GENERIC -dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c +dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/demod2400x.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/flavor.generic.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o @@ -36,7 +36,7 @@ dsp/generated/dispatcher.o: dsp/generated/dispatcher.c dsp/generated/starch.h STARCH_OBJS := dsp/generated/flavor.generic.o dsp/generated/dispatcher.o -dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c +dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/demod2400x.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/benchmark.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/benchmark.c -o $(STARCH_OBJ_PATH)dsp/generated/benchmark.o diff --git a/dsp/generated/makefile.x86 b/dsp/generated/makefile.x86 index 1b7bcef46..d81fbc179 100644 --- a/dsp/generated/makefile.x86 +++ b/dsp/generated/makefile.x86 @@ -24,11 +24,11 @@ STARCH_CFLAGS := -DSTARCH_MIX_X86 -dsp/generated/flavor.x86_avx2.o: dsp/generated/flavor.x86_avx2.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c +dsp/generated/flavor.x86_avx2.o: dsp/generated/flavor.x86_avx2.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/demod2400x.avx2.c dsp/impl/demod2400x.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.x86_avx2.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) -mavx2 -ffast-math dsp/generated/flavor.x86_avx2.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.x86_avx2.o -dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c +dsp/generated/flavor.generic.o: dsp/generated/flavor.generic.c dsp/generated/starch.h dsp/impl/count_above_u16.generic.c dsp/impl/demod2400x.generic.c dsp/impl/magnitude_power_uc8.generic.c dsp/impl/magnitude_sc16.generic.c dsp/impl/magnitude_sc16q11.generic.c dsp/impl/magnitude_uc8.generic.c dsp/impl/mean_power_u16.generic.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/flavor.generic.c -o $(STARCH_OBJ_PATH)dsp/generated/flavor.generic.o @@ -40,7 +40,7 @@ dsp/generated/dispatcher.o: dsp/generated/dispatcher.c dsp/generated/starch.h STARCH_OBJS := dsp/generated/flavor.x86_avx2.o dsp/generated/flavor.generic.o dsp/generated/dispatcher.o -dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c +dsp/generated/benchmark.o: dsp/generated/benchmark.c dsp/generated/starch.h dsp/impl/count_above_u16.benchmark.c dsp/impl/demod2400x.benchmark.c dsp/impl/magnitude_power_uc8.benchmark.c dsp/impl/magnitude_sc16.benchmark.c dsp/impl/magnitude_sc16q11.benchmark.c dsp/impl/magnitude_uc8.benchmark.c dsp/impl/mean_power_u16.benchmark.c @$(MKDIR_P) $(dir $(STARCH_OBJ_PATH)dsp/generated/benchmark.o) $(STARCH_COMPILE) $(STARCH_CFLAGS) dsp/generated/benchmark.c -o $(STARCH_OBJ_PATH)dsp/generated/benchmark.o diff --git a/dsp/generated/starch.h b/dsp/generated/starch.h index fa2db8dbd..dc3980b18 100644 --- a/dsp/generated/starch.h +++ b/dsp/generated/starch.h @@ -183,6 +183,36 @@ extern starch_iqdata_conv_sc16q11_regentry starch_iqdata_conv_sc16q11_registry[] starch_iqdata_conv_sc16q11_regentry * starch_iqdata_conv_sc16q11_select(); void starch_iqdata_conv_sc16q11_set_wisdom( const char * const * received_wisdom ); +typedef int (* starch_upsample12_ptr) ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +extern starch_upsample12_ptr starch_upsample12; + +typedef struct { + int rank; + const char *name; + const char *flavor; + starch_upsample12_ptr callable; + int (*flavor_supported)(); +} starch_upsample12_regentry; + +extern starch_upsample12_regentry starch_upsample12_registry[]; +starch_upsample12_regentry * starch_upsample12_select(); +void starch_upsample12_set_wisdom( const char * const * received_wisdom ); + +typedef int (* starch_locate_preamble_ptr) ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +extern starch_locate_preamble_ptr starch_locate_preamble; + +typedef struct { + int rank; + const char *name; + const char *flavor; + starch_locate_preamble_ptr callable; + int (*flavor_supported)(); +} starch_locate_preamble_regentry; + +extern starch_locate_preamble_regentry starch_locate_preamble_registry[]; +starch_locate_preamble_regentry * starch_locate_preamble_select(); +void starch_locate_preamble_set_wisdom( const char * const * received_wisdom ); + /* flavors and prototypes */ @@ -192,6 +222,8 @@ void starch_magnitude_power_uc8_twopass_armv7a_neon_vfpv4 ( const uc8_t * arg0, void starch_magnitude_power_uc8_lookup_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_iqdata_conv_uc8_arith_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +int starch_upsample12_generic_armv7a_neon_vfpv4 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); void starch_magnitude_sc16_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_iqdata_conv_sc16_arith_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); @@ -213,6 +245,8 @@ void starch_count_above_u16_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsi void starch_magnitude_power_uc8_neon_vrsqrte_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +int starch_upsample12_neon_armv7a_neon_vfpv4 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); #endif /* STARCH_FLAVOR_ARMV7A_NEON_VFPV4 */ int starch_read_wisdom (const char * path); @@ -223,6 +257,8 @@ void starch_magnitude_power_uc8_twopass_armv8_neon_simd ( const uc8_t * arg0, ui void starch_magnitude_power_uc8_lookup_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_iqdata_conv_uc8_arith_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +int starch_upsample12_generic_armv8_neon_simd ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_generic_armv8_neon_simd ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); void starch_magnitude_sc16_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_iqdata_conv_sc16_arith_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); @@ -244,6 +280,8 @@ void starch_count_above_u16_neon_armv8_neon_simd ( const uint16_t * arg0, unsign void starch_magnitude_power_uc8_neon_vrsqrte_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); +int starch_upsample12_neon_armv8_neon_simd ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_neon_armv8_neon_simd ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); #endif /* STARCH_FLAVOR_ARMV8_NEON_SIMD */ int starch_read_wisdom (const char * path); @@ -253,6 +291,8 @@ void starch_magnitude_power_uc8_twopass_generic ( const uc8_t * arg0, uint16_t * void starch_magnitude_power_uc8_lookup_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_iqdata_conv_uc8_arith_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +int starch_upsample12_generic_generic ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_generic_generic ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); void starch_magnitude_sc16_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_iqdata_conv_sc16_arith_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); @@ -278,6 +318,8 @@ void starch_magnitude_power_uc8_twopass_x86_avx2 ( const uc8_t * arg0, uint16_t void starch_magnitude_power_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_iqdata_conv_uc8_arith_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +int starch_upsample12_generic_x86_avx2 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_generic_x86_avx2 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); void starch_magnitude_sc16_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_iqdata_conv_sc16_arith_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); @@ -293,6 +335,8 @@ void starch_count_above_u16_generic_x86_avx2 ( const uint16_t * arg0, unsigned a void starch_magnitude_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_exact_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); +int starch_upsample12_avx2_x86_avx2 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); +int starch_locate_preamble_avx2_x86_avx2 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); #endif /* STARCH_FLAVOR_X86_AVX2 */ int starch_read_wisdom (const char * path); diff --git a/dsp/helpers/demod2400x_dsp.h b/dsp/helpers/demod2400x_dsp.h new file mode 100644 index 000000000..edede0540 --- /dev/null +++ b/dsp/helpers/demod2400x_dsp.h @@ -0,0 +1,45 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod2400x_dsp.h: constants shared by the DEMOD_2400X starch kernels +// (dsp/impl/demod2400x.*.c) and demod_2400x.c +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. + +#ifndef DEMOD2400X_DSP_H +#define DEMOD2400X_DSP_H + +#include + +#ifndef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif +#ifndef MAX +#define MAX(a,b) ((a) < (b) ? (b) : (a)) +#endif + +#define LONG_FRAME 112 +#define SHORT_FRAME 56 + +#define NU_SAMP(t) (5*((t)-7)) +#define NU_STRIDE(t) ((((t)+5)/6+7)&~7) +#define METRIC_SIZE(t) ((t)-15-12*LONG_FRAME) + +#define PREAMBLE_KK 2 +#define PREAMBLE_MIN_PULSE + +#define OVERLAP_WINDOW \ + (7 + ((15 + 1 + 2*LONG_FRAME + 4) / 5)) + +// 2.4 -> 12 MHz polyphase FIR, 5 phases x 8 taps, gain 128 per phase +static const int8_t upsample_coeffs[5][8] = { + { -1, 4, -10, 125, 14, -6, 3, -1 }, // phase=0 sum=128 energy=15984 gd=3.0938 + { -3, 8, -21, 109, 47, -16, 7, -3 }, // phase=1 sum=128 energy=14918 gd=3.2969 + { -4, 10, -23, 81, 81, -23, 10, -4 }, // phase=2 sum=128 energy=14412 gd=3.5000 + { -3, 7, -16, 47, 109, -21, 8, -3 }, // phase=3 sum=128 energy=14918 gd=3.7031 + { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 +}; + +#endif diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c new file mode 100644 index 000000000..a12537e2d --- /dev/null +++ b/dsp/impl/demod2400x.avx2.c @@ -0,0 +1,185 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod2400x.avx2.c: DEMOD_2400X upsampler and preamble search, x86 AVX2. +// Structured the same way as demod2400x.neon.c. +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. + +#ifndef __AVX2__ +# error AVX2 support not available +#endif + +#include + +#include "dsp/helpers/demod2400x_dsp.h" + +// Two vectors of 8 x int16: val[0] = I, val[1] = Q (same layout as NEON's int16x8x2_t) +typedef struct { + __m128i val[2]; +} iq16x8x2_avx2_t; + +// Horizontal add +static inline __attribute__((always_inline)) int32_t hsum_s32x4_avx2(const __m128i v) +{ + __m128i s = _mm_add_epi32(v, _mm_shuffle_epi32(v, _MM_SHUFFLE(1, 0, 3, 2))); + s = _mm_add_epi32(s, _mm_shuffle_epi32(s, _MM_SHUFFLE(2, 3, 0, 1))); + return _mm_cvtsi128_si32(s); +} + +// Horizontal add +static inline __attribute__((always_inline)) uint32_t hsum_u32x4_avx2(const __m128i v) +{ + return (uint32_t) hsum_s32x4_avx2(v); +} + +// Load 8 interleaved I/Q samples and deinterleave them (like NEON's vld2q_s16) +static inline __attribute__((always_inline)) iq16x8x2_avx2_t load2_s16_avx2(const int16_t *in) +{ + const __m128i deint = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); + + __m128i a = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) (in + 0)), deint); // I0-3 Q0-3 + __m128i b = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) (in + 8)), deint); // I4-7 Q4-7 + + iq16x8x2_avx2_t iq; + iq.val[0] = _mm_unpacklo_epi64(a, b); // I0-7 + iq.val[1] = _mm_unpackhi_epi64(a, b); // Q0-7 + return iq; +} + + +static inline __attribute__((always_inline)) unsigned getpow_avx2(const __m128i cv8, const iq16x8x2_avx2_t iq, int shift) +{ + const __m128i cv = _mm_cvtepi8_epi16(cv8); // widen coefficients to 16-bit + + // madd: multiply 8 pairs and add neighbours -> 4 x int32 partial sums + int32_t resi = (hsum_s32x4_avx2(_mm_madd_epi16(iq.val[0], cv)) + 16) >> 5; + int32_t resq = (hsum_s32x4_avx2(_mm_madd_epi16(iq.val[1], cv)) + 16) >> 5; + + uint32_t pwr = (uint32_t)(resi*resi) + (uint32_t)(resq*resq); + + pwr >>= shift; // Set gain + if (pwr > 65535) pwr = 65535; + + return pwr; +} + + +// Upsample +int STARCH_IMPL(upsample12, avx2) (const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) +{ + if (len < 8) return 0; + + __m128i c0 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[0]); // 8 x int8 + __m128i c1 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[1]); // 8 x int8 + __m128i c2 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[2]); // 8 x int8 + __m128i c3 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[3]); // 8 x int8 + __m128i c4 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[4]); // 8 x int8 + + iq16x8x2_avx2_t iq = load2_s16_avx2(in); + + int nt = NU_SAMP(len); // Total samples + int stride = NU_STRIDE(nt); // Stride + int stride6 = 6*stride; + int bpos = 0; + + in += 16; + len -= 8; + while (1) { + + // Rearrage output + oo[bpos] = getpow_avx2(c0, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_avx2(c1, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_avx2(c2, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_avx2(c3, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_avx2(c4, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + if (len == 0) break; + len--; + + // Shift + iq.val[0] = _mm_insert_epi16(_mm_srli_si128(iq.val[0], 2), in[0], 7); + iq.val[1] = _mm_insert_epi16(_mm_srli_si128(iq.val[1], 2), in[1], 7); + in += 2; + } + + return nt; +} + + +// Locate preamble +int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +{ + int stride = NU_STRIDE(nt); // Stride + const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; + + const __m128i gm1 = _mm_loadu_si128((const __m128i *) (mask_g + 0)); + const __m128i gm2 = _mm_loadu_si128((const __m128i *) (mask_g + 8)); + const __m128i zero = _mm_setzero_si128(); + + for (int j = 0; j < 6; j++) { // Look at each phase in turn + + const uint16_t * restrict p = &pp[j*stride]; + int plen = (nt + 5 - j) / 6; // Length of each phase + + plen -= 2 * LONG_FRAME; // No point in searching at end + if (plen < 16) continue; + + __m128i p1 = _mm_loadu_si128((const __m128i *) (p + 0)); + __m128i p2 = _mm_loadu_si128((const __m128i *) (p + 8)); + + plen -= 16; + p += 16; + uint8_t * restrict mp = &mm[j]; + + while (1) { + + // Preamble detection + // P G P G, G G G P, G P G G, G G G G + + // Extract gaps and sum together (widened to 32-bit, values are unsigned) + __m128i g1 = _mm_and_si128(p1, gm1); + __m128i g2 = _mm_and_si128(p2, gm2); + __m128i gs = _mm_add_epi32(_mm_add_epi32(_mm_unpacklo_epi16(g1, zero), _mm_unpackhi_epi16(g1, zero)), + _mm_add_epi32(_mm_unpacklo_epi16(g2, zero), _mm_unpackhi_epi16(g2, zero))); + uint32_t gap = hsum_u32x4_avx2(gs); + + // Extract pulses: low 4 lanes of px = p[0], p[9], p[2], p[7] + __m128i dm1 = _mm_andnot_si128(gm1, p1); + __m128i dm2 = _mm_andnot_si128(gm2, p2); + + __m128i px = _mm_or_si128(_mm_or_si128(dm1, _mm_srli_si128(dm1, 8)), dm2); + +#ifdef PREAMBLE_MIN_PULSE + __m128i m = _mm_min_epu16(px, _mm_srli_si128(px, 4)); + m = _mm_min_epu16(m, _mm_srli_si128(m, 2)); + unsigned minp = (unsigned) _mm_extract_epi16(m, 0); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32_t minp = hsum_u32x4_avx2(_mm_cvtepu16_epi32(px)); +#endif + // Metric + *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); + mp += 6; + + if (plen == 0) break; + plen--; + + unsigned nv = *p++; + p1 = _mm_alignr_epi8(p2, p1, 2); + p2 = _mm_insert_epi16(_mm_srli_si128(p2, 2), (int) nv, 7); + } + } + return METRIC_SIZE(nt); +} diff --git a/dsp/impl/demod2400x.benchmark.c b/dsp/impl/demod2400x.benchmark.c new file mode 100644 index 000000000..63b016120 --- /dev/null +++ b/dsp/impl/demod2400x.benchmark.c @@ -0,0 +1,123 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod2400x.benchmark.c: starch benchmarks for the DEMOD_2400X kernels. +// Every implementation is checked to be bit-exact against the plain C +// version (generic_generic) before it is timed. +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. + +#include +#include +#include + +#include "dsp/helpers/demod2400x_dsp.h" + +#define DEMOD2400X_BENCH_LEN 131072 // one FIFO buffer of I/Q samples +#define DEMOD2400X_BENCH_SHIFT 8 + +// Size of the upsample12 output (and locate_preamble input) for 'len' I/Q samples +static size_t demod2400x_upsample_size(int len) +{ + return 6 * (size_t) NU_STRIDE(NU_SAMP(len)); +} + +// 12-bit I/Q: background noise, occasional strong bursts and a few full-scale samples +static int16_t *demod2400x_make_iq(int len) +{ + int16_t *iq = malloc(2 * (size_t) len * sizeof(*iq)); + if (!iq) + return NULL; + + srand(1); + for (int n = 0; n < len; ++n) { + int amp = (rand() % 50 == 0) ? 2047 : 200; + int i = (rand() % (2 * amp + 1)) - amp; + int q = (rand() % (2 * amp + 1)) - amp; + if (rand() % 1000 == 0) { + i = -2048; + q = -2048; + } + iq[2 * n] = (int16_t) i; + iq[2 * n + 1] = (int16_t) q; + } + return iq; +} + + +void STARCH_BENCHMARK(upsample12) (void) +{ + const int len = DEMOD2400X_BENCH_LEN; + int16_t *in = demod2400x_make_iq(len); + uint16_t *out = calloc(demod2400x_upsample_size(len), sizeof(*out)); + + if (in && out) + STARCH_BENCHMARK_RUN( upsample12, in, len, DEMOD2400X_BENCH_SHIFT, out ); + + free(in); + free(out); +} + +bool STARCH_BENCHMARK_VERIFY(upsample12) (const int16_t *in, int len, int shift, uint16_t *out) +{ + size_t size = demod2400x_upsample_size(len); + uint16_t *ref = calloc(size, sizeof(*ref)); + if (!ref) + return false; + + starch_upsample12_generic_generic(in, len, shift, ref); + + bool okay = true; + for (size_t k = 0; k < size; ++k) { + if (out[k] != ref[k]) { + fprintf(stderr, "verification failed: out[%zu]=%u, expected %u\n", k, out[k], ref[k]); + okay = false; + break; + } + } + + free(ref); + return okay; +} + + +void STARCH_BENCHMARK(locate_preamble) (void) +{ + const int len = DEMOD2400X_BENCH_LEN; + int16_t *iq = demod2400x_make_iq(len); + uint16_t *pp = calloc(demod2400x_upsample_size(len), sizeof(*pp)); + uint8_t *mm = NULL; + + if (iq && pp) { + int nt = starch_upsample12_generic_generic(iq, len, DEMOD2400X_BENCH_SHIFT, pp); + if ((mm = calloc((size_t) nt, 1))) + STARCH_BENCHMARK_RUN( locate_preamble, pp, nt, mm ); + } + + free(iq); + free(pp); + free(mm); +} + +bool STARCH_BENCHMARK_VERIFY(locate_preamble) (const uint16_t *pp, int nt, uint8_t *mm) +{ + uint8_t *ref = calloc((size_t) nt, 1); + if (!ref) + return false; + + starch_locate_preamble_generic_generic(pp, nt, ref); + + bool okay = true; + for (int k = 0; k < nt; ++k) { + if (mm[k] != ref[k]) { + fprintf(stderr, "verification failed: mm[%d]=%u, expected %u\n", k, mm[k], ref[k]); + okay = false; + break; + } + } + + free(ref); + return okay; +} diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c new file mode 100644 index 000000000..434bfa6f8 --- /dev/null +++ b/dsp/impl/demod2400x.generic.c @@ -0,0 +1,160 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod2400x.generic.c: DEMOD_2400X upsampler and preamble search, plain C. +// Also built into every SIMD flavour, where the compiler may auto-vectorise it. +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. + +#include "dsp/helpers/demod2400x_dsp.h" + +// Get power +static inline unsigned getpow_generic( + const int8_t *cv, + const int16_t *ii, + const int16_t *qq, + int shift) +{ + int32_t si = + cv[0] * ii[0] + + cv[1] * ii[1] + + cv[2] * ii[2] + + cv[3] * ii[3] + + cv[4] * ii[4] + + cv[5] * ii[5] + + cv[6] * ii[6] + + cv[7] * ii[7]; + + int32_t sq = + cv[0] * qq[0] + + cv[1] * qq[1] + + cv[2] * qq[2] + + cv[3] * qq[3] + + cv[4] * qq[4] + + cv[5] * qq[5] + + cv[6] * qq[6] + + cv[7] * qq[7]; + + si = (si+16)>>5; + sq = (sq+16)>>5; + uint32_t pwr = (uint32_t)(si*si) + (uint32_t)(sq*sq); + + pwr >>= shift; // Set gain + if (pwr > 65535) pwr = 65535; + + return pwr; +} + + +// Upsample +int STARCH_IMPL(upsample12, generic) (const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) +{ + if (len < 8) + return 0; + + int16_t ii[8]; + int16_t qq[8]; + + int nt = NU_SAMP(len); + int stride = NU_STRIDE(nt); + int stride6 = 6 * stride; + int bpos = 0; + + /* Initial 8-sample window */ + for (int i = 0; i < 8; i++) { + + ii[i] = in[0]; + qq[i] = in[1]; + in += 2; + } + len -= 8; + + while (1) { + + oo[bpos] = getpow_generic(upsample_coeffs[0], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow_generic(upsample_coeffs[1], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow_generic(upsample_coeffs[2], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow_generic(upsample_coeffs[3], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + oo[bpos] = getpow_generic(upsample_coeffs[4], ii, qq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6 - 1; + + if (len == 0) + break; + + len--; + + /* Shift window left by one sample */ + for (int i = 0; i < 7; i++) { + ii[i] = ii[i + 1]; + qq[i] = qq[i + 1]; + } + + ii[7] = in[0]; + qq[7] = in[1]; + in += 2; + } + + return nt; +} + + +// Locate preamble +int STARCH_IMPL(locate_preamble, generic) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +{ + int stride = NU_STRIDE(nt); + + for (int j = 0; j < 6; j++) { + + const uint16_t *restrict p = pp + j * stride; + uint8_t *restrict mp = mm + j; + + int plen = (nt + 5 - j) / 6; + + plen -= 2 * LONG_FRAME; + + for (int i = 0; i < plen - 15; i++) { + +#ifdef PREAMBLE_MIN_PULSE + uint32_t minp = + MIN( + MIN(p[0], p[2]), + MIN(p[7], p[9]) + ); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32_t minp = p[0] + p[2] + p[7] + p[9]; +#endif + uint32_t gap = + p[ 1] + + p[ 3] + + p[ 4] + + p[ 5] + + p[ 6] + + p[ 8] + + p[10] + + p[11] + + p[12] + + p[13] + + p[14] + + p[15]; + + *mp = (256 * PREAMBLE_KK * minp) + / (PREAMBLE_KK * minp + gap + 1); + + p++; + mp += 6; + } + } + + return METRIC_SIZE(nt); +} diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c new file mode 100644 index 000000000..3dd56b733 --- /dev/null +++ b/dsp/impl/demod2400x.neon.c @@ -0,0 +1,193 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod2400x.neon.c: DEMOD_2400X upsampler and preamble search, ARM NEON. +// Built for both 32-bit (armv7a_neon_vfpv4) and 64-bit (armv8_neon_simd) ARM. +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. + +#ifndef __ARM_NEON +# error Neon support not available +#endif + +#include + +#include "dsp/helpers/demod2400x_dsp.h" + +// Horizontal add +static inline __attribute__((always_inline)) +uint32_t hsum_u32x4_neon(const uint32x4_t v) +{ +#ifdef __aarch64__ + return vaddvq_u32(v); +#else + uint32x2_t s = vadd_u32(vget_low_u32(v), + vget_high_u32(v)); + s = vpadd_u32(s, s); + return vget_lane_u32(s, 0); +#endif +} + +// Horizontal add of two vectors at once: returns [sum of a, sum of b] +static inline __attribute__((always_inline)) +int32x2_t hsum2_s32x4_neon(const int32x4_t a, const int32x4_t b) +{ +#ifdef __aarch64__ + int32x4_t ab = vpaddq_s32(a, b); // [a0+a1, a2+a3, b0+b1, b2+b3] + return vpadd_s32(vget_low_s32(ab), vget_high_s32(ab)); +#else + return vpadd_s32(vadd_s32(vget_low_s32(a), vget_high_s32(a)), + vadd_s32(vget_low_s32(b), vget_high_s32(b))); +#endif +} + + +static inline __attribute__((always_inline)) unsigned getpow_neon(const int8x8_t cv8, const int16x8x2_t iq, int shift) +{ + const int16x8_t cv = vmovl_s8(cv8); // widen coefficients to 16-bit + int32x4_t si, sq; + + si = vmull_s16(vget_low_s16(iq.val[0]), vget_low_s16(cv)); + si = vmlal_s16(si, vget_high_s16(iq.val[0]), vget_high_s16(cv)); + + sq = vmull_s16(vget_low_s16(iq.val[1]), vget_low_s16(cv)); + sq = vmlal_s16(sq, vget_high_s16(iq.val[1]), vget_high_s16(cv)); + + int32x2_t res = hsum2_s32x4_neon(si, sq); // [resi, resq] + res = vrshr_n_s32(res, 5); // (x+16)>>5 on both + + uint32x2_t sqr = vreinterpret_u32_s32(vmul_s32(res, res)); // [resi^2, resq^2] + uint32_t pwr = vget_lane_u32(sqr, 0) + vget_lane_u32(sqr, 1); + + pwr >>= shift; // Set gain + if (pwr > 65535) pwr = 65535; + + return pwr; +} + +// Upsample +int STARCH_IMPL(upsample12, neon) (const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) +{ + if (len < 8) return 0; + + int8x8_t c0 = vld1_s8(upsample_coeffs[0]); + int8x8_t c1 = vld1_s8(upsample_coeffs[1]); + int8x8_t c2 = vld1_s8(upsample_coeffs[2]); + int8x8_t c3 = vld1_s8(upsample_coeffs[3]); + int8x8_t c4 = vld1_s8(upsample_coeffs[4]); + + int16x8x2_t iq = vld2q_s16(in); + + int nt = NU_SAMP(len); // Total samples + int stride = NU_STRIDE(nt); // Stride + int stride6 = 6*stride; + int bpos = 0; + + in += 16; + len -= 8; + while (1) { + + // Rearrage output + oo[bpos] = getpow_neon(c0, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_neon(c1, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_neon(c2, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_neon(c3, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + oo[bpos] = getpow_neon(c4, iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + + if (len == 0) break; + len--; + + // Shift + iq.val[0] = vextq_s16(iq.val[0], iq.val[0], 1); + iq.val[1] = vextq_s16(iq.val[1], iq.val[1], 1); + + iq.val[0] = vsetq_lane_s16(in[0], iq.val[0], 7); + iq.val[1] = vsetq_lane_s16(in[1], iq.val[1], 7); + in += 2; + } + + return nt; +} + + +// Locate preamble +int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +{ + int stride = NU_STRIDE(nt); // Stride + const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; + + uint16x8_t gm1 = vld1q_u16((const uint16_t *)mask_g+0); + uint16x8_t gm2 = vld1q_u16((const uint16_t *)mask_g+8); + + for (int j = 0; j < 6; j++) { // Look at each phase in turn + + const uint16_t * restrict p = &pp[j*stride]; + int plen = (nt + 5 - j) / 6; // Length of each phase + + plen -= 2 * LONG_FRAME; // No point in searching at end + if (plen < 16) continue; + + uint16x8_t p1 = vld1q_u16(p + 0); + uint16x8_t p2 = vld1q_u16(p + 8); + + plen -= 16; + p += 16; + uint8_t * restrict mp = &mm[j]; + + while (1) { + + // Preamble detection + // P G P G, G G G P, G P G G, G G G G + + // Extract gaps and sum together + uint32x4_t g1 = vpaddlq_u16(vandq_u16(p1, gm1)); + uint32x4_t g2 = vpaddlq_u16(vandq_u16(p2, gm2)); + uint32x4_t gs = vaddq_u32(g1, g2); + uint32_t gap = hsum_u32x4_neon(gs); + + // Extract pulses + uint16x8_t dm1 = vbicq_u16(p1, gm1); + uint16x8_t dm2 = vbicq_u16(p2, gm2); + + uint16x4_t d0 = vget_low_u16(dm1); + uint16x4_t d1 = vget_high_u16(dm1); + uint16x4_t d2 = vget_low_u16(dm2); + + uint16x4_t px = vorr_u16(vorr_u16(d0, d1), d2); + +#ifdef PREAMBLE_MIN_PULSE + uint16x4_t m = vpmin_u16(px, px); + m = vpmin_u16(m, m); + unsigned minp = vget_lane_u16(m, 0); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32x2_t s = vpaddl_u16(px); + s = vpadd_u32(s, s); + uint32_t minp = vget_lane_u32(s, 0); +#endif + // Metric + *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); + mp += 6; + + if (plen == 0) break; + plen--; + + unsigned nv = *p++; + p1 = vextq_u16(p1, p2, 1); + p2 = vextq_u16(p2, p2, 1); + p2 = vsetq_lane_u16(nv, p2, 7); + } + } + return METRIC_SIZE(nt); +} diff --git a/dsp/starchgen.py b/dsp/starchgen.py index 9b7018f52..f619d90da 100755 --- a/dsp/starchgen.py +++ b/dsp/starchgen.py @@ -42,7 +42,15 @@ gen.add_function( name = 'iqdata_conv_sc16q11', argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) - +gen.add_function( + name = 'upsample12', + argtypes = ['const int16_t *', 'int', 'int', 'uint16_t *'], + returntype = 'int') +gen.add_function( + name = 'locate_preamble', + argtypes = ['const uint16_t *', 'int', 'uint8_t *'], + returntype = 'int') + gen.add_flavor( name = 'generic', description = 'Generic build, default compiler options', @@ -86,6 +94,7 @@ gen.scan_impls('dsp/impl/*.generic.c') gen.scan_impls('dsp/impl/*.neon.c', flavors=[armv7a_neon_vfpv4, armv8_neon_simd]) +gen.scan_impls('dsp/impl/*.avx2.c', flavors=['x86_avx2']) gen.scan_benchmarks('dsp/impl/*.benchmark.c') gen.generate() diff --git a/dump1090.h b/dump1090.h index 46fe55f28..4ebbf5c98 100644 --- a/dump1090.h +++ b/dump1090.h @@ -288,6 +288,7 @@ typedef enum { #include "net_io.h" #include "crc.h" #include "demod_2400.h" +#include "demod_2400x.h" #include "stats.h" #include "cpr.h" #include "icao_filter.h" diff --git a/wisdom.aarch64 b/wisdom.aarch64 index 81917273d..3077a1376 100644 --- a/wisdom.aarch64 +++ b/wisdom.aarch64 @@ -19,3 +19,13 @@ magnitude_uc8 lookup_unroll_4_generic mean_power_u16 u32_armv8_neon_simd # 45663 ns/call mean_power_u16 u64_generic # 990367 ns/call + +# DEMOD_2400X kernels: provisional order (hand-written NEON first). +# Replace with real timings: ./starch-benchmark -i 5 -o wisdom.local upsample12 locate_preamble +upsample12 neon_armv8_neon_simd +upsample12 generic_armv8_neon_simd +upsample12 generic_generic + +locate_preamble neon_armv8_neon_simd +locate_preamble generic_armv8_neon_simd +locate_preamble generic_generic diff --git a/wisdom.arm b/wisdom.arm index ab30cb7e7..8ffe8615d 100644 --- a/wisdom.arm +++ b/wisdom.arm @@ -23,3 +23,13 @@ mean_power_u16 u64_generic count_above_u16 neon_armv7a_neon_vfpv4 # 35 ns/call count_above_u16 generic_generic # 178 ns/call + +# DEMOD_2400X kernels: provisional order (hand-written NEON first). +# Replace with real timings: ./starch-benchmark -i 5 -o wisdom.local upsample12 locate_preamble +upsample12 neon_armv7a_neon_vfpv4 +upsample12 generic_armv7a_neon_vfpv4 +upsample12 generic_generic + +locate_preamble neon_armv7a_neon_vfpv4 +locate_preamble generic_armv7a_neon_vfpv4 +locate_preamble generic_generic diff --git a/wisdom.x86 b/wisdom.x86 index d1d632084..4d3d41e4c 100644 --- a/wisdom.x86 +++ b/wisdom.x86 @@ -23,3 +23,12 @@ mean_power_u16 u32_generic count_above_u16 generic_x86_avx2 # 20 ns/call count_above_u16 generic_generic # 30 ns/call + +# DEMOD_2400X kernels, measured with starch-benchmark -i 5 (x86-64 with AVX2) +upsample12 avx2_x86_avx2 # 1877779 ns/call +upsample12 generic_generic # 3290448 ns/call +upsample12 generic_x86_avx2 # 3709713 ns/call + +locate_preamble generic_x86_avx2 # 2140715 ns/call +locate_preamble generic_generic # 2293071 ns/call +locate_preamble avx2_x86_avx2 # 2808931 ns/call From ae948e63ac552285c993224dd7eb291aa54da584 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Sun, 18 Oct 2026 16:54:07 +0100 Subject: [PATCH 04/25] initial working version --- Makefile | 2 +- demod_2400x.c | 435 +++++++++++++++++++++++++++++++- demod_2400x.h | 6 + dsp/helpers/demod2400x_coeffs.h | 18 ++ dsp/helpers/demod2400x_dsp.h | 14 +- dsp/impl/demod2400x.avx2.c | 1 + dsp/impl/demod2400x.generic.c | 1 + dsp/impl/demod2400x.neon.c | 1 + dump1090.c | 4 + 9 files changed, 464 insertions(+), 18 deletions(-) create mode 100644 dsp/helpers/demod2400x_coeffs.h diff --git a/Makefile b/Makefile index ceabba9f3..d3b2e7b04 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PROGNAME=dump1090 DUMP1090_VERSION ?= unknown CFLAGS ?= -O3 -g -DUMP1090_CFLAGS := -std=c11 -fno-common -Wall -Wmissing-declarations -Werror -Wformat-signedness -W +DUMP1090_CFLAGS := -std=c11 -fno-common -Wall -Wmissing-declarations -Wformat-signedness -W DUMP1090_CPPFLAGS := -I. -D_POSIX_C_SOURCE=200112L -DMODES_DUMP1090_VERSION=\"$(DUMP1090_VERSION)\" -DMODES_DUMP1090_VARIANT=\"dump1090-fa\" DUMP1090_CPPFLAGS += -DDEMOD_2400X diff --git a/demod_2400x.c b/demod_2400x.c index b20e05e4b..7c17189bd 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -1,8 +1,6 @@ // Part of dump1090, a Mode S message decoder for RTLSDR devices. // -// demod_2400.c: 2.4MHz Mode S demodulator. -// -// Copyright (c) 2014,2015 Oliver Jowett +// demod_2400x.c: 2.4MHz Mode S demodulator working on I/Q samples // // This file is free software: you may copy, redistribute and/or modify it // under the terms of the GNU General Public License as published by the @@ -19,7 +17,436 @@ #include "dump1090.h" +#ifdef DEMOD_2400X + +#include + +#include "dsp/helpers/demod2400x_dsp.h" + +// Static data +struct staticdata_t { + + int isize; + int16_t *idata; // DC-removed I/Q, laid out like the FIFO buffer: [overlap | new samples] + + int usize; + uint16_t *udata; + + int msize; + uint8_t *metric; + + int32_t offset_i; + int32_t offset_q; + + unsigned prev_mlen; // new samples in the previous buffer (0 = no previous buffer) + int next_start; // first peak position to search in this buffer (skips the rest of a + // frame decoded at the end of the previous buffer) +}; + +// One preamble candidate +struct candidate { + int pos; // metric index = 12 MHz sample index from mag->data[0] + unsigned quality; // data quality of the first 56 bits (0..256) + uint8_t msg[MODES_LONG_MSG_BYTES]; // demodulated frame +}; + +// The metric index is this many 12 MHz samples (3 input samples, the FIR delay) +// before the true start of the preamble +#define UPSAMPLE_DELAY 15 + +#define PEAMBLE_THRES 80 +#define DATA_THRES (128+40) + + +// Allocate holding buffers +static int allocBuffer(int nsize, int *osize, void **bptr) { + + void *nptr; + if (nsize > *osize) { + + if ((nptr = realloc(*bptr, nsize)) == NULL) { + + free(*bptr); + *bptr = NULL; + *osize = 0; + return -1; + } + + *bptr = nptr; + *osize = nsize; + } + + return 0; +} + + +// Allocate buffers +static int allocateBuffers(struct staticdata_t *stdata, int totalLength) { + + int nt = NU_SAMP(totalLength); + int stride = NU_STRIDE(nt); + int mt = METRIC_SIZE(nt); + + if (allocBuffer(2*totalLength*sizeof(int16_t), &stdata->isize, (void **)&stdata->idata) < 0) return -1; + if (allocBuffer(6*stride*sizeof(uint16_t), &stdata->usize, (void **)&stdata->udata) < 0) return -1; + if (allocBuffer(mt*sizeof(uint8_t), &stdata->msize, (void **)&stdata->metric) < 0) return -1; + + return 0; +} + + +//#define CHECK_METRIC +#ifdef CHECK_METRIC +static void checkMetric(const uint8_t *metric, int mlen) { + + unsigned max_noise = 0; + unsigned sum_noise = 0; + for (int i = 0; i < mlen>>1; i++) { + + if (metric[i] > max_noise) + max_noise = metric[i]; + + sum_noise += metric[i]; + } + unsigned peak1 = 0, peak2 = 0; + int pos1 = 0, pos2 = 0; + + /* Find largest peak */ + for (int i = mlen >> 1; i < mlen; i++) { + if (metric[i] > peak1) { + peak1 = metric[i]; + pos1 = i; + } + } + + /* Find largest peak sufficiently far away */ + #define SEP 6 + for (int i = mlen >> 1; i < mlen; i++) { + if ((i > pos1 ? i - pos1 : pos1 - i) < SEP) + continue; + if (metric[i] > peak2) { + peak2 = metric[i]; + pos2 = i; + } + } + + fprintf(stderr, "noise=%u avg=%u p1=%u@%d p2=%u@%d\n" + "nr=%.2f fr=%.2f\n", + max_noise, + sum_noise/(unsigned)(mlen>>1), + peak1, pos1, + peak2, pos2, + (double)peak1/max_noise, + (double)peak1/peak2); +} +#endif + + +// Decode 1 bit +#define DECODE_BIT(pp, ofs, bt, qq, pw) ({ \ + int aa = (pp)[2*(ofs)+0]; \ + int bb = (pp)[2*(ofs)+1]; \ + int dd = bb - aa; \ + if (dd < 0) { \ + dd = -dd; \ + bt |= (1<<(7-ofs)); \ + } \ + qq += dd; \ + pw += aa+bb; \ +}) + + +// Decode 56 bits starting at metric position 'pos'; returns a quality score (0..256) +static unsigned decodeShortAdbsData(int pos, const uint16_t * restrict udata, int nt, uint8_t * restrict msg) { + + int stride = NU_STRIDE(nt); + int phase = pos % 6; + int offs = (pos / 6) + 16; // Start of data + uint32_t qual = 0; + uint32_t pow = 0; + + const uint16_t * restrict p = &udata[phase * stride + offs]; + for (int i = 0; i < SHORT_FRAME / 8; i++) { // 7 bytes = 56 bits, 16 chips per byte + + unsigned bit = 0; + + DECODE_BIT(p, 0, bit, qual, pow); + DECODE_BIT(p, 1, bit, qual, pow); + DECODE_BIT(p, 2, bit, qual, pow); + DECODE_BIT(p, 3, bit, qual, pow); + DECODE_BIT(p, 4, bit, qual, pow); + DECODE_BIT(p, 5, bit, qual, pow); + DECODE_BIT(p, 6, bit, qual, pow); + DECODE_BIT(p, 7, bit, qual, pow); + + msg[i] = bit; + p += 16; + } + unsigned score = (256 * qual) / (pow + 1); + return score; +} + + +// Is metric position 'p' a preamble peak? +static inline int isPeak(const uint8_t *metric, int p) +{ + unsigned m0 = metric[p - 1], m1 = metric[p], m2 = metric[p + 1]; + + return m1 >= PEAMBLE_THRES && m1 >= m0 && m1 > m2; +} + + +// Try the peak and its two neighbours, keep the one whose first 56 bits look best +static void bestPhase(int peak, const uint16_t *udata, int nt, struct candidate *c) +{ + uint8_t msg[MODES_SHORT_MSG_BYTES]; + + for (int p = peak - 1; p <= peak + 1; p++) { + unsigned q = decodeShortAdbsData(p, udata, nt, msg); + if (p == peak - 1 || q > c->quality) { + c->pos = p; + c->quality = q; + memcpy(c->msg, msg, sizeof(msg)); + } + } +} + + +// Demodulate the rest of the frame if the DF says it is a long one. +// Returns the frame length in bits, or 0 if the second half doesn't hold up. +static int demodFrame(struct candidate *c, const uint16_t *udata, int nt) +{ + int bits = modesMessageLenByType(c->msg[0] >> 3); + + if (bits == MODES_LONG_MSG_BITS && + decodeShortAdbsData(c->pos + 12 * SHORT_FRAME, udata, nt, c->msg + MODES_SHORT_MSG_BYTES) < DATA_THRES) + return 0; + + return bits; +} + + +// Signal power over the frame's data bits, as a fraction of full scale +// (same scale as mag->mean_power). *energy gets the total, for the noise estimate. +static double signalPower(const int16_t *iq, int pos, int bits, double *energy) +{ + int start = (pos + UPSAMPLE_DELAY) / 5 + 19; // skip the 8 us preamble (19.2 samples) + int len = bits * 12 / 5; // 2.4 samples per bit + uint64_t sum = 0; + + for (int k = start; k < start + len; k++) { + int32_t I = iq[2 * k], Q = iq[2 * k + 1]; + sum += (uint32_t) (I * I + Q * Q); + } + + *energy = sum / 2048.0 / 2048.0; + Modes.stats_current.signal_power_sum += *energy; + Modes.stats_current.signal_power_count += len; + + return *energy / len; +} + + +// Score, decode and pass one frame on to the rest of dump1090. +// Returns 1 if it was accepted; *energy gets its signal energy. +static int emitFrame(const struct mag_buf *mag, const struct candidate *c, int bits, + const int16_t *iq, double *energy) +{ + static const struct modesMessage zeroMessage; + struct modesMessage mm; + + int score = scoreModesMessage(c->msg); + if (score < SR_ACCEPT_THRESHOLD) { + if (score >= SR_UNKNOWN_THRESHOLD) + Modes.stats_current.demod_rejected_unknown_icao++; + else + Modes.stats_current.demod_rejected_bad++; + return 0; + } + + mm = zeroMessage; + + // Timestamp at the end of bit 56, even for 112-bit frames (Beast / Radarcape convention) + mm.timestampMsg = mag->sampleTimestamp + c->pos + UPSAMPLE_DELAY + (8 + 56) * 12; + mm.sysTimestampMsg = mag->sysTimestamp + receiveclock_ms_elapsed(mag->sampleTimestamp, mm.timestampMsg); + mm.score = score; + + if (decodeModesMessage(&mm, c->msg) < 0) { + Modes.stats_current.demod_rejected_bad++; + return 0; + } + Modes.stats_current.demod_accepted[mm.correctedbits]++; + + mm.signalLevel = signalPower(iq, c->pos, bits, energy); + if (mm.signalLevel > Modes.stats_current.peak_signal_power) + Modes.stats_current.peak_signal_power = mm.signalLevel; + if (mm.signalLevel > 0.50119) + Modes.stats_current.strong_signal_count++; // above -3 dBFS + + useModesMessage(&mm); + return 1; +} + + +// Search the metric for preambles and decode them. Only peaks at positions +// 1..limit belong to this buffer; later ones are in the next buffer's overlap +// and are searched there, so nothing is decoded twice. +// Returns the total signal energy of the accepted frames. +static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, int mlen, int nt) +{ + int limit = 5 * (int) (mag->validLength - mag->overlap); + int last = MIN(limit, mlen - 2); + double energy_sum = 0; + int p; + + for (p = MAX(1, st->next_start); p <= last; p++) { + + if (!isPeak(st->metric, p)) + continue; + Modes.stats_current.demod_preambles++; + + struct candidate c; + bestPhase(p, st->udata, nt, &c); + if (c.quality < DATA_THRES) + continue; + + int bits = demodFrame(&c, st->udata, nt); + if (!bits) + continue; + + double energy; + if (emitFrame(mag, &c, bits, st->idata, &energy)) { + energy_sum += energy; + // Skip to 8 bits before the end of the frame, as demod_2400.c does + p = c.pos + bits * 12 - 1; + } + } + + // A skip that runs past 'limit' carries on into the next buffer, + // whose position 0 is this buffer's 'limit' + st->next_start = MAX(1, p - limit); + return energy_sum; +} + + +#define AVG_SHIFT 12 +// Remove DC from 'dlen' new I/Q samples, 'in' -> 'out' (both interleaved I,Q). +// Returns the gain shift for upsample12. +static int removeDcOffset(const int16_t * restrict in, + int16_t * restrict out, + struct staticdata_t * restrict stdata, + int dlen) +{ + int32_t off_i = stdata->offset_i; + int32_t off_q = stdata->offset_q; + uint64_t pow = 0; + + if (dlen <= 0) return 0; + + for (int i = 0; i < dlen; i++) { + + int32_t ii = in[0]; + int32_t qq = in[1]; + + ii -= (off_i + (1<<(AVG_SHIFT-1))) >> AVG_SHIFT; + qq -= (off_q + (1<<(AVG_SHIFT-1))) >> AVG_SHIFT; + + off_i += ii; + off_q += qq; + + if (ii > 2047) ii = 2047; + if (ii < -2048) ii = -2048; + + if (qq > 2047) qq = 2047; + if (qq < -2048) qq = -2048; + + out[0] = (int16_t) ii; + out[1] = (int16_t) qq; + + // Accumulate power + pow += (uint32_t)(ii*ii) + (uint32_t)(qq*qq); + + in += 2; + out += 2; + } + + stdata->offset_i = off_i; + stdata->offset_q = off_q; + + // Calculate a shift to try and scale the power so it fits in a uint16_t + uint32_t mean = (uint32_t) (pow / (unsigned) dlen); + if (mean == 0) mean = 1; + int shift = 31 - __builtin_clz(mean); + shift += 4; // check: puts the mean at ~16 >> 8, see notes + if (shift < 0) shift = 0; + + return shift; +} + + +// Demodulate 2400 +//#define CHECK_USAMP +//#define CHECK_POW +// mag->data contains signed 12-bit I/Q sample pairs (-2048..2047): [overlap | new samples] void demodulate2400x(struct mag_buf *mag) { - MODES_NOTUSED(mag); // remove once the body uses mag + static struct staticdata_t stdata; + + if (allocateBuffers(&stdata, mag->totalLength) < 0) + return; + + unsigned overlap = mag->overlap; + unsigned dlen = mag->validLength; // whole buffer, overlap included + unsigned newlen = dlen - overlap; // new samples in this buffer + + assert(overlap >= OVERLAP_WINDOW); + + // This buffer's overlap = the filtered tail of the previous buffer, + // the same way the FIFO does it for the raw samples + if (!stdata.prev_mlen || (mag->flags & MAGBUF_DISCONTINUOUS)) { + memset(stdata.idata, 0, 2 * overlap * sizeof(int16_t)); + stdata.next_start = 0; + } else { + memmove(stdata.idata, stdata.idata + 2 * stdata.prev_mlen, 2 * overlap * sizeof(int16_t)); + } + + // Remove DC offset from the new samples only + int pshift = removeDcOffset((const int16_t *) &mag->data[2 * overlap], + stdata.idata + 2 * overlap, &stdata, (int) newlen); + stdata.prev_mlen = newlen; + + // Upsample the whole buffer + int ulen = starch_upsample12(stdata.idata, (int) dlen, pshift, stdata.udata); + +#ifdef CHECK_POW + int max = 0; + for (int i = 0; i < 6 * NU_STRIDE(ulen); i++) { + int mm = stdata.udata[i]; + if (mm > max) max = mm; + } + fprintf(stderr, "pshift = %d, Max Power = %d\n", pshift, max); +#endif +#ifdef CHECK_USAMP + int stride = NU_STRIDE(ulen); + for (int i = 0; i < ulen; i++) { + int phase = i % 6; + int pos = i / 6; + fprintf(stderr, "%4d %u\n", i, stdata.udata[phase * stride + pos]); + } +#endif + + // Locate possible preambles + int mlen = starch_locate_preamble(stdata.udata, ulen, stdata.metric); +#ifdef CHECK_METRIC + checkMetric(stdata.metric, mlen); +#endif + + // Decode the frames this buffer owns + double signal_energy = searchFrames(mag, &stdata, mlen, ulen); + + // Noise power = everything in the new samples that wasn't a decoded frame + Modes.stats_current.noise_power_sum += mag->mean_power * newlen - signal_energy; + Modes.stats_current.noise_power_count += newlen; } + +#endif /* DEMOD_2400X */ diff --git a/demod_2400x.h b/demod_2400x.h index 0ba7d3727..f02631778 100644 --- a/demod_2400x.h +++ b/demod_2400x.h @@ -1,6 +1,12 @@ #ifndef DEMOD_2400X_H #define DEMOD_2400X_H +// Input samples (2.4 MHz) a buffer must reach past its last searched position: +// 15 + 1 + 2*LONG_FRAME chips (preamble window, +1 for the peak search, long frame), +// x6/5 to convert 2 MHz chips to 2.4 MHz samples (rounded up), plus 7 for the 8-tap FIR history. +#define OVERLAP_WINDOW \ + (7 + ((15 + 1 + 2*LONG_FRAME) * 6 + 4) / 5) + extern void demodulate2400x(struct mag_buf *mbuf); #endif diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h new file mode 100644 index 000000000..35a3d6039 --- /dev/null +++ b/dsp/helpers/demod2400x_coeffs.h @@ -0,0 +1,18 @@ +// demod2400x_coeffs.h: upsampler coefficients for the DEMOD_2400X starch kernels +// (dsp/impl/demod2400x.*.c only) + +#ifndef DEMOD2400X_COEFFS_H +#define DEMOD2400X_COEFFS_H + +#include + +// 2.4 -> 12 MHz polyphase FIR, 5 phases x 8 taps, gain 128 per phase +static const int8_t upsample_coeffs[5][8] = { + { -1, 4, -10, 125, 14, -6, 3, -1 }, // phase=0 sum=128 energy=15984 gd=3.0938 + { -3, 8, -21, 109, 47, -16, 7, -3 }, // phase=1 sum=128 energy=14918 gd=3.2969 + { -4, 10, -23, 81, 81, -23, 10, -4 }, // phase=2 sum=128 energy=14412 gd=3.5000 + { -3, 7, -16, 47, 109, -21, 8, -3 }, // phase=3 sum=128 energy=14918 gd=3.7031 + { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 +}; + +#endif diff --git a/dsp/helpers/demod2400x_dsp.h b/dsp/helpers/demod2400x_dsp.h index edede0540..9b5bb14fe 100644 --- a/dsp/helpers/demod2400x_dsp.h +++ b/dsp/helpers/demod2400x_dsp.h @@ -25,21 +25,9 @@ #define NU_SAMP(t) (5*((t)-7)) #define NU_STRIDE(t) ((((t)+5)/6+7)&~7) -#define METRIC_SIZE(t) ((t)-15-12*LONG_FRAME) +#define METRIC_SIZE(t) ((t)-6*15-12*LONG_FRAME) #define PREAMBLE_KK 2 #define PREAMBLE_MIN_PULSE -#define OVERLAP_WINDOW \ - (7 + ((15 + 1 + 2*LONG_FRAME + 4) / 5)) - -// 2.4 -> 12 MHz polyphase FIR, 5 phases x 8 taps, gain 128 per phase -static const int8_t upsample_coeffs[5][8] = { - { -1, 4, -10, 125, 14, -6, 3, -1 }, // phase=0 sum=128 energy=15984 gd=3.0938 - { -3, 8, -21, 109, 47, -16, 7, -3 }, // phase=1 sum=128 energy=14918 gd=3.2969 - { -4, 10, -23, 81, 81, -23, 10, -4 }, // phase=2 sum=128 energy=14412 gd=3.5000 - { -3, 7, -16, 47, 109, -21, 8, -3 }, // phase=3 sum=128 energy=14918 gd=3.7031 - { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 -}; - #endif diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index a12537e2d..e7f1fdcd8 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -15,6 +15,7 @@ #include #include "dsp/helpers/demod2400x_dsp.h" +#include "dsp/helpers/demod2400x_coeffs.h" // Two vectors of 8 x int16: val[0] = I, val[1] = Q (same layout as NEON's int16x8x2_t) typedef struct { diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c index 434bfa6f8..d6133e3fb 100644 --- a/dsp/impl/demod2400x.generic.c +++ b/dsp/impl/demod2400x.generic.c @@ -9,6 +9,7 @@ // option) any later version. #include "dsp/helpers/demod2400x_dsp.h" +#include "dsp/helpers/demod2400x_coeffs.h" // Get power static inline unsigned getpow_generic( diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 3dd56b733..42f7f2f30 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -15,6 +15,7 @@ #include #include "dsp/helpers/demod2400x_dsp.h" +#include "dsp/helpers/demod2400x_coeffs.h" // Horizontal add static inline __attribute__((always_inline)) diff --git a/dump1090.c b/dump1090.c index 2540790fe..d03989668 100644 --- a/dump1090.c +++ b/dump1090.c @@ -902,7 +902,11 @@ int main(int argc, char **argv) { // Process one buffer start_cpu_timing(&start_time); +#ifdef DEMOD_2400X + demodulate2400x(buf); +#else demodulate2400(buf); +#endif if (Modes.mode_ac) { demodulate2400AC(buf); } From 307f2cd9ea3be83953cef1b1f18b352215e14b1c Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Sun, 18 Oct 2026 18:24:05 +0100 Subject: [PATCH 05/25] added magnitude rather than power detection --- Makefile | 2 +- demod_2400x.c | 60 ++++++++++++---------- dsp/helpers/demod2400x_coeffs.h | 89 +++++++++++++++++++++++++++++++++ dsp/impl/demod2400x.avx2.c | 5 ++ dsp/impl/demod2400x.generic.c | 5 ++ dsp/impl/demod2400x.neon.c | 5 ++ 6 files changed, 139 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index d3b2e7b04..7bcbd18d5 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ DUMP1090_VERSION ?= unknown CFLAGS ?= -O3 -g DUMP1090_CFLAGS := -std=c11 -fno-common -Wall -Wmissing-declarations -Wformat-signedness -W DUMP1090_CPPFLAGS := -I. -D_POSIX_C_SOURCE=200112L -DMODES_DUMP1090_VERSION=\"$(DUMP1090_VERSION)\" -DMODES_DUMP1090_VARIANT=\"dump1090-fa\" -DUMP1090_CPPFLAGS += -DDEMOD_2400X +DUMP1090_CPPFLAGS += -DDEMOD_2400X -DUSE_MAGNITUDE LIBS = -lpthread -lm SDR_OBJ = cpu.o sdr.o fifo.o sdr_ifile.o dsp/helpers/tables.o diff --git a/demod_2400x.c b/demod_2400x.c index 7c17189bd..f6575740e 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -23,6 +23,18 @@ #include "dsp/helpers/demod2400x_dsp.h" +// The metric index is this many 12 MHz samples (3 input samples, the FIR delay) +// before the true start of the preamble +#define UPSAMPLE_DELAY 15 + +#ifdef USE_MAGNITUDE +#define PEAMBLE_THRES 50 +#define DATA_THRES 80 +#else +#define PEAMBLE_THRES 80 +#define DATA_THRES 150 +#endif + // Static data struct staticdata_t { @@ -50,13 +62,6 @@ struct candidate { uint8_t msg[MODES_LONG_MSG_BYTES]; // demodulated frame }; -// The metric index is this many 12 MHz samples (3 input samples, the FIR delay) -// before the true start of the preamble -#define UPSAMPLE_DELAY 15 - -#define PEAMBLE_THRES 80 -#define DATA_THRES (128+40) - // Allocate holding buffers static int allocBuffer(int nsize, int *osize, void **bptr) { @@ -329,19 +334,16 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i } -#define AVG_SHIFT 12 // Remove DC from 'dlen' new I/Q samples, 'in' -> 'out' (both interleaved I,Q). // Returns the gain shift for upsample12. -static int removeDcOffset(const int16_t * restrict in, +#define AVG_SHIFT 12 +static void removeDcOffset(const int16_t * restrict in, int16_t * restrict out, struct staticdata_t * restrict stdata, int dlen) { int32_t off_i = stdata->offset_i; int32_t off_q = stdata->offset_q; - uint64_t pow = 0; - - if (dlen <= 0) return 0; for (int i = 0; i < dlen; i++) { @@ -363,26 +365,26 @@ static int removeDcOffset(const int16_t * restrict in, out[0] = (int16_t) ii; out[1] = (int16_t) qq; - // Accumulate power - pow += (uint32_t)(ii*ii) + (uint32_t)(qq*qq); - in += 2; out += 2; } stdata->offset_i = off_i; stdata->offset_q = off_q; - - // Calculate a shift to try and scale the power so it fits in a uint16_t - uint32_t mean = (uint32_t) (pow / (unsigned) dlen); - if (mean == 0) mean = 1; - int shift = 31 - __builtin_clz(mean); - shift += 4; // check: puts the mean at ~16 >> 8, see notes - if (shift < 0) shift = 0; - - return shift; } +#ifndef USE_MAGNITUDE +#define DEMOD_SHIFT_ADJ 3 +static int powerShift(double mean_power) // mean_power: fraction of 12-bit full scale +{ + if (!(mean_power > 0)) + return 0; + int e; + frexp(mean_power, &e); // mean_power = m * 2^e, 0.5 <= m < 1 + int shift = (e - 1) + 22 + DEMOD_SHIFT_ADJ; // 2048^2 = 2^22 + return shift < 0 ? 0 : shift; +} +#endif // Demodulate 2400 //#define CHECK_USAMP @@ -411,8 +413,14 @@ void demodulate2400x(struct mag_buf *mag) } // Remove DC offset from the new samples only - int pshift = removeDcOffset((const int16_t *) &mag->data[2 * overlap], - stdata.idata + 2 * overlap, &stdata, (int) newlen); + removeDcOffset((const int16_t *) &mag->data[2 * overlap], + stdata.idata + 2 * overlap, &stdata, (int) newlen); + +#ifdef USE_MAGNITUDE + int pshift = 0; // magnitude needs no gain shift +#else + int pshift = powerShift(mag->mean_power); +#endif stdata.prev_mlen = newlen; // Upsample the whole buffer diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h index 35a3d6039..9ec1927f5 100644 --- a/dsp/helpers/demod2400x_coeffs.h +++ b/dsp/helpers/demod2400x_coeffs.h @@ -15,4 +15,93 @@ static const int8_t upsample_coeffs[5][8] = { { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 }; +// floor(sqrt(m) * 128) for m = 0..1023 (max 4088, fits uint16) +static const uint16_t demod_sqrt_table[1024] = { + 0, 128, 181, 221, 256, 286, 313, 338, 362, 384, 404, 424, 443, 461, 478, 495, + 512, 527, 543, 557, 572, 586, 600, 613, 627, 640, 652, 665, 677, 689, 701, 712, + 724, 735, 746, 757, 768, 778, 789, 799, 809, 819, 829, 839, 849, 858, 868, 877, + 886, 896, 905, 914, 923, 931, 940, 949, 957, 966, 974, 983, 991, 999, 1007, 1015, + 1024, 1031, 1039, 1047, 1055, 1063, 1070, 1078, 1086, 1093, 1101, 1108, 1115, 1123, 1130, 1137, + 1144, 1152, 1159, 1166, 1173, 1180, 1187, 1193, 1200, 1207, 1214, 1221, 1227, 1234, 1241, 1247, + 1254, 1260, 1267, 1273, 1280, 1286, 1292, 1299, 1305, 1311, 1317, 1324, 1330, 1336, 1342, 1348, + 1354, 1360, 1366, 1372, 1378, 1384, 1390, 1396, 1402, 1408, 1413, 1419, 1425, 1431, 1436, 1442, + 1448, 1453, 1459, 1465, 1470, 1476, 1481, 1487, 1492, 1498, 1503, 1509, 1514, 1519, 1525, 1530, + 1536, 1541, 1546, 1551, 1557, 1562, 1567, 1572, 1578, 1583, 1588, 1593, 1598, 1603, 1608, 1614, + 1619, 1624, 1629, 1634, 1639, 1644, 1649, 1654, 1659, 1664, 1668, 1673, 1678, 1683, 1688, 1693, + 1698, 1702, 1707, 1712, 1717, 1722, 1726, 1731, 1736, 1740, 1745, 1750, 1755, 1759, 1764, 1768, + 1773, 1778, 1782, 1787, 1792, 1796, 1801, 1805, 1810, 1814, 1819, 1823, 1828, 1832, 1837, 1841, + 1846, 1850, 1854, 1859, 1863, 1868, 1872, 1876, 1881, 1885, 1889, 1894, 1898, 1902, 1907, 1911, + 1915, 1920, 1924, 1928, 1932, 1936, 1941, 1945, 1949, 1953, 1958, 1962, 1966, 1970, 1974, 1978, + 1982, 1987, 1991, 1995, 1999, 2003, 2007, 2011, 2015, 2019, 2023, 2027, 2031, 2035, 2039, 2043, + 2048, 2051, 2055, 2059, 2063, 2067, 2071, 2075, 2079, 2083, 2087, 2091, 2095, 2099, 2103, 2107, + 2111, 2114, 2118, 2122, 2126, 2130, 2134, 2138, 2141, 2145, 2149, 2153, 2157, 2160, 2164, 2168, + 2172, 2176, 2179, 2183, 2187, 2191, 2194, 2198, 2202, 2205, 2209, 2213, 2217, 2220, 2224, 2228, + 2231, 2235, 2239, 2242, 2246, 2250, 2253, 2257, 2260, 2264, 2268, 2271, 2275, 2278, 2282, 2286, + 2289, 2293, 2296, 2300, 2304, 2307, 2311, 2314, 2318, 2321, 2325, 2328, 2332, 2335, 2339, 2342, + 2346, 2349, 2353, 2356, 2360, 2363, 2367, 2370, 2374, 2377, 2380, 2384, 2387, 2391, 2394, 2398, + 2401, 2404, 2408, 2411, 2415, 2418, 2421, 2425, 2428, 2432, 2435, 2438, 2442, 2445, 2448, 2452, + 2455, 2458, 2462, 2465, 2468, 2472, 2475, 2478, 2482, 2485, 2488, 2491, 2495, 2498, 2501, 2505, + 2508, 2511, 2514, 2518, 2521, 2524, 2527, 2531, 2534, 2537, 2540, 2543, 2547, 2550, 2553, 2556, + 2560, 2563, 2566, 2569, 2572, 2575, 2579, 2582, 2585, 2588, 2591, 2594, 2598, 2601, 2604, 2607, + 2610, 2613, 2616, 2620, 2623, 2626, 2629, 2632, 2635, 2638, 2641, 2644, 2648, 2651, 2654, 2657, + 2660, 2663, 2666, 2669, 2672, 2675, 2678, 2681, 2684, 2688, 2691, 2694, 2697, 2700, 2703, 2706, + 2709, 2712, 2715, 2718, 2721, 2724, 2727, 2730, 2733, 2736, 2739, 2742, 2745, 2748, 2751, 2754, + 2757, 2760, 2763, 2766, 2769, 2772, 2774, 2777, 2780, 2783, 2786, 2789, 2792, 2795, 2798, 2801, + 2804, 2807, 2810, 2813, 2816, 2818, 2821, 2824, 2827, 2830, 2833, 2836, 2839, 2842, 2844, 2847, + 2850, 2853, 2856, 2859, 2862, 2865, 2867, 2870, 2873, 2876, 2879, 2882, 2884, 2887, 2890, 2893, + 2896, 2899, 2901, 2904, 2907, 2910, 2913, 2916, 2918, 2921, 2924, 2927, 2930, 2932, 2935, 2938, + 2941, 2944, 2946, 2949, 2952, 2955, 2957, 2960, 2963, 2966, 2968, 2971, 2974, 2977, 2979, 2982, + 2985, 2988, 2990, 2993, 2996, 2999, 3001, 3004, 3007, 3010, 3012, 3015, 3018, 3020, 3023, 3026, + 3029, 3031, 3034, 3037, 3039, 3042, 3045, 3047, 3050, 3053, 3055, 3058, 3061, 3063, 3066, 3069, + 3072, 3074, 3077, 3079, 3082, 3085, 3087, 3090, 3093, 3095, 3098, 3101, 3103, 3106, 3109, 3111, + 3114, 3117, 3119, 3122, 3124, 3127, 3130, 3132, 3135, 3137, 3140, 3143, 3145, 3148, 3150, 3153, + 3156, 3158, 3161, 3163, 3166, 3169, 3171, 3174, 3176, 3179, 3182, 3184, 3187, 3189, 3192, 3194, + 3197, 3200, 3202, 3205, 3207, 3210, 3212, 3215, 3217, 3220, 3222, 3225, 3228, 3230, 3233, 3235, + 3238, 3240, 3243, 3245, 3248, 3250, 3253, 3255, 3258, 3260, 3263, 3265, 3268, 3270, 3273, 3275, + 3278, 3280, 3283, 3285, 3288, 3290, 3293, 3295, 3298, 3300, 3303, 3305, 3308, 3310, 3313, 3315, + 3318, 3320, 3323, 3325, 3328, 3330, 3332, 3335, 3337, 3340, 3342, 3345, 3347, 3350, 3352, 3354, + 3357, 3359, 3362, 3364, 3367, 3369, 3372, 3374, 3376, 3379, 3381, 3384, 3386, 3388, 3391, 3393, + 3396, 3398, 3401, 3403, 3405, 3408, 3410, 3413, 3415, 3417, 3420, 3422, 3425, 3427, 3429, 3432, + 3434, 3436, 3439, 3441, 3444, 3446, 3448, 3451, 3453, 3456, 3458, 3460, 3463, 3465, 3467, 3470, + 3472, 3474, 3477, 3479, 3481, 3484, 3486, 3489, 3491, 3493, 3496, 3498, 3500, 3503, 3505, 3507, + 3510, 3512, 3514, 3517, 3519, 3521, 3524, 3526, 3528, 3531, 3533, 3535, 3537, 3540, 3542, 3544, + 3547, 3549, 3551, 3554, 3556, 3558, 3561, 3563, 3565, 3567, 3570, 3572, 3574, 3577, 3579, 3581, + 3584, 3586, 3588, 3590, 3593, 3595, 3597, 3599, 3602, 3604, 3606, 3609, 3611, 3613, 3615, 3618, + 3620, 3622, 3624, 3627, 3629, 3631, 3633, 3636, 3638, 3640, 3642, 3645, 3647, 3649, 3651, 3654, + 3656, 3658, 3660, 3663, 3665, 3667, 3669, 3672, 3674, 3676, 3678, 3680, 3683, 3685, 3687, 3689, + 3692, 3694, 3696, 3698, 3700, 3703, 3705, 3707, 3709, 3712, 3714, 3716, 3718, 3720, 3723, 3725, + 3727, 3729, 3731, 3734, 3736, 3738, 3740, 3742, 3744, 3747, 3749, 3751, 3753, 3755, 3758, 3760, + 3762, 3764, 3766, 3768, 3771, 3773, 3775, 3777, 3779, 3781, 3784, 3786, 3788, 3790, 3792, 3794, + 3797, 3799, 3801, 3803, 3805, 3807, 3810, 3812, 3814, 3816, 3818, 3820, 3822, 3825, 3827, 3829, + 3831, 3833, 3835, 3837, 3840, 3842, 3844, 3846, 3848, 3850, 3852, 3854, 3857, 3859, 3861, 3863, + 3865, 3867, 3869, 3871, 3873, 3876, 3878, 3880, 3882, 3884, 3886, 3888, 3890, 3892, 3895, 3897, + 3899, 3901, 3903, 3905, 3907, 3909, 3911, 3913, 3916, 3918, 3920, 3922, 3924, 3926, 3928, 3930, + 3932, 3934, 3936, 3938, 3941, 3943, 3945, 3947, 3949, 3951, 3953, 3955, 3957, 3959, 3961, 3963, + 3965, 3968, 3970, 3972, 3974, 3976, 3978, 3980, 3982, 3984, 3986, 3988, 3990, 3992, 3994, 3996, + 3998, 4000, 4002, 4004, 4007, 4009, 4011, 4013, 4015, 4017, 4019, 4021, 4023, 4025, 4027, 4029, + 4031, 4033, 4035, 4037, 4039, 4041, 4043, 4045, 4047, 4049, 4051, 4053, 4055, 4057, 4059, 4061, + 4063, 4065, 4067, 4069, 4071, 4073, 4075, 4077, 4079, 4081, 4083, 4085, 4087, 4089, 4091, 4093, +}; + +// Approximate floor(sqrt(x)) for x < 2^32, for USE_MAGNITUDE. +// The exponent is taken in bit pairs with clz and the top 10 significant bits +// index the table: never above the exact value, at most 0.4% below it for +// results >= 256. Pure integer arithmetic, so generic, NEON and AVX2 give +// bit-identical results. +static inline __attribute__((always_inline)) uint32_t demod_isqrt(uint32_t x) +{ + if (x < 1024) + return demod_sqrt_table[x] >> 7; + + int e = (31 - __builtin_clz(x)) & ~1; // even exponent: 2^e <= x < 2^(e+2) + uint32_t m = x >> (e - 8); // top 10 bits: 256..1023 + return ((uint32_t) demod_sqrt_table[m] << ((e - 8) >> 1)) >> 7; +} + +// Square root used by getpow() in USE_MAGNITUDE mode. Override on the command line +// to compare, e.g. -DDEMOD_SQRT(x)='((uint32_t)sqrtf((float)(x)))' +#ifndef DEMOD_SQRT +#define DEMOD_SQRT(x) demod_isqrt(x) +#endif + #endif diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index e7f1fdcd8..ded22bbf0 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -61,8 +61,13 @@ static inline __attribute__((always_inline)) unsigned getpow_avx2(const __m128i uint32_t pwr = (uint32_t)(resi*resi) + (uint32_t)(resq*resq); +#ifdef USE_MAGNITUDE + (void)shift; + pwr = demod_isqrt(pwr); +#else pwr >>= shift; // Set gain if (pwr > 65535) pwr = 65535; +#endif return pwr; } diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c index d6133e3fb..9e6817aa0 100644 --- a/dsp/impl/demod2400x.generic.c +++ b/dsp/impl/demod2400x.generic.c @@ -42,8 +42,13 @@ static inline unsigned getpow_generic( sq = (sq+16)>>5; uint32_t pwr = (uint32_t)(si*si) + (uint32_t)(sq*sq); +#ifdef USE_MAGNITUDE + (void)shift; + pwr = demod_isqrt(pwr); +#else pwr >>= shift; // Set gain if (pwr > 65535) pwr = 65535; +#endif return pwr; } diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 42f7f2f30..14a9eee54 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -62,8 +62,13 @@ static inline __attribute__((always_inline)) unsigned getpow_neon(const int8x8_t uint32x2_t sqr = vreinterpret_u32_s32(vmul_s32(res, res)); // [resi^2, resq^2] uint32_t pwr = vget_lane_u32(sqr, 0) + vget_lane_u32(sqr, 1); +#ifdef USE_MAGNITUDE + (void)shift; + pwr = demod_isqrt(pwr); +#else pwr >>= shift; // Set gain if (pwr > 65535) pwr = 65535; +#endif return pwr; } From 3a7afd3cd88297674a6df69f93df3d317dc1e409 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Sun, 18 Oct 2026 21:10:06 +0100 Subject: [PATCH 06/25] improved speed --- demod_2400x.c | 4 +- dsp/helpers/demod2400x_coeffs.h | 107 +++------ dsp/impl/demod2400x.avx2.c | 298 ++++++++++++++++++------- dsp/impl/demod2400x.neon.c | 374 +++++++++++++++++++++++--------- 4 files changed, 521 insertions(+), 262 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index f6575740e..e360b7586 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -34,6 +34,8 @@ #define PEAMBLE_THRES 80 #define DATA_THRES 150 #endif +#define DATA2_THRES ((4*DATA_THRES)/5) + // Static data struct staticdata_t { @@ -224,7 +226,7 @@ static int demodFrame(struct candidate *c, const uint16_t *udata, int nt) int bits = modesMessageLenByType(c->msg[0] >> 3); if (bits == MODES_LONG_MSG_BITS && - decodeShortAdbsData(c->pos + 12 * SHORT_FRAME, udata, nt, c->msg + MODES_SHORT_MSG_BYTES) < DATA_THRES) + decodeShortAdbsData(c->pos + 12 * SHORT_FRAME, udata, nt, c->msg + MODES_SHORT_MSG_BYTES) < DATA2_THRES) return 0; return bits; diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h index 9ec1927f5..272036ef2 100644 --- a/dsp/helpers/demod2400x_coeffs.h +++ b/dsp/helpers/demod2400x_coeffs.h @@ -7,95 +7,42 @@ #include // 2.4 -> 12 MHz polyphase FIR, 5 phases x 8 taps, gain 128 per phase -static const int8_t upsample_coeffs[5][8] = { +/*static const int8_t upsample_coeffs[5][8] = { { -1, 4, -10, 125, 14, -6, 3, -1 }, // phase=0 sum=128 energy=15984 gd=3.0938 { -3, 8, -21, 109, 47, -16, 7, -3 }, // phase=1 sum=128 energy=14918 gd=3.2969 { -4, 10, -23, 81, 81, -23, 10, -4 }, // phase=2 sum=128 energy=14412 gd=3.5000 { -3, 7, -16, 47, 109, -21, 8, -3 }, // phase=3 sum=128 energy=14918 gd=3.7031 { -1, 3, -6, 14, 125, -10, 4, -1 }, // phase=4 sum=128 energy=15984 gd=3.9062 +};*/ + +// sinc fc=1.2 win=7.8 a=0 gain=128 +// gain 128 per phase, noise gain 0.992 +/*static const int8_t upsample_coeffs[5][8] = { + { -3, 5, -11, 127, 14, -6, 3, -1 }, // phase=0 sum=128 + { -6, 12, -24, 113, 47, -18, 9, -5 }, // phase=1 sum=128 + { -7, 13, -26, 84, 84, -26, 13, -7 }, // phase=2 sum=128 + { -5, 9, -18, 47, 113, -24, 12, -6 }, // phase=3 sum=128 + { -1, 3, -6, 14, 127, -11, 5, -3 }, // phase=4 sum=128 +};*/ + +// sinc fc=1.2 win=12 gain=126 +// gain 126 per phase, noise gain 1.035 +static const int8_t upsample_coeffs[5][8] = { + { -4, 6, -11, 126, 14, -6, 4, -3 }, // phase=0 sum=126 + { -9, 14, -25, 113, 48, -19, 11, -7 }, // phase=1 sum=126 + { -10, 15, -27, 85, 85, -27, 15, -10 }, // phase=2 sum=126 + { -7, 11, -19, 48, 113, -25, 14, -9 }, // phase=3 sum=126 + { -3, 4, -6, 14, 126, -11, 6, -4 }, // phase=4 sum=126 }; -// floor(sqrt(m) * 128) for m = 0..1023 (max 4088, fits uint16) -static const uint16_t demod_sqrt_table[1024] = { - 0, 128, 181, 221, 256, 286, 313, 338, 362, 384, 404, 424, 443, 461, 478, 495, - 512, 527, 543, 557, 572, 586, 600, 613, 627, 640, 652, 665, 677, 689, 701, 712, - 724, 735, 746, 757, 768, 778, 789, 799, 809, 819, 829, 839, 849, 858, 868, 877, - 886, 896, 905, 914, 923, 931, 940, 949, 957, 966, 974, 983, 991, 999, 1007, 1015, - 1024, 1031, 1039, 1047, 1055, 1063, 1070, 1078, 1086, 1093, 1101, 1108, 1115, 1123, 1130, 1137, - 1144, 1152, 1159, 1166, 1173, 1180, 1187, 1193, 1200, 1207, 1214, 1221, 1227, 1234, 1241, 1247, - 1254, 1260, 1267, 1273, 1280, 1286, 1292, 1299, 1305, 1311, 1317, 1324, 1330, 1336, 1342, 1348, - 1354, 1360, 1366, 1372, 1378, 1384, 1390, 1396, 1402, 1408, 1413, 1419, 1425, 1431, 1436, 1442, - 1448, 1453, 1459, 1465, 1470, 1476, 1481, 1487, 1492, 1498, 1503, 1509, 1514, 1519, 1525, 1530, - 1536, 1541, 1546, 1551, 1557, 1562, 1567, 1572, 1578, 1583, 1588, 1593, 1598, 1603, 1608, 1614, - 1619, 1624, 1629, 1634, 1639, 1644, 1649, 1654, 1659, 1664, 1668, 1673, 1678, 1683, 1688, 1693, - 1698, 1702, 1707, 1712, 1717, 1722, 1726, 1731, 1736, 1740, 1745, 1750, 1755, 1759, 1764, 1768, - 1773, 1778, 1782, 1787, 1792, 1796, 1801, 1805, 1810, 1814, 1819, 1823, 1828, 1832, 1837, 1841, - 1846, 1850, 1854, 1859, 1863, 1868, 1872, 1876, 1881, 1885, 1889, 1894, 1898, 1902, 1907, 1911, - 1915, 1920, 1924, 1928, 1932, 1936, 1941, 1945, 1949, 1953, 1958, 1962, 1966, 1970, 1974, 1978, - 1982, 1987, 1991, 1995, 1999, 2003, 2007, 2011, 2015, 2019, 2023, 2027, 2031, 2035, 2039, 2043, - 2048, 2051, 2055, 2059, 2063, 2067, 2071, 2075, 2079, 2083, 2087, 2091, 2095, 2099, 2103, 2107, - 2111, 2114, 2118, 2122, 2126, 2130, 2134, 2138, 2141, 2145, 2149, 2153, 2157, 2160, 2164, 2168, - 2172, 2176, 2179, 2183, 2187, 2191, 2194, 2198, 2202, 2205, 2209, 2213, 2217, 2220, 2224, 2228, - 2231, 2235, 2239, 2242, 2246, 2250, 2253, 2257, 2260, 2264, 2268, 2271, 2275, 2278, 2282, 2286, - 2289, 2293, 2296, 2300, 2304, 2307, 2311, 2314, 2318, 2321, 2325, 2328, 2332, 2335, 2339, 2342, - 2346, 2349, 2353, 2356, 2360, 2363, 2367, 2370, 2374, 2377, 2380, 2384, 2387, 2391, 2394, 2398, - 2401, 2404, 2408, 2411, 2415, 2418, 2421, 2425, 2428, 2432, 2435, 2438, 2442, 2445, 2448, 2452, - 2455, 2458, 2462, 2465, 2468, 2472, 2475, 2478, 2482, 2485, 2488, 2491, 2495, 2498, 2501, 2505, - 2508, 2511, 2514, 2518, 2521, 2524, 2527, 2531, 2534, 2537, 2540, 2543, 2547, 2550, 2553, 2556, - 2560, 2563, 2566, 2569, 2572, 2575, 2579, 2582, 2585, 2588, 2591, 2594, 2598, 2601, 2604, 2607, - 2610, 2613, 2616, 2620, 2623, 2626, 2629, 2632, 2635, 2638, 2641, 2644, 2648, 2651, 2654, 2657, - 2660, 2663, 2666, 2669, 2672, 2675, 2678, 2681, 2684, 2688, 2691, 2694, 2697, 2700, 2703, 2706, - 2709, 2712, 2715, 2718, 2721, 2724, 2727, 2730, 2733, 2736, 2739, 2742, 2745, 2748, 2751, 2754, - 2757, 2760, 2763, 2766, 2769, 2772, 2774, 2777, 2780, 2783, 2786, 2789, 2792, 2795, 2798, 2801, - 2804, 2807, 2810, 2813, 2816, 2818, 2821, 2824, 2827, 2830, 2833, 2836, 2839, 2842, 2844, 2847, - 2850, 2853, 2856, 2859, 2862, 2865, 2867, 2870, 2873, 2876, 2879, 2882, 2884, 2887, 2890, 2893, - 2896, 2899, 2901, 2904, 2907, 2910, 2913, 2916, 2918, 2921, 2924, 2927, 2930, 2932, 2935, 2938, - 2941, 2944, 2946, 2949, 2952, 2955, 2957, 2960, 2963, 2966, 2968, 2971, 2974, 2977, 2979, 2982, - 2985, 2988, 2990, 2993, 2996, 2999, 3001, 3004, 3007, 3010, 3012, 3015, 3018, 3020, 3023, 3026, - 3029, 3031, 3034, 3037, 3039, 3042, 3045, 3047, 3050, 3053, 3055, 3058, 3061, 3063, 3066, 3069, - 3072, 3074, 3077, 3079, 3082, 3085, 3087, 3090, 3093, 3095, 3098, 3101, 3103, 3106, 3109, 3111, - 3114, 3117, 3119, 3122, 3124, 3127, 3130, 3132, 3135, 3137, 3140, 3143, 3145, 3148, 3150, 3153, - 3156, 3158, 3161, 3163, 3166, 3169, 3171, 3174, 3176, 3179, 3182, 3184, 3187, 3189, 3192, 3194, - 3197, 3200, 3202, 3205, 3207, 3210, 3212, 3215, 3217, 3220, 3222, 3225, 3228, 3230, 3233, 3235, - 3238, 3240, 3243, 3245, 3248, 3250, 3253, 3255, 3258, 3260, 3263, 3265, 3268, 3270, 3273, 3275, - 3278, 3280, 3283, 3285, 3288, 3290, 3293, 3295, 3298, 3300, 3303, 3305, 3308, 3310, 3313, 3315, - 3318, 3320, 3323, 3325, 3328, 3330, 3332, 3335, 3337, 3340, 3342, 3345, 3347, 3350, 3352, 3354, - 3357, 3359, 3362, 3364, 3367, 3369, 3372, 3374, 3376, 3379, 3381, 3384, 3386, 3388, 3391, 3393, - 3396, 3398, 3401, 3403, 3405, 3408, 3410, 3413, 3415, 3417, 3420, 3422, 3425, 3427, 3429, 3432, - 3434, 3436, 3439, 3441, 3444, 3446, 3448, 3451, 3453, 3456, 3458, 3460, 3463, 3465, 3467, 3470, - 3472, 3474, 3477, 3479, 3481, 3484, 3486, 3489, 3491, 3493, 3496, 3498, 3500, 3503, 3505, 3507, - 3510, 3512, 3514, 3517, 3519, 3521, 3524, 3526, 3528, 3531, 3533, 3535, 3537, 3540, 3542, 3544, - 3547, 3549, 3551, 3554, 3556, 3558, 3561, 3563, 3565, 3567, 3570, 3572, 3574, 3577, 3579, 3581, - 3584, 3586, 3588, 3590, 3593, 3595, 3597, 3599, 3602, 3604, 3606, 3609, 3611, 3613, 3615, 3618, - 3620, 3622, 3624, 3627, 3629, 3631, 3633, 3636, 3638, 3640, 3642, 3645, 3647, 3649, 3651, 3654, - 3656, 3658, 3660, 3663, 3665, 3667, 3669, 3672, 3674, 3676, 3678, 3680, 3683, 3685, 3687, 3689, - 3692, 3694, 3696, 3698, 3700, 3703, 3705, 3707, 3709, 3712, 3714, 3716, 3718, 3720, 3723, 3725, - 3727, 3729, 3731, 3734, 3736, 3738, 3740, 3742, 3744, 3747, 3749, 3751, 3753, 3755, 3758, 3760, - 3762, 3764, 3766, 3768, 3771, 3773, 3775, 3777, 3779, 3781, 3784, 3786, 3788, 3790, 3792, 3794, - 3797, 3799, 3801, 3803, 3805, 3807, 3810, 3812, 3814, 3816, 3818, 3820, 3822, 3825, 3827, 3829, - 3831, 3833, 3835, 3837, 3840, 3842, 3844, 3846, 3848, 3850, 3852, 3854, 3857, 3859, 3861, 3863, - 3865, 3867, 3869, 3871, 3873, 3876, 3878, 3880, 3882, 3884, 3886, 3888, 3890, 3892, 3895, 3897, - 3899, 3901, 3903, 3905, 3907, 3909, 3911, 3913, 3916, 3918, 3920, 3922, 3924, 3926, 3928, 3930, - 3932, 3934, 3936, 3938, 3941, 3943, 3945, 3947, 3949, 3951, 3953, 3955, 3957, 3959, 3961, 3963, - 3965, 3968, 3970, 3972, 3974, 3976, 3978, 3980, 3982, 3984, 3986, 3988, 3990, 3992, 3994, 3996, - 3998, 4000, 4002, 4004, 4007, 4009, 4011, 4013, 4015, 4017, 4019, 4021, 4023, 4025, 4027, 4029, - 4031, 4033, 4035, 4037, 4039, 4041, 4043, 4045, 4047, 4049, 4051, 4053, 4055, 4057, 4059, 4061, - 4063, 4065, 4067, 4069, 4071, 4073, 4075, 4077, 4079, 4081, 4083, 4085, 4087, 4089, 4091, 4093, -}; - -// Approximate floor(sqrt(x)) for x < 2^32, for USE_MAGNITUDE. -// The exponent is taken in bit pairs with clz and the top 10 significant bits -// index the table: never above the exact value, at most 0.4% below it for -// results >= 256. Pure integer arithmetic, so generic, NEON and AVX2 give -// bit-identical results. +// Exact floor(sqrt(x)) for x < 2^31, for USE_MAGNITUDE. +// sqrtf is correctly rounded, so after adding 2^-6 the truncated value is the exact +// result or one more, fixed with an integer check. Exact, so generic, NEON and AVX2 +// (which use the same steps in vector form) give bit-identical results. static inline __attribute__((always_inline)) uint32_t demod_isqrt(uint32_t x) { - if (x < 1024) - return demod_sqrt_table[x] >> 7; - - int e = (31 - __builtin_clz(x)) & ~1; // even exponent: 2^e <= x < 2^(e+2) - uint32_t m = x >> (e - 8); // top 10 bits: 256..1023 - return ((uint32_t) demod_sqrt_table[m] << ((e - 8) >> 1)) >> 7; + uint32_t q = (uint32_t) (__builtin_sqrtf((float) x) + 1.0f / 64); + return q - (q * q > x); } // Square root used by getpow() in USE_MAGNITUDE mode. Override on the command line diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index ded22bbf0..9e44f5435 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -30,12 +30,6 @@ static inline __attribute__((always_inline)) int32_t hsum_s32x4_avx2(const __m12 return _mm_cvtsi128_si32(s); } -// Horizontal add -static inline __attribute__((always_inline)) uint32_t hsum_u32x4_avx2(const __m128i v) -{ - return (uint32_t) hsum_s32x4_avx2(v); -} - // Load 8 interleaved I/Q samples and deinterleave them (like NEON's vld2q_s16) static inline __attribute__((always_inline)) iq16x8x2_avx2_t load2_s16_avx2(const int16_t *in) { @@ -73,119 +67,257 @@ static inline __attribute__((always_inline)) unsigned getpow_avx2(const __m128i } +// Exact floor(sqrt(x)) for x < 2^31, same steps as demod_isqrt() +static inline __attribute__((always_inline)) +__m256i isqrt_u32x8_avx2(const __m256i x) +{ + // sqrt is correctly rounded, so after adding 2^-6 truncation is exact or one too large + const __m256 f = _mm256_add_ps(_mm256_sqrt_ps(_mm256_cvtepi32_ps(x)), _mm256_set1_ps(1.0f / 64)); + const __m256i q = _mm256_cvttps_epi32(f); + // q * q > x (unsigned, q * q may reach 2^31): q - 1 unless max(q * q, x) == x + const __m256i qq = _mm256_mullo_epi32(q, q); + const __m256i ok = _mm256_cmpeq_epi32(_mm256_max_epu32(qq, x), x); + return _mm256_sub_epi32(_mm256_sub_epi32(q, _mm256_set1_epi32(1)), ok); +} + +// Power (or magnitude) of 8 filter outputs, as getpow_avx2() +static inline __attribute__((always_inline)) +__m128i getpow8_avx2(__m256i si, __m256i sq, int shift) +{ + const __m256i round = _mm256_set1_epi32(16); + si = _mm256_srai_epi32(_mm256_add_epi32(si, round), 5); // (x+16)>>5 + sq = _mm256_srai_epi32(_mm256_add_epi32(sq, round), 5); + __m256i pwr = _mm256_add_epi32(_mm256_mullo_epi32(si, si), _mm256_mullo_epi32(sq, sq)); + +#ifdef USE_MAGNITUDE + (void)shift; + pwr = isqrt_u32x8_avx2(pwr); +#else + pwr = _mm256_srl_epi32(pwr, _mm_cvtsi32_si128(shift)); // Set gain + pwr = _mm256_min_epu32(pwr, _mm256_set1_epi32(65535)); +#endif + + return _mm_packus_epi32(_mm256_castsi256_si128(pwr), _mm256_extracti128_si256(pwr, 1)); +} + +// 16-bit pairs (v[k], v[k+1]) for 8 consecutive windows, from v[0..15] = lo, hi +#define PAIRS_AVX2(lo, hi, k) \ + _mm256_inserti128_si256( \ + _mm256_castsi128_si256(_mm_unpacklo_epi16(_mm_alignr_epi8(hi, lo, 2 * (k)), \ + _mm_alignr_epi8(hi, lo, 2 * (k) + 2))), \ + _mm_unpackhi_epi16(_mm_alignr_epi8(hi, lo, 2 * (k)), _mm_alignr_epi8(hi, lo, 2 * (k) + 2)), 1) + // Upsample int STARCH_IMPL(upsample12, avx2) (const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) { if (len < 8) return 0; - __m128i c0 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[0]); // 8 x int8 - __m128i c1 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[1]); // 8 x int8 - __m128i c2 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[2]); // 8 x int8 - __m128i c3 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[3]); // 8 x int8 - __m128i c4 = _mm_loadl_epi64((const __m128i *) upsample_coeffs[4]); // 8 x int8 - - iq16x8x2_avx2_t iq = load2_s16_avx2(in); - int nt = NU_SAMP(len); // Total samples int stride = NU_STRIDE(nt); // Stride int stride6 = 6*stride; int bpos = 0; + int nw = len - 7; // Windows, 5 outputs each + int n = 0; + + // Coefficient pairs (c[2j], c[2j+1]) for madd, two taps at a time + __m256i cp[5][4]; + for (int c = 0; c < 5; c++) + for (int j = 0; j < 4; j++) + cp[c][j] = _mm256_set1_epi32((int32_t) ((uint16_t) upsample_coeffs[c][2 * j] | + ((uint32_t) (uint16_t) upsample_coeffs[c][2 * j + 1] << 16))); + + // 8 windows at a time; reads samples n .. n+15 + for (; n + 9 <= nw; n += 8) { + const iq16x8x2_avx2_t a = load2_s16_avx2(in + 2 * n); // samples n .. n+7 + const iq16x8x2_avx2_t b = load2_s16_avx2(in + 2 * n + 16); // samples n+8 .. n+15 + + const __m256i pi0 = PAIRS_AVX2(a.val[0], b.val[0], 0), pq0 = PAIRS_AVX2(a.val[1], b.val[1], 0); + const __m256i pi2 = PAIRS_AVX2(a.val[0], b.val[0], 2), pq2 = PAIRS_AVX2(a.val[1], b.val[1], 2); + const __m256i pi4 = PAIRS_AVX2(a.val[0], b.val[0], 4), pq4 = PAIRS_AVX2(a.val[1], b.val[1], 4); + const __m256i pi6 = PAIRS_AVX2(a.val[0], b.val[0], 6), pq6 = PAIRS_AVX2(a.val[1], b.val[1], 6); + + uint16_t t[5][8]; + for (int c = 0; c < 5; c++) { + const __m256i si = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(pi0, cp[c][0]), _mm256_madd_epi16(pi2, cp[c][1])), + _mm256_add_epi32(_mm256_madd_epi16(pi4, cp[c][2]), _mm256_madd_epi16(pi6, cp[c][3]))); + const __m256i sq = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(pq0, cp[c][0]), _mm256_madd_epi16(pq2, cp[c][1])), + _mm256_add_epi32(_mm256_madd_epi16(pq4, cp[c][2]), _mm256_madd_epi16(pq6, cp[c][3]))); + _mm_storeu_si128((__m128i *) t[c], getpow8_avx2(si, sq, shift)); + } - in += 16; - len -= 8; - while (1) { - - // Rearrage output - oo[bpos] = getpow_avx2(c0, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow_avx2(c1, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow_avx2(c2, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow_avx2(c3, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; - - oo[bpos] = getpow_avx2(c4, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; + // Rearrange output + for (int w = 0; w < 8; w++) { + for (int c = 0; c < 5; c++) { + oo[bpos] = t[c][w]; + if ((bpos += stride) >= stride6) bpos -= stride6-1; + } + } + } - if (len == 0) break; - len--; + // Remainder + __m128i cv[5]; + for (int c = 0; c < 5; c++) + cv[c] = _mm_loadl_epi64((const __m128i *) upsample_coeffs[c]); // 8 x int8 - // Shift - iq.val[0] = _mm_insert_epi16(_mm_srli_si128(iq.val[0], 2), in[0], 7); - iq.val[1] = _mm_insert_epi16(_mm_srli_si128(iq.val[1], 2), in[1], 7); - in += 2; + for (; n < nw; n++) { + const iq16x8x2_avx2_t iq = load2_s16_avx2(in + 2 * n); + for (int c = 0; c < 5; c++) { + oo[bpos] = getpow_avx2(cv[c], iq, shift); + if ((bpos += stride) >= stride6) bpos -= stride6-1; + } } return nt; } -// Locate preamble -int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +// Exact floor(n / d) for d > 0, n < 2^28 and n / d < 256 (the preamble metric range) +static inline __attribute__((always_inline)) +__m256i div_u32x8_avx2(const __m256i n, const __m256i d) { - int stride = NU_STRIDE(nt); // Stride - const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; - - const __m128i gm1 = _mm_loadu_si128((const __m128i *) (mask_g + 0)); - const __m128i gm2 = _mm_loadu_si128((const __m128i *) (mask_g + 8)); - const __m128i zero = _mm_setzero_si128(); - - for (int j = 0; j < 6; j++) { // Look at each phase in turn + // Values are < 2^31, so the signed conversions and compares are fine + // Correctly rounded quotient: truncation is at most one too large + const __m256i q = _mm256_cvttps_epi32(_mm256_div_ps(_mm256_cvtepi32_ps(n), _mm256_cvtepi32_ps(d))); + return _mm256_add_epi32(q, _mm256_cmpgt_epi32(_mm256_mullo_epi32(q, d), n)); +} - const uint16_t * restrict p = &pp[j*stride]; - int plen = (nt + 5 - j) / 6; // Length of each phase +// Scalar preamble metric at p[0] +static inline __attribute__((always_inline)) +uint8_t preamble_metric1(const uint16_t * restrict p) +{ +#ifdef PREAMBLE_MIN_PULSE + uint32_t minp = MIN(MIN(p[0], p[2]), MIN(p[7], p[9])); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32_t minp = p[0] + p[2] + p[7] + p[9]; +#endif + uint32_t gap = p[1] + p[3] + p[4] + p[5] + p[6] + p[8] + + p[10] + p[11] + p[12] + p[13] + p[14] + p[15]; - plen -= 2 * LONG_FRAME; // No point in searching at end - if (plen < 16) continue; + return (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); +} - __m128i p1 = _mm_loadu_si128((const __m128i *) (p + 0)); - __m128i p2 = _mm_loadu_si128((const __m128i *) (p + 8)); +#define PREAMBLE_BLOCK 64 // Positions per phase per block, multiple of 8 - plen -= 16; - p += 16; - uint8_t * restrict mp = &mm[j]; +// Load 8 x uint16 and widen to 8 x uint32 +static inline __attribute__((always_inline)) +__m256i load_u16x8_u32_avx2(const uint16_t * restrict p) +{ + return _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *) p)); +} - while (1) { +// Widened sum of 4 consecutive samples, p[0..3] .. p[7..10] +static inline __attribute__((always_inline)) +__m256i sum4_u16x8_avx2(const uint16_t * restrict p) +{ + return _mm256_add_epi32(_mm256_add_epi32(load_u16x8_u32_avx2(p + 0), load_u16x8_u32_avx2(p + 1)), + _mm256_add_epi32(load_u16x8_u32_avx2(p + 2), load_u16x8_u32_avx2(p + 3))); +} - // Preamble detection - // P G P G, G G G P, G P G G, G G G G +// Preamble metrics for 8*n8 consecutive positions of one phase +static inline __attribute__((always_inline)) +void preamble_metric_block_avx2(const uint16_t * restrict p, int n8, uint8_t * restrict out) +{ + // Preamble detection + // P G P G, G G G P, G P G G, G G G G + // gap = p[1] + p[8] + p[10] + p[15] + s4[3] + s4[11], s4[x] = p[x] + .. + p[x+3] + // s4[11] for these 8 positions is s4[3] for the next 8 - // Extract gaps and sum together (widened to 32-bit, values are unsigned) - __m128i g1 = _mm_and_si128(p1, gm1); - __m128i g2 = _mm_and_si128(p2, gm2); - __m128i gs = _mm_add_epi32(_mm_add_epi32(_mm_unpacklo_epi16(g1, zero), _mm_unpackhi_epi16(g1, zero)), - _mm_add_epi32(_mm_unpacklo_epi16(g2, zero), _mm_unpackhi_epi16(g2, zero))); - uint32_t gap = hsum_u32x4_avx2(gs); + const __m256i one = _mm256_set1_epi32(1); + __m256i s4a = sum4_u16x8_avx2(p + 3); - // Extract pulses: low 4 lanes of px = p[0], p[9], p[2], p[7] - __m128i dm1 = _mm_andnot_si128(gm1, p1); - __m128i dm2 = _mm_andnot_si128(gm2, p2); + for (int k = 0; k < 8 * n8; k += 8, p += 8) { + const __m256i s4b = sum4_u16x8_avx2(p + 11); - __m128i px = _mm_or_si128(_mm_or_si128(dm1, _mm_srli_si128(dm1, 8)), dm2); + __m256i gap = _mm256_add_epi32(_mm256_add_epi32(load_u16x8_u32_avx2(p + 1), load_u16x8_u32_avx2(p + 8)), + _mm256_add_epi32(load_u16x8_u32_avx2(p + 10), load_u16x8_u32_avx2(p + 15))); + gap = _mm256_add_epi32(_mm256_add_epi32(gap, one), _mm256_add_epi32(s4a, s4b)); + s4a = s4b; + // Metric = (256 * KK * minp) / (KK * minp + gap + 1) #ifdef PREAMBLE_MIN_PULSE - __m128i m = _mm_min_epu16(px, _mm_srli_si128(px, 4)); - m = _mm_min_epu16(m, _mm_srli_si128(m, 2)); - unsigned minp = (unsigned) _mm_extract_epi16(m, 0); + const __m128i m16 = _mm_min_epu16(_mm_min_epu16(_mm_loadu_si128((const __m128i *) (p + 0)), + _mm_loadu_si128((const __m128i *) (p + 2))), + _mm_min_epu16(_mm_loadu_si128((const __m128i *) (p + 7)), + _mm_loadu_si128((const __m128i *) (p + 9)))); + const __m256i minp = _mm256_cvtepu16_epi32(m16); #endif #ifdef PREAMBLE_SUM_PULSE - uint32_t minp = hsum_u32x4_avx2(_mm_cvtepu16_epi32(px)); + const __m256i minp = _mm256_add_epi32(_mm256_add_epi32(load_u16x8_u32_avx2(p + 0), load_u16x8_u32_avx2(p + 2)), + _mm256_add_epi32(load_u16x8_u32_avx2(p + 7), load_u16x8_u32_avx2(p + 9))); #endif - // Metric - *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); - mp += 6; + const __m256i q = div_u32x8_avx2(_mm256_mullo_epi32(minp, _mm256_set1_epi32(256 * PREAMBLE_KK)), + _mm256_add_epi32(gap, _mm256_mullo_epi32(minp, _mm256_set1_epi32(PREAMBLE_KK)))); + + // Metric < 256 + const __m128i q16 = _mm_packus_epi32(_mm256_castsi256_si128(q), _mm256_extracti128_si256(q, 1)); + _mm_storel_epi64((__m128i *) (out + k), _mm_packus_epi16(q16, q16)); + } +} + +// Locate preamble +int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +{ + int stride = NU_STRIDE(nt); // Stride + int count[6]; // Number of positions to evaluate in each phase + int cmin = 0x7fffffff; - if (plen == 0) break; - plen--; + for (int j = 0; j < 6; j++) { + int plen = (nt + 5 - j) / 6; // Length of each phase + plen -= 2 * LONG_FRAME; // No point in searching at end + count[j] = plen - 15; + cmin = MIN(cmin, count[j]); + } - unsigned nv = *p++; - p1 = _mm_alignr_epi8(p2, p1, 2); - p2 = _mm_insert_epi16(_mm_srli_si128(p2, 2), (int) nv, 7); + // 3-way interleave of 16-bit elements a, b, c: o0 = a0 b0 c0 a1 b1 c1 a2 b2, + // o1 = c2 a3 b3 c3 a4 b4 c4 a5, o2 = b5 c5 a6 b6 c6 a7 b7 c7 + const __m128i s0a = _mm_setr_epi8( 0, 1, -1, -1, -1, -1, 2, 3, -1, -1, -1, -1, 4, 5, -1, -1); + const __m128i s0b = _mm_setr_epi8(-1, -1, 0, 1, -1, -1, -1, -1, 2, 3, -1, -1, -1, -1, 4, 5); + const __m128i s0c = _mm_setr_epi8(-1, -1, -1, -1, 0, 1, -1, -1, -1, -1, 2, 3, -1, -1, -1, -1); + const __m128i s1a = _mm_setr_epi8(-1, -1, 6, 7, -1, -1, -1, -1, 8, 9, -1, -1, -1, -1, 10, 11); + const __m128i s1b = _mm_setr_epi8(-1, -1, -1, -1, 6, 7, -1, -1, -1, -1, 8, 9, -1, -1, -1, -1); + const __m128i s1c = _mm_setr_epi8( 4, 5, -1, -1, -1, -1, 6, 7, -1, -1, -1, -1, 8, 9, -1, -1); + const __m128i s2a = _mm_setr_epi8(-1, -1, -1, -1, 12, 13, -1, -1, -1, -1, 14, 15, -1, -1, -1, -1); + const __m128i s2b = _mm_setr_epi8(10, 11, -1, -1, -1, -1, 12, 13, -1, -1, -1, -1, 14, 15, -1, -1); + const __m128i s2c = _mm_setr_epi8(-1, -1, 10, 11, -1, -1, -1, -1, 12, 13, -1, -1, -1, -1, 14, 15); + + // All phases together, a block at a time, so the outputs can be interleaved on store + int i = 0; + while (i + 8 <= cmin) { + const int n8 = MIN((cmin - i) / 8, PREAMBLE_BLOCK / 8); + uint8_t t[6][PREAMBLE_BLOCK]; + + for (int j = 0; j < 6; j++) + preamble_metric_block_avx2(&pp[j*stride + i], n8, t[j]); + + // mm[6*(i+k) + j] = t[j][k]: interleave phases in pairs, then 3-way as 16-bit + uint8_t * restrict mp = &mm[6*i]; + for (int k = 0; k < 8 * n8; k += 8, mp += 48) { + const __m128i a = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *) (t[0] + k)), + _mm_loadl_epi64((const __m128i *) (t[1] + k))); + const __m128i b = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *) (t[2] + k)), + _mm_loadl_epi64((const __m128i *) (t[3] + k))); + const __m128i c = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i *) (t[4] + k)), + _mm_loadl_epi64((const __m128i *) (t[5] + k))); + + const __m128i o0 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(a, s0a), _mm_shuffle_epi8(b, s0b)), + _mm_shuffle_epi8(c, s0c)); + const __m128i o1 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(a, s1a), _mm_shuffle_epi8(b, s1b)), + _mm_shuffle_epi8(c, s1c)); + const __m128i o2 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(a, s2a), _mm_shuffle_epi8(b, s2b)), + _mm_shuffle_epi8(c, s2c)); + + _mm_storeu_si128((__m128i *) (mp + 0), o0); + _mm_storeu_si128((__m128i *) (mp + 16), o1); + _mm_storeu_si128((__m128i *) (mp + 32), o2); } + i += 8 * n8; } + + // Remainder + for (int j = 0; j < 6; j++) + for (int k = i; k < count[j]; k++) + mm[6*k + j] = preamble_metric1(&pp[j*stride + k]); + return METRIC_SIZE(nt); } diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 14a9eee54..657962b51 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -17,20 +17,6 @@ #include "dsp/helpers/demod2400x_dsp.h" #include "dsp/helpers/demod2400x_coeffs.h" -// Horizontal add -static inline __attribute__((always_inline)) -uint32_t hsum_u32x4_neon(const uint32x4_t v) -{ -#ifdef __aarch64__ - return vaddvq_u32(v); -#else - uint32x2_t s = vadd_u32(vget_low_u32(v), - vget_high_u32(v)); - s = vpadd_u32(s, s); - return vget_lane_u32(s, 0); -#endif -} - // Horizontal add of two vectors at once: returns [sum of a, sum of b] static inline __attribute__((always_inline)) int32x2_t hsum2_s32x4_neon(const int32x4_t a, const int32x4_t b) @@ -73,127 +59,319 @@ static inline __attribute__((always_inline)) unsigned getpow_neon(const int8x8_t return pwr; } -// Upsample -int STARCH_IMPL(upsample12, neon) (const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) +// Exact floor(sqrt(x)) for x < 2^31, same result as demod_isqrt() +static inline __attribute__((always_inline)) +uint32x4_t isqrt_u32x4_neon(const uint32x4_t x) { - if (len < 8) return 0; + const float32x4_t f = vcvtq_f32_u32(x); +#ifdef __aarch64__ + // sqrt is correctly rounded, so after adding 2^-6 truncation is exact or one too large + const uint32x4_t q = vcvtq_u32_f32(vaddq_f32(vsqrtq_f32(f), vdupq_n_f32(1.0f / 64))); + return vaddq_u32(q, vcgtq_u32(vmulq_u32(q, q), x)); +#else + // Reciprocal square root estimate + two Newton steps: within one either way + float32x4_t r = vrsqrteq_f32(f); + r = vmulq_f32(vrsqrtsq_f32(vmulq_f32(f, r), r), r); + r = vmulq_f32(vrsqrtsq_f32(vmulq_f32(f, r), r), r); + uint32x4_t q = vcvtq_u32_f32(vmulq_f32(f, r)); // x = 0 gives NaN, converted to 0 + q = vaddq_u32(q, vcgtq_u32(vmulq_u32(q, q), x)); // too large: -1 + const uint32x4_t q1 = vaddq_u32(q, vdupq_n_u32(1)); + return vsubq_u32(q, vcleq_u32(vmulq_u32(q1, q1), x)); // too small: +1 +#endif +} - int8x8_t c0 = vld1_s8(upsample_coeffs[0]); - int8x8_t c1 = vld1_s8(upsample_coeffs[1]); - int8x8_t c2 = vld1_s8(upsample_coeffs[2]); - int8x8_t c3 = vld1_s8(upsample_coeffs[3]); - int8x8_t c4 = vld1_s8(upsample_coeffs[4]); +// Power (or magnitude) of 4 filter outputs, as getpow_neon() +static inline __attribute__((always_inline)) +uint16x4_t getpow4_neon(int32x4_t si, int32x4_t sq, int shift) +{ + si = vrshrq_n_s32(si, 5); // (x+16)>>5 + sq = vrshrq_n_s32(sq, 5); + const uint32x4_t pwr = vreinterpretq_u32_s32(vmlaq_s32(vmulq_s32(si, si), sq, sq)); - int16x8x2_t iq = vld2q_s16(in); +#ifdef USE_MAGNITUDE + (void)shift; + return vmovn_u32(isqrt_u32x4_neon(pwr)); +#else + return vqmovn_u32(vshlq_u32(pwr, vdupq_n_s32(-shift))); // Set gain, saturate to 65535 +#endif +} - int nt = NU_SAMP(len); // Total samples - int stride = NU_STRIDE(nt); // Stride - int stride6 = 6*stride; - int bpos = 0; +// One phase of the upsampler for 8 consecutive windows; s[k] = I/Q at window + k +static inline __attribute__((always_inline)) +uint16x8_t upsample_phase8_neon(const int16x8x2_t * restrict s, const int8_t * restrict coeffs, int shift) +{ + const int16x8_t cv = vmovl_s8(vld1_s8(coeffs)); + const int16x4_t c0 = vget_low_s16(cv); + const int16x4_t c1 = vget_high_s16(cv); + + int32x4_t si_lo = vmull_lane_s16(vget_low_s16(s[0].val[0]), c0, 0); + int32x4_t si_hi = vmull_lane_s16(vget_high_s16(s[0].val[0]), c0, 0); + int32x4_t sq_lo = vmull_lane_s16(vget_low_s16(s[0].val[1]), c0, 0); + int32x4_t sq_hi = vmull_lane_s16(vget_high_s16(s[0].val[1]), c0, 0); + +#define TAP(k, c, l) \ + si_lo = vmlal_lane_s16(si_lo, vget_low_s16(s[k].val[0]), c, l); \ + si_hi = vmlal_lane_s16(si_hi, vget_high_s16(s[k].val[0]), c, l); \ + sq_lo = vmlal_lane_s16(sq_lo, vget_low_s16(s[k].val[1]), c, l); \ + sq_hi = vmlal_lane_s16(sq_hi, vget_high_s16(s[k].val[1]), c, l) + + TAP(1, c0, 1); TAP(2, c0, 2); TAP(3, c0, 3); + TAP(4, c1, 0); TAP(5, c1, 1); TAP(6, c1, 2); TAP(7, c1, 3); +#undef TAP + + return vcombine_u16(getpow4_neon(si_lo, sq_lo, shift), getpow4_neon(si_hi, sq_hi, shift)); +} + +#ifdef __aarch64__ +// 5-way interleave of 16-bit lanes: bytes from ph[0..3] (tbl4), then ph[4] (tbx1) +static const uint8_t interleave5_tbl4[5][16] = { + { 0, 1, 16, 17, 32, 33, 48, 49, 255, 255, 2, 3, 18, 19, 34, 35 }, + { 50, 51, 255, 255, 4, 5, 20, 21, 36, 37, 52, 53, 255, 255, 6, 7 }, + { 22, 23, 38, 39, 54, 55, 255, 255, 8, 9, 24, 25, 40, 41, 56, 57 }, + { 255, 255, 10, 11, 26, 27, 42, 43, 58, 59, 255, 255, 12, 13, 28, 29 }, + { 44, 45, 60, 61, 255, 255, 14, 15, 30, 31, 46, 47, 62, 63, 255, 255 }, +}; +static const uint8_t interleave5_tbx1[5][16] = { + { 255, 255, 255, 255, 255, 255, 255, 255, 0, 1, 255, 255, 255, 255, 255, 255 }, + { 255, 255, 2, 3, 255, 255, 255, 255, 255, 255, 255, 255, 4, 5, 255, 255 }, + { 255, 255, 255, 255, 255, 255, 6, 7, 255, 255, 255, 255, 255, 255, 255, 255 }, + { 8, 9, 255, 255, 255, 255, 255, 255, 255, 255, 10, 11, 255, 255, 255, 255 }, + { 255, 255, 255, 255, 12, 13, 255, 255, 255, 255, 255, 255, 255, 255, 14, 15 }, +}; +#endif + +// out[5*w + c] = ph[c][w], w = 0..7 +static inline __attribute__((always_inline)) +void interleave5_u16x8_neon(const uint16x8_t * restrict ph, uint16_t * restrict out) +{ +#ifdef __aarch64__ + uint8x16x4_t t; + t.val[0] = vreinterpretq_u8_u16(ph[0]); + t.val[1] = vreinterpretq_u8_u16(ph[1]); + t.val[2] = vreinterpretq_u8_u16(ph[2]); + t.val[3] = vreinterpretq_u8_u16(ph[3]); + const uint8x16_t e = vreinterpretq_u8_u16(ph[4]); + + for (int v = 0; v < 5; v++) { + const uint8x16_t o = vqtbx1q_u8(vqtbl4q_u8(t, vld1q_u8(interleave5_tbl4[v])), e, vld1q_u8(interleave5_tbx1[v])); + vst1q_u16(out + 8 * v, vreinterpretq_u16_u8(o)); + } +#else + uint16_t t[5][8]; + for (int c = 0; c < 5; c++) + vst1q_u16(t[c], ph[c]); + for (int w = 0; w < 8; w++) + for (int c = 0; c < 5; c++) + out[5 * w + c] = t[c][w]; +#endif +} - in += 16; - len -= 8; - while (1) { +#define UPSAMPLE_BLOCK 48 // Windows per block, multiple of 48: 5*48 outputs = 40 per phase - // Rearrage output - oo[bpos] = getpow_neon(c0, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; +// Upsample +int STARCH_IMPL(upsample12, neon) (const int16_t * restrict in, int len, int shift, uint16_t * restrict oo) +{ + if (len < 8) return 0; - oo[bpos] = getpow_neon(c1, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; + int nt = NU_SAMP(len); // Total samples + int stride = NU_STRIDE(nt); // Stride + int nw = len - 7; // Windows: 5 outputs each, output o goes to oo[(o % 6) * stride + o / 6] + int n = 0; - oo[bpos] = getpow_neon(c2, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; + // 8 windows at a time for each phase, a block at a time so the outputs + // can be put in order then split into the 6 phases with vector loads/stores + for (; n + UPSAMPLE_BLOCK <= nw; n += UPSAMPLE_BLOCK) { + uint16_t tmp[5 * UPSAMPLE_BLOCK] __attribute__((aligned(16))); // tmp[o - 5*n] - oo[bpos] = getpow_neon(c3, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; + for (int w = 0; w < UPSAMPLE_BLOCK; w += 8) { + const int16_t * restrict p = in + 2 * (n + w); + int16x8x2_t s[8]; + for (int k = 0; k < 8; k++) + s[k] = vld2q_s16(p + 2 * k); - oo[bpos] = getpow_neon(c4, iq, shift); - if ((bpos += stride) >= stride6) bpos -= stride6-1; + uint16x8_t ph[5]; + for (int c = 0; c < 5; c++) + ph[c] = upsample_phase8_neon(s, upsample_coeffs[c], shift); - if (len == 0) break; - len--; + interleave5_u16x8_neon(ph, tmp + 5 * w); + } - // Shift - iq.val[0] = vextq_s16(iq.val[0], iq.val[0], 1); - iq.val[1] = vextq_s16(iq.val[1], iq.val[1], 1); + // 5*n is a multiple of 6, so tmp[6*m + r] -> oo[r * stride + 5*n/6 + m] + uint16_t * restrict op = oo + 5 * n / 6; + for (int m = 0; m < 5 * UPSAMPLE_BLOCK / 6; m += 8) { + const uint32x4x3_t a = vld3q_u32((const uint32_t *) (tmp + 6 * m)); // pairs (r, r+1) + const uint32x4x3_t b = vld3q_u32((const uint32_t *) (tmp + 6 * m + 24)); + for (int r = 0; r < 3; r++) { + const uint16x8x2_t z = vuzpq_u16(vreinterpretq_u16_u32(a.val[r]), vreinterpretq_u16_u32(b.val[r])); + vst1q_u16(op + (2 * r) * stride + m, z.val[0]); + vst1q_u16(op + (2 * r + 1) * stride + m, z.val[1]); + } + } + } - iq.val[0] = vsetq_lane_s16(in[0], iq.val[0], 7); - iq.val[1] = vsetq_lane_s16(in[1], iq.val[1], 7); - in += 2; + // Remainder + for (; n < nw; n++) { + const int16x8x2_t iq = vld2q_s16(in + 2 * n); + for (int c = 0; c < 5; c++) { + const int o = 5 * n + c; + oo[(o % 6) * stride + o / 6] = getpow_neon(vld1_s8(upsample_coeffs[c]), iq, shift); + } } return nt; } -// Locate preamble -int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +// Exact floor(n / d) for d > 0, n < 2^28 and n / d < 256 (the preamble metric range) +static inline __attribute__((always_inline)) +uint32x4_t div_u32x4_neon(const uint32x4_t n, const uint32x4_t d) { - int stride = NU_STRIDE(nt); // Stride - const int16_t mask_g[] = { 0,-1, 0,-1,-1,-1,-1, 0,-1, 0,-1,-1,-1,-1,-1,-1 }; - - uint16x8_t gm1 = vld1q_u16((const uint16_t *)mask_g+0); - uint16x8_t gm2 = vld1q_u16((const uint16_t *)mask_g+8); - - for (int j = 0; j < 6; j++) { // Look at each phase in turn - - const uint16_t * restrict p = &pp[j*stride]; - int plen = (nt + 5 - j) / 6; // Length of each phase + const float32x4_t nf = vcvtq_f32_u32(n); + const float32x4_t df = vcvtq_f32_u32(d); +#ifdef __aarch64__ + // Correctly rounded quotient: truncation is at most one too large + uint32x4_t q = vcvtq_u32_f32(vdivq_f32(nf, df)); + return vaddq_u32(q, vcgtq_u32(vmulq_u32(q, d), n)); +#else + // Reciprocal estimate + two Newton steps: truncation is within one either way + float32x4_t r = vrecpeq_f32(df); + r = vmulq_f32(vrecpsq_f32(df, r), r); + r = vmulq_f32(vrecpsq_f32(df, r), r); + uint32x4_t q = vcvtq_u32_f32(vmulq_f32(nf, r)); + const uint32x4_t qd = vmulq_u32(q, d); + q = vaddq_u32(q, vcgtq_u32(qd, n)); // too large: -1 + q = vsubq_u32(q, vcleq_u32(vaddq_u32(qd, d), n)); // too small: +1 + return q; +#endif +} - plen -= 2 * LONG_FRAME; // No point in searching at end - if (plen < 16) continue; +// Scalar preamble metric at p[0] +static inline __attribute__((always_inline)) +uint8_t preamble_metric1(const uint16_t * restrict p) +{ +#ifdef PREAMBLE_MIN_PULSE + uint32_t minp = MIN(MIN(p[0], p[2]), MIN(p[7], p[9])); +#endif +#ifdef PREAMBLE_SUM_PULSE + uint32_t minp = p[0] + p[2] + p[7] + p[9]; +#endif + uint32_t gap = p[1] + p[3] + p[4] + p[5] + p[6] + p[8] + + p[10] + p[11] + p[12] + p[13] + p[14] + p[15]; - uint16x8_t p1 = vld1q_u16(p + 0); - uint16x8_t p2 = vld1q_u16(p + 8); + return (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); +} - plen -= 16; - p += 16; - uint8_t * restrict mp = &mm[j]; +#define PREAMBLE_BLOCK 64 // Positions per phase per block, multiple of 8 - while (1) { +// Widened sum of 4 consecutive samples, p[0..3] .. p[7..10] +static inline __attribute__((always_inline)) +uint32x4x2_t sum4_u16x8_neon(const uint16_t * restrict p) +{ + const uint16x8_t a = vld1q_u16(p + 0), b = vld1q_u16(p + 1); + const uint16x8_t c = vld1q_u16(p + 2), d = vld1q_u16(p + 3); + uint32x4x2_t s; + s.val[0] = vaddq_u32(vaddl_u16(vget_low_u16(a), vget_low_u16(b)), + vaddl_u16(vget_low_u16(c), vget_low_u16(d))); + s.val[1] = vaddq_u32(vaddl_u16(vget_high_u16(a), vget_high_u16(b)), + vaddl_u16(vget_high_u16(c), vget_high_u16(d))); + return s; +} - // Preamble detection - // P G P G, G G G P, G P G G, G G G G +// Preamble metrics for 8*n8 consecutive positions of one phase +static inline __attribute__((always_inline)) +void preamble_metric_block_neon(const uint16_t * restrict p, int n8, uint8_t * restrict out) +{ + // Preamble detection + // P G P G, G G G P, G P G G, G G G G + // gap = p[1] + p[8] + p[10] + p[15] + s4[3] + s4[11], s4[x] = p[x] + .. + p[x+3] + // s4[11] for these 8 positions is s4[3] for the next 8 - // Extract gaps and sum together - uint32x4_t g1 = vpaddlq_u16(vandq_u16(p1, gm1)); - uint32x4_t g2 = vpaddlq_u16(vandq_u16(p2, gm2)); - uint32x4_t gs = vaddq_u32(g1, g2); - uint32_t gap = hsum_u32x4_neon(gs); + const uint32x4_t one = vdupq_n_u32(1); + uint32x4x2_t s4a = sum4_u16x8_neon(p + 3); - // Extract pulses - uint16x8_t dm1 = vbicq_u16(p1, gm1); - uint16x8_t dm2 = vbicq_u16(p2, gm2); + for (int k = 0; k < 8 * n8; k += 8, p += 8) { + const uint32x4x2_t s4b = sum4_u16x8_neon(p + 11); - uint16x4_t d0 = vget_low_u16(dm1); - uint16x4_t d1 = vget_high_u16(dm1); - uint16x4_t d2 = vget_low_u16(dm2); + const uint16x8_t v1 = vld1q_u16(p + 1), v8 = vld1q_u16(p + 8); + const uint16x8_t v10 = vld1q_u16(p + 10), v15 = vld1q_u16(p + 15); - uint16x4_t px = vorr_u16(vorr_u16(d0, d1), d2); + uint32x4_t gap_lo = vaddq_u32(vaddl_u16(vget_low_u16(v1), vget_low_u16(v8)), + vaddl_u16(vget_low_u16(v10), vget_low_u16(v15))); + uint32x4_t gap_hi = vaddq_u32(vaddl_u16(vget_high_u16(v1), vget_high_u16(v8)), + vaddl_u16(vget_high_u16(v10), vget_high_u16(v15))); + gap_lo = vaddq_u32(vaddq_u32(gap_lo, one), vaddq_u32(s4a.val[0], s4b.val[0])); + gap_hi = vaddq_u32(vaddq_u32(gap_hi, one), vaddq_u32(s4a.val[1], s4b.val[1])); + s4a = s4b; + // Metric = (256 * KK * minp) / (KK * minp + gap + 1) #ifdef PREAMBLE_MIN_PULSE - uint16x4_t m = vpmin_u16(px, px); - m = vpmin_u16(m, m); - unsigned minp = vget_lane_u16(m, 0); + const uint16x8_t m = vminq_u16(vminq_u16(vld1q_u16(p + 0), vld1q_u16(p + 2)), + vminq_u16(vld1q_u16(p + 7), vld1q_u16(p + 9))); + const uint32x4_t q_lo = div_u32x4_neon(vmull_n_u16(vget_low_u16(m), 256 * PREAMBLE_KK), + vmlal_n_u16(gap_lo, vget_low_u16(m), PREAMBLE_KK)); + const uint32x4_t q_hi = div_u32x4_neon(vmull_n_u16(vget_high_u16(m), 256 * PREAMBLE_KK), + vmlal_n_u16(gap_hi, vget_high_u16(m), PREAMBLE_KK)); #endif #ifdef PREAMBLE_SUM_PULSE - uint32x2_t s = vpaddl_u16(px); - s = vpadd_u32(s, s); - uint32_t minp = vget_lane_u32(s, 0); + const uint16x8_t v0 = vld1q_u16(p + 0), v2 = vld1q_u16(p + 2); + const uint16x8_t v7 = vld1q_u16(p + 7), v9 = vld1q_u16(p + 9); + const uint32x4_t m_lo = vaddq_u32(vaddl_u16(vget_low_u16(v0), vget_low_u16(v2)), + vaddl_u16(vget_low_u16(v7), vget_low_u16(v9))); + const uint32x4_t m_hi = vaddq_u32(vaddl_u16(vget_high_u16(v0), vget_high_u16(v2)), + vaddl_u16(vget_high_u16(v7), vget_high_u16(v9))); + const uint32x4_t q_lo = div_u32x4_neon(vmulq_n_u32(m_lo, 256 * PREAMBLE_KK), + vmlaq_n_u32(gap_lo, m_lo, PREAMBLE_KK)); + const uint32x4_t q_hi = div_u32x4_neon(vmulq_n_u32(m_hi, 256 * PREAMBLE_KK), + vmlaq_n_u32(gap_hi, m_hi, PREAMBLE_KK)); #endif - // Metric - *mp = (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); - mp += 6; - if (plen == 0) break; - plen--; + // Metric < 256 + vst1_u8(out + k, vmovn_u16(vcombine_u16(vmovn_u32(q_lo), vmovn_u32(q_hi)))); + } +} + +// Locate preamble +int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +{ + int stride = NU_STRIDE(nt); // Stride + int count[6]; // Number of positions to evaluate in each phase + int cmin = 0x7fffffff; + + for (int j = 0; j < 6; j++) { + int plen = (nt + 5 - j) / 6; // Length of each phase + plen -= 2 * LONG_FRAME; // No point in searching at end + count[j] = plen - 15; + cmin = MIN(cmin, count[j]); + } - unsigned nv = *p++; - p1 = vextq_u16(p1, p2, 1); - p2 = vextq_u16(p2, p2, 1); - p2 = vsetq_lane_u16(nv, p2, 7); + // All phases together, a block at a time, so the outputs can be interleaved on store + int i = 0; + while (i + 8 <= cmin) { + const int n8 = MIN((cmin - i) / 8, PREAMBLE_BLOCK / 8); + uint8_t t[6][PREAMBLE_BLOCK]; + + for (int j = 0; j < 6; j++) + preamble_metric_block_neon(&pp[j*stride + i], n8, t[j]); + + // mm[6*(i+k) + j] = t[j][k]: interleave phases in pairs, then 3-way as 16-bit + uint8_t * restrict mp = &mm[6*i]; + for (int k = 0; k < 8 * n8; k += 8, mp += 48) { + const uint8x8x2_t z01 = vzip_u8(vld1_u8(t[0] + k), vld1_u8(t[1] + k)); + const uint8x8x2_t z23 = vzip_u8(vld1_u8(t[2] + k), vld1_u8(t[3] + k)); + const uint8x8x2_t z45 = vzip_u8(vld1_u8(t[4] + k), vld1_u8(t[5] + k)); + uint16x8x3_t o; + o.val[0] = vreinterpretq_u16_u8(vcombine_u8(z01.val[0], z01.val[1])); + o.val[1] = vreinterpretq_u16_u8(vcombine_u8(z23.val[0], z23.val[1])); + o.val[2] = vreinterpretq_u16_u8(vcombine_u8(z45.val[0], z45.val[1])); + vst3q_u16((uint16_t *) mp, o); } + i += 8 * n8; } + + // Remainder + for (int j = 0; j < 6; j++) + for (int k = i; k < count[j]; k++) + mm[6*k + j] = preamble_metric1(&pp[j*stride + k]); + return METRIC_SIZE(nt); } From 5a51c13efbe8e798b48e9de23bfc7a438daa8394 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 04:24:07 +0100 Subject: [PATCH 07/25] more speed improvements --- dsp/impl/demod2400x.benchmark.c | 22 ++++++++++++++++++---- dsp/impl/demod2400x.neon.c | 4 ++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/dsp/impl/demod2400x.benchmark.c b/dsp/impl/demod2400x.benchmark.c index 63b016120..335c0ee86 100644 --- a/dsp/impl/demod2400x.benchmark.c +++ b/dsp/impl/demod2400x.benchmark.c @@ -18,6 +18,12 @@ #define DEMOD2400X_BENCH_LEN 131072 // one FIFO buffer of I/Q samples #define DEMOD2400X_BENCH_SHIFT 8 +// All implementations write to the same output buffer in turn. The verify +// functions refill it with this pattern after checking, so values left by the +// previous implementation can't hide ones that were never written; the +// reference starts from the same pattern, so untouched padding still matches. +#define DEMOD2400X_BENCH_POISON 0xA5 + // Size of the upsample12 output (and locate_preamble input) for 'len' I/Q samples static size_t demod2400x_upsample_size(int len) { @@ -53,8 +59,10 @@ void STARCH_BENCHMARK(upsample12) (void) int16_t *in = demod2400x_make_iq(len); uint16_t *out = calloc(demod2400x_upsample_size(len), sizeof(*out)); - if (in && out) + if (in && out) { + memset(out, DEMOD2400X_BENCH_POISON, demod2400x_upsample_size(len) * sizeof(*out)); STARCH_BENCHMARK_RUN( upsample12, in, len, DEMOD2400X_BENCH_SHIFT, out ); + } free(in); free(out); @@ -63,10 +71,11 @@ void STARCH_BENCHMARK(upsample12) (void) bool STARCH_BENCHMARK_VERIFY(upsample12) (const int16_t *in, int len, int shift, uint16_t *out) { size_t size = demod2400x_upsample_size(len); - uint16_t *ref = calloc(size, sizeof(*ref)); + uint16_t *ref = malloc(size * sizeof(*ref)); if (!ref) return false; + memset(ref, DEMOD2400X_BENCH_POISON, size * sizeof(*ref)); starch_upsample12_generic_generic(in, len, shift, ref); bool okay = true; @@ -78,6 +87,7 @@ bool STARCH_BENCHMARK_VERIFY(upsample12) (const int16_t *in, int len, int shift, } } + memset(out, DEMOD2400X_BENCH_POISON, size * sizeof(*out)); free(ref); return okay; } @@ -92,8 +102,10 @@ void STARCH_BENCHMARK(locate_preamble) (void) if (iq && pp) { int nt = starch_upsample12_generic_generic(iq, len, DEMOD2400X_BENCH_SHIFT, pp); - if ((mm = calloc((size_t) nt, 1))) + if ((mm = malloc((size_t) nt))) { + memset(mm, DEMOD2400X_BENCH_POISON, (size_t) nt); STARCH_BENCHMARK_RUN( locate_preamble, pp, nt, mm ); + } } free(iq); @@ -103,10 +115,11 @@ void STARCH_BENCHMARK(locate_preamble) (void) bool STARCH_BENCHMARK_VERIFY(locate_preamble) (const uint16_t *pp, int nt, uint8_t *mm) { - uint8_t *ref = calloc((size_t) nt, 1); + uint8_t *ref = malloc((size_t) nt); if (!ref) return false; + memset(ref, DEMOD2400X_BENCH_POISON, (size_t) nt); starch_locate_preamble_generic_generic(pp, nt, ref); bool okay = true; @@ -118,6 +131,7 @@ bool STARCH_BENCHMARK_VERIFY(locate_preamble) (const uint16_t *pp, int nt, uint8 } } + memset(mm, DEMOD2400X_BENCH_POISON, (size_t) nt); free(ref); return okay; } diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 657962b51..7ced30957 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -69,10 +69,10 @@ uint32x4_t isqrt_u32x4_neon(const uint32x4_t x) const uint32x4_t q = vcvtq_u32_f32(vaddq_f32(vsqrtq_f32(f), vdupq_n_f32(1.0f / 64))); return vaddq_u32(q, vcgtq_u32(vmulq_u32(q, q), x)); #else - // Reciprocal square root estimate + two Newton steps: within one either way + // Reciprocal square root estimate + one Newton step: within one either way + // (worst error 0.57 for x < 2^31) float32x4_t r = vrsqrteq_f32(f); r = vmulq_f32(vrsqrtsq_f32(vmulq_f32(f, r), r), r); - r = vmulq_f32(vrsqrtsq_f32(vmulq_f32(f, r), r), r); uint32x4_t q = vcvtq_u32_f32(vmulq_f32(f, r)); // x = 0 gives NaN, converted to 0 q = vaddq_u32(q, vcgtq_u32(vmulq_u32(q, q), x)); // too large: -1 const uint32x4_t q1 = vaddq_u32(q, vdupq_n_u32(1)); From d70f5d42f5d5f5d78ce4b02d7a33fb794a9d4072 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 05:03:34 +0100 Subject: [PATCH 08/25] merged DC removal and iq data into one stage --- convert.c | 37 ++- demod_2400x.c | 65 +---- dsp/generated/benchmark.c | 372 ++++++++++++++++++++++++- dsp/generated/dispatcher.c | 63 +++-- dsp/generated/starch.h | 39 ++- dsp/helpers/demod2400x_iqconv.h | 103 +++++++ dsp/impl/demod2400x.avx2.c | 129 ++++++++- dsp/impl/demod2400x.benchmark.c | 187 ++++++++++++- dsp/impl/demod2400x.generic.c | 64 ++++- dsp/impl/demod2400x.neon.c | 128 ++++++++- dsp/impl/magnitude_power_uc8.generic.c | 35 --- dsp/impl/magnitude_sc16.generic.c | 29 -- dsp/impl/magnitude_sc16q11.generic.c | 35 --- dsp/starchgen.py | 6 +- fifo.h | 13 + sdr_bladerf.c | 2 +- sdr_hackrf.c | 2 +- sdr_ifile.c | 2 +- sdr_limesdr.c | 2 +- sdr_rtlsdr.c | 2 +- sdr_soapy.c | 2 +- 21 files changed, 1080 insertions(+), 237 deletions(-) create mode 100644 dsp/helpers/demod2400x_iqconv.h diff --git a/convert.c b/convert.c index 0e91b7f0f..8f728a0e0 100644 --- a/convert.c +++ b/convert.c @@ -27,9 +27,14 @@ // instead of magnitudes. The names match the stock converters so that // init_converter() below is the same for both builds. // -// The iqdata_conv_* kernels accept NULL for the level/power outputs. +// The iqdata_conv_* kernels also remove DC; the filter state is kept in +// converter_state between calls. They accept NULL for the level/power outputs. // +struct converter_state { + int32_t dc_offset[2]; // DC filter state (I, Q): running mean << 12 +}; + static void convert_uc8(void *iq_data, uint16_t *mag_data, unsigned nsamples, @@ -37,11 +42,9 @@ static void convert_uc8(void *iq_data, double *out_mean_level, double *out_mean_power) { - MODES_NOTUSED(state); - const uc8_t *in = (const uc8_t *) iq_data; - starch_iqdata_conv_uc8(in, mag_data, nsamples, out_mean_level, out_mean_power); + starch_iqdata_conv_uc8(in, mag_data, nsamples, state->dc_offset, state->dc_offset, out_mean_level, out_mean_power); } static void convert_sc16(void *iq_data, @@ -51,11 +54,9 @@ static void convert_sc16(void *iq_data, double *out_mean_level, double *out_mean_power) { - MODES_NOTUSED(state); - const sc16_t *in = (const sc16_t *) iq_data; - starch_iqdata_conv_sc16(in, mag_data, nsamples, out_mean_level, out_mean_power); + starch_iqdata_conv_sc16(in, mag_data, nsamples, state->dc_offset, state->dc_offset, out_mean_level, out_mean_power); } static void convert_sc16q11(void *iq_data, @@ -65,11 +66,9 @@ static void convert_sc16q11(void *iq_data, double *out_mean_level, double *out_mean_power) { - MODES_NOTUSED(state); - const sc16_t *in = (const sc16_t *) iq_data; - starch_iqdata_conv_sc16q11(in, mag_data, nsamples, out_mean_level, out_mean_power); + starch_iqdata_conv_sc16q11(in, mag_data, nsamples, state->dc_offset, state->dc_offset, out_mean_level, out_mean_power); } #else /* !DEMOD_2400X */ @@ -136,13 +135,21 @@ iq_convert_fn init_converter(input_format_t format, struct converter_state **out_state) { MODES_NOTUSED(sample_rate); - MODES_NOTUSED(out_state); if (filter_dc) { fprintf(stderr, "DC filtering not supported (yet)\n"); return NULL; } +#ifdef DEMOD_2400X + if (!(*out_state = calloc(1, sizeof(struct converter_state)))) { + fprintf(stderr, "can't allocate converter state\n"); + return NULL; + } +#else + MODES_NOTUSED(out_state); +#endif + switch (format) { case INPUT_UC8: return convert_uc8; @@ -152,12 +159,20 @@ iq_convert_fn init_converter(input_format_t format, return convert_sc16q11; default: fprintf(stderr, "no suitable converter for format=%u\n", (unsigned) format); +#ifdef DEMOD_2400X + free(*out_state); + *out_state = NULL; +#endif return NULL; } } void cleanup_converter(struct converter_state *state) { +#ifdef DEMOD_2400X + free(state); +#else MODES_NOTUSED(state); +#endif } diff --git a/demod_2400x.c b/demod_2400x.c index e360b7586..d25013cd3 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -40,18 +40,12 @@ // Static data struct staticdata_t { - int isize; - int16_t *idata; // DC-removed I/Q, laid out like the FIFO buffer: [overlap | new samples] - int usize; uint16_t *udata; int msize; uint8_t *metric; - int32_t offset_i; - int32_t offset_q; - unsigned prev_mlen; // new samples in the previous buffer (0 = no previous buffer) int next_start; // first peak position to search in this buffer (skips the rest of a // frame decoded at the end of the previous buffer) @@ -94,7 +88,6 @@ static int allocateBuffers(struct staticdata_t *stdata, int totalLength) { int stride = NU_STRIDE(nt); int mt = METRIC_SIZE(nt); - if (allocBuffer(2*totalLength*sizeof(int16_t), &stdata->isize, (void **)&stdata->idata) < 0) return -1; if (allocBuffer(6*stride*sizeof(uint16_t), &stdata->usize, (void **)&stdata->udata) < 0) return -1; if (allocBuffer(mt*sizeof(uint8_t), &stdata->msize, (void **)&stdata->metric) < 0) return -1; @@ -322,7 +315,7 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i continue; double energy; - if (emitFrame(mag, &c, bits, st->idata, &energy)) { + if (emitFrame(mag, &c, bits, (const int16_t *) mag->data, &energy)) { energy_sum += energy; // Skip to 8 bits before the end of the frame, as demod_2400.c does p = c.pos + bits * 12 - 1; @@ -336,45 +329,6 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i } -// Remove DC from 'dlen' new I/Q samples, 'in' -> 'out' (both interleaved I,Q). -// Returns the gain shift for upsample12. -#define AVG_SHIFT 12 -static void removeDcOffset(const int16_t * restrict in, - int16_t * restrict out, - struct staticdata_t * restrict stdata, - int dlen) -{ - int32_t off_i = stdata->offset_i; - int32_t off_q = stdata->offset_q; - - for (int i = 0; i < dlen; i++) { - - int32_t ii = in[0]; - int32_t qq = in[1]; - - ii -= (off_i + (1<<(AVG_SHIFT-1))) >> AVG_SHIFT; - qq -= (off_q + (1<<(AVG_SHIFT-1))) >> AVG_SHIFT; - - off_i += ii; - off_q += qq; - - if (ii > 2047) ii = 2047; - if (ii < -2048) ii = -2048; - - if (qq > 2047) qq = 2047; - if (qq < -2048) qq = -2048; - - out[0] = (int16_t) ii; - out[1] = (int16_t) qq; - - in += 2; - out += 2; - } - - stdata->offset_i = off_i; - stdata->offset_q = off_q; -} - #ifndef USE_MAGNITUDE #define DEMOD_SHIFT_ADJ 3 static int powerShift(double mean_power) // mean_power: fraction of 12-bit full scale @@ -391,7 +345,7 @@ static int powerShift(double mean_power) // mean_power: fraction of 12-bit // Demodulate 2400 //#define CHECK_USAMP //#define CHECK_POW -// mag->data contains signed 12-bit I/Q sample pairs (-2048..2047): [overlap | new samples] +// mag->data contains DC-removed signed 12-bit I/Q sample pairs (-2048..2047): [overlap | new samples] void demodulate2400x(struct mag_buf *mag) { static struct staticdata_t stdata; @@ -405,19 +359,10 @@ void demodulate2400x(struct mag_buf *mag) assert(overlap >= OVERLAP_WINDOW); - // This buffer's overlap = the filtered tail of the previous buffer, - // the same way the FIFO does it for the raw samples - if (!stdata.prev_mlen || (mag->flags & MAGBUF_DISCONTINUOUS)) { - memset(stdata.idata, 0, 2 * overlap * sizeof(int16_t)); + // No usable skip from the previous buffer + if (!stdata.prev_mlen || (mag->flags & MAGBUF_DISCONTINUOUS)) stdata.next_start = 0; - } else { - memmove(stdata.idata, stdata.idata + 2 * stdata.prev_mlen, 2 * overlap * sizeof(int16_t)); - } - // Remove DC offset from the new samples only - removeDcOffset((const int16_t *) &mag->data[2 * overlap], - stdata.idata + 2 * overlap, &stdata, (int) newlen); - #ifdef USE_MAGNITUDE int pshift = 0; // magnitude needs no gain shift #else @@ -426,7 +371,7 @@ void demodulate2400x(struct mag_buf *mag) stdata.prev_mlen = newlen; // Upsample the whole buffer - int ulen = starch_upsample12(stdata.idata, (int) dlen, pshift, stdata.udata); + int ulen = starch_upsample12((const int16_t *) mag->data, (int) dlen, pshift, stdata.udata); #ifdef CHECK_POW int max = 0; diff --git a/dsp/generated/benchmark.c b/dsp/generated/benchmark.c index e25c50d28..95fb4a6d3 100644 --- a/dsp/generated/benchmark.c +++ b/dsp/generated/benchmark.c @@ -184,6 +184,345 @@ static void starch_benchmark_run_count_above_u16( const uint16_t * arg0, unsigne } } +/* prototypes for benchmark helpers provided by user code */ +void starch_iqdata_conv_sc16_benchmark (void); +bool starch_iqdata_conv_sc16_benchmark_verify ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); + +/* prototype the benchmarking function so that we can build with -Wmissing-declarations */ +void starch_iqdata_conv_sc16_benchmark(void); + +static void starch_benchmark_one_iqdata_conv_sc16( starch_iqdata_conv_sc16_regentry * _entry, const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) +{ + fprintf(stderr, " %-40s ", _entry->name); + + /* test for support */ + if (_entry->flavor_supported && !(_entry->flavor_supported())) { + fprintf(stderr, "unsupported\n"); + return; + } + + if (starch_benchmark_flavor_whitelist && !starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_whitelist)) { + fprintf(stderr, "skipped (not whitelisted)\n"); + return; + } + + if (starch_benchmark_flavor_blacklist && starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_blacklist)) { + fprintf(stderr, "skipped (blacklisted)\n"); + return; + } + + if (starch_benchmark_list_only) { + fprintf(stderr, "supported\n"); + return; + } + + /* initial warmup */ + for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + + /* verify correctness of the output */ + if (! starch_iqdata_conv_sc16_benchmark_verify ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 )) { + fprintf(stderr, "skipped (verification failed)\n"); + starch_benchmark_validation_failed = true; + return; + } + if (starch_benchmark_validate_only) { + fprintf(stderr, "validation ok\n"); + return; + } + + /* pre-benchmark, find a loop count that takes at least 100ms */ + starch_benchmark_time _start, _end; + uint64_t _elapsed = 0; + uint64_t _loops = 127; + while (_elapsed < 100000000) { + _loops *= 2; + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + starch_benchmark_get_time(&_end); + _elapsed = starch_benchmark_elapsed(&_start, &_end); + } + + /* real benchmark, run for approx 1 second */ + _loops = _loops * 1000000000 / _elapsed; + + _elapsed = 0; + uint64_t _elapsed_min = UINT64_MAX; + uint64_t _elapsed_max = 0; + for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + starch_benchmark_get_time(&_end); + uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); + if (_elapsed_one < _elapsed_min) + _elapsed_min = _elapsed_one; + if (_elapsed_one > _elapsed_max) + _elapsed_max = _elapsed_one; + _elapsed += _elapsed_one; + } + + uint64_t _per_loop; + if (starch_benchmark_iterations > 2) + _per_loop = (_elapsed - _elapsed_min - _elapsed_max) / _loops / (starch_benchmark_iterations - 2); + else + _per_loop = _elapsed / _loops / starch_benchmark_iterations; + + fprintf(stderr, "%" PRIu64 " ns/call\n", _per_loop); + + if (starch_benchmark_result_count >= starch_benchmark_result_size) { + if (!starch_benchmark_result_size) + starch_benchmark_result_size = 64; + else + starch_benchmark_result_size *= 2; + starch_benchmark_results = realloc(starch_benchmark_results, starch_benchmark_result_size * sizeof(*starch_benchmark_results)); + if (!starch_benchmark_results) { + fprintf(stderr, "realloc: %s\n", strerror(errno)); + exit(1); + } + } + + starch_benchmark_results[starch_benchmark_result_count].name = "iqdata_conv_sc16"; + starch_benchmark_results[starch_benchmark_result_count].impl = _entry->name; + starch_benchmark_results[starch_benchmark_result_count].ns = _per_loop; + starch_benchmark_results[starch_benchmark_result_count].flavor = _entry->flavor; + ++starch_benchmark_result_count; +} + +static void starch_benchmark_run_iqdata_conv_sc16( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) +{ + for (starch_iqdata_conv_sc16_regentry *_entry = starch_iqdata_conv_sc16_registry; _entry->name; ++_entry) { + starch_benchmark_one_iqdata_conv_sc16( _entry, arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + } +} + +/* prototypes for benchmark helpers provided by user code */ +void starch_iqdata_conv_sc16q11_benchmark (void); +bool starch_iqdata_conv_sc16q11_benchmark_verify ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); + +/* prototype the benchmarking function so that we can build with -Wmissing-declarations */ +void starch_iqdata_conv_sc16q11_benchmark(void); + +static void starch_benchmark_one_iqdata_conv_sc16q11( starch_iqdata_conv_sc16q11_regentry * _entry, const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) +{ + fprintf(stderr, " %-40s ", _entry->name); + + /* test for support */ + if (_entry->flavor_supported && !(_entry->flavor_supported())) { + fprintf(stderr, "unsupported\n"); + return; + } + + if (starch_benchmark_flavor_whitelist && !starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_whitelist)) { + fprintf(stderr, "skipped (not whitelisted)\n"); + return; + } + + if (starch_benchmark_flavor_blacklist && starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_blacklist)) { + fprintf(stderr, "skipped (blacklisted)\n"); + return; + } + + if (starch_benchmark_list_only) { + fprintf(stderr, "supported\n"); + return; + } + + /* initial warmup */ + for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + + /* verify correctness of the output */ + if (! starch_iqdata_conv_sc16q11_benchmark_verify ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 )) { + fprintf(stderr, "skipped (verification failed)\n"); + starch_benchmark_validation_failed = true; + return; + } + if (starch_benchmark_validate_only) { + fprintf(stderr, "validation ok\n"); + return; + } + + /* pre-benchmark, find a loop count that takes at least 100ms */ + starch_benchmark_time _start, _end; + uint64_t _elapsed = 0; + uint64_t _loops = 127; + while (_elapsed < 100000000) { + _loops *= 2; + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + starch_benchmark_get_time(&_end); + _elapsed = starch_benchmark_elapsed(&_start, &_end); + } + + /* real benchmark, run for approx 1 second */ + _loops = _loops * 1000000000 / _elapsed; + + _elapsed = 0; + uint64_t _elapsed_min = UINT64_MAX; + uint64_t _elapsed_max = 0; + for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + starch_benchmark_get_time(&_end); + uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); + if (_elapsed_one < _elapsed_min) + _elapsed_min = _elapsed_one; + if (_elapsed_one > _elapsed_max) + _elapsed_max = _elapsed_one; + _elapsed += _elapsed_one; + } + + uint64_t _per_loop; + if (starch_benchmark_iterations > 2) + _per_loop = (_elapsed - _elapsed_min - _elapsed_max) / _loops / (starch_benchmark_iterations - 2); + else + _per_loop = _elapsed / _loops / starch_benchmark_iterations; + + fprintf(stderr, "%" PRIu64 " ns/call\n", _per_loop); + + if (starch_benchmark_result_count >= starch_benchmark_result_size) { + if (!starch_benchmark_result_size) + starch_benchmark_result_size = 64; + else + starch_benchmark_result_size *= 2; + starch_benchmark_results = realloc(starch_benchmark_results, starch_benchmark_result_size * sizeof(*starch_benchmark_results)); + if (!starch_benchmark_results) { + fprintf(stderr, "realloc: %s\n", strerror(errno)); + exit(1); + } + } + + starch_benchmark_results[starch_benchmark_result_count].name = "iqdata_conv_sc16q11"; + starch_benchmark_results[starch_benchmark_result_count].impl = _entry->name; + starch_benchmark_results[starch_benchmark_result_count].ns = _per_loop; + starch_benchmark_results[starch_benchmark_result_count].flavor = _entry->flavor; + ++starch_benchmark_result_count; +} + +static void starch_benchmark_run_iqdata_conv_sc16q11( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) +{ + for (starch_iqdata_conv_sc16q11_regentry *_entry = starch_iqdata_conv_sc16q11_registry; _entry->name; ++_entry) { + starch_benchmark_one_iqdata_conv_sc16q11( _entry, arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + } +} + +/* prototypes for benchmark helpers provided by user code */ +void starch_iqdata_conv_uc8_benchmark (void); +bool starch_iqdata_conv_uc8_benchmark_verify ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); + +/* prototype the benchmarking function so that we can build with -Wmissing-declarations */ +void starch_iqdata_conv_uc8_benchmark(void); + +static void starch_benchmark_one_iqdata_conv_uc8( starch_iqdata_conv_uc8_regentry * _entry, const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) +{ + fprintf(stderr, " %-40s ", _entry->name); + + /* test for support */ + if (_entry->flavor_supported && !(_entry->flavor_supported())) { + fprintf(stderr, "unsupported\n"); + return; + } + + if (starch_benchmark_flavor_whitelist && !starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_whitelist)) { + fprintf(stderr, "skipped (not whitelisted)\n"); + return; + } + + if (starch_benchmark_flavor_blacklist && starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_blacklist)) { + fprintf(stderr, "skipped (blacklisted)\n"); + return; + } + + if (starch_benchmark_list_only) { + fprintf(stderr, "supported\n"); + return; + } + + /* initial warmup */ + for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + + /* verify correctness of the output */ + if (! starch_iqdata_conv_uc8_benchmark_verify ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 )) { + fprintf(stderr, "skipped (verification failed)\n"); + starch_benchmark_validation_failed = true; + return; + } + if (starch_benchmark_validate_only) { + fprintf(stderr, "validation ok\n"); + return; + } + + /* pre-benchmark, find a loop count that takes at least 100ms */ + starch_benchmark_time _start, _end; + uint64_t _elapsed = 0; + uint64_t _loops = 127; + while (_elapsed < 100000000) { + _loops *= 2; + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + starch_benchmark_get_time(&_end); + _elapsed = starch_benchmark_elapsed(&_start, &_end); + } + + /* real benchmark, run for approx 1 second */ + _loops = _loops * 1000000000 / _elapsed; + + _elapsed = 0; + uint64_t _elapsed_min = UINT64_MAX; + uint64_t _elapsed_max = 0; + for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + starch_benchmark_get_time(&_end); + uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); + if (_elapsed_one < _elapsed_min) + _elapsed_min = _elapsed_one; + if (_elapsed_one > _elapsed_max) + _elapsed_max = _elapsed_one; + _elapsed += _elapsed_one; + } + + uint64_t _per_loop; + if (starch_benchmark_iterations > 2) + _per_loop = (_elapsed - _elapsed_min - _elapsed_max) / _loops / (starch_benchmark_iterations - 2); + else + _per_loop = _elapsed / _loops / starch_benchmark_iterations; + + fprintf(stderr, "%" PRIu64 " ns/call\n", _per_loop); + + if (starch_benchmark_result_count >= starch_benchmark_result_size) { + if (!starch_benchmark_result_size) + starch_benchmark_result_size = 64; + else + starch_benchmark_result_size *= 2; + starch_benchmark_results = realloc(starch_benchmark_results, starch_benchmark_result_size * sizeof(*starch_benchmark_results)); + if (!starch_benchmark_results) { + fprintf(stderr, "realloc: %s\n", strerror(errno)); + exit(1); + } + } + + starch_benchmark_results[starch_benchmark_result_count].name = "iqdata_conv_uc8"; + starch_benchmark_results[starch_benchmark_result_count].impl = _entry->name; + starch_benchmark_results[starch_benchmark_result_count].ns = _per_loop; + starch_benchmark_results[starch_benchmark_result_count].flavor = _entry->flavor; + ++starch_benchmark_result_count; +} + +static void starch_benchmark_run_iqdata_conv_uc8( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) +{ + for (starch_iqdata_conv_uc8_regentry *_entry = starch_iqdata_conv_uc8_registry; _entry->name; ++_entry) { + starch_benchmark_one_iqdata_conv_uc8( _entry, arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); + } +} + /* prototypes for benchmark helpers provided by user code */ void starch_locate_preamble_benchmark (void); bool starch_locate_preamble_benchmark_verify ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); @@ -999,6 +1338,21 @@ static void starch_benchmark_all_count_above_u16(void) fprintf(stderr, "==== count_above_u16 ===\n"); starch_count_above_u16_benchmark (); } +static void starch_benchmark_all_iqdata_conv_sc16(void) +{ + fprintf(stderr, "==== iqdata_conv_sc16 ===\n"); + starch_iqdata_conv_sc16_benchmark (); +} +static void starch_benchmark_all_iqdata_conv_sc16q11(void) +{ + fprintf(stderr, "==== iqdata_conv_sc16q11 ===\n"); + starch_iqdata_conv_sc16q11_benchmark (); +} +static void starch_benchmark_all_iqdata_conv_uc8(void) +{ + fprintf(stderr, "==== iqdata_conv_uc8 ===\n"); + starch_iqdata_conv_uc8_benchmark (); +} static void starch_benchmark_all_locate_preamble(void) { fprintf(stderr, "==== locate_preamble ===\n"); @@ -1206,20 +1560,17 @@ int main(int argc, char **argv) } if (!strcmp(argv[i], "iqdata_conv_sc16")) { specific = 1; - fprintf(stderr, "=== iqdata_conv_sc16 ===\n"); - fprintf(stderr, " (no benchmark support defined)\n"); + starch_benchmark_all_iqdata_conv_sc16(); continue; } if (!strcmp(argv[i], "iqdata_conv_sc16q11")) { specific = 1; - fprintf(stderr, "=== iqdata_conv_sc16q11 ===\n"); - fprintf(stderr, " (no benchmark support defined)\n"); + starch_benchmark_all_iqdata_conv_sc16q11(); continue; } if (!strcmp(argv[i], "iqdata_conv_uc8")) { specific = 1; - fprintf(stderr, "=== iqdata_conv_uc8 ===\n"); - fprintf(stderr, " (no benchmark support defined)\n"); + starch_benchmark_all_iqdata_conv_uc8(); continue; } if (!strcmp(argv[i], "locate_preamble")) { @@ -1264,12 +1615,9 @@ int main(int argc, char **argv) if (!specific) { starch_benchmark_all_count_above_u16(); - fprintf(stderr, "=== iqdata_conv_sc16 ===\n"); - fprintf(stderr, " (no benchmark support defined)\n"); - fprintf(stderr, "=== iqdata_conv_sc16q11 ===\n"); - fprintf(stderr, " (no benchmark support defined)\n"); - fprintf(stderr, "=== iqdata_conv_uc8 ===\n"); - fprintf(stderr, " (no benchmark support defined)\n"); + starch_benchmark_all_iqdata_conv_sc16(); + starch_benchmark_all_iqdata_conv_sc16q11(); + starch_benchmark_all_iqdata_conv_uc8(); starch_benchmark_all_locate_preamble(); starch_benchmark_all_magnitude_power_uc8(); starch_benchmark_all_magnitude_sc16(); diff --git a/dsp/generated/dispatcher.c b/dsp/generated/dispatcher.c index 52803bfc9..536a9246e 100644 --- a/dsp/generated/dispatcher.c +++ b/dsp/generated/dispatcher.c @@ -110,13 +110,13 @@ starch_iqdata_conv_sc16_regentry * starch_iqdata_conv_sc16_select() { return NULL; } -static void starch_iqdata_conv_sc16_dispatch ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ) { +static void starch_iqdata_conv_sc16_dispatch ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) { starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_select(); if (!entry) abort(); starch_iqdata_conv_sc16 = entry->callable; - starch_iqdata_conv_sc16 ( arg0, arg1, arg2, arg3, arg4 ); + starch_iqdata_conv_sc16 ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); } starch_iqdata_conv_sc16_ptr starch_iqdata_conv_sc16 = starch_iqdata_conv_sc16_dispatch; @@ -151,22 +151,25 @@ void starch_iqdata_conv_sc16_set_wisdom (const char * const * received_wisdom) starch_iqdata_conv_sc16_regentry starch_iqdata_conv_sc16_registry[] = { #ifdef STARCH_MIX_AARCH64 - { 0, "arith_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_arith_armv8_neon_simd, cpu_supports_armv8_simd }, - { 1, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, + { 0, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_generic_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 2, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_AARCH64 */ #ifdef STARCH_MIX_ARM - { 0, "arith_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_arith_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, - { 1, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, + { 0, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 2, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_ARM */ #ifdef STARCH_MIX_GENERIC - { 0, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, + { 0, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "arith_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_arith_x86_avx2, cpu_supports_avx2 }, - { 1, "arith_generic", "generic", starch_iqdata_conv_sc16_arith_generic, NULL }, + { 0, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_generic_x86_avx2, cpu_supports_avx2 }, + { 1, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_avx2_x86_avx2, cpu_supports_avx2 }, + { 2, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } }; @@ -185,13 +188,13 @@ starch_iqdata_conv_sc16q11_regentry * starch_iqdata_conv_sc16q11_select() { return NULL; } -static void starch_iqdata_conv_sc16q11_dispatch ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ) { +static void starch_iqdata_conv_sc16q11_dispatch ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) { starch_iqdata_conv_sc16q11_regentry *entry = starch_iqdata_conv_sc16q11_select(); if (!entry) abort(); starch_iqdata_conv_sc16q11 = entry->callable; - starch_iqdata_conv_sc16q11 ( arg0, arg1, arg2, arg3, arg4 ); + starch_iqdata_conv_sc16q11 ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); } starch_iqdata_conv_sc16q11_ptr starch_iqdata_conv_sc16q11 = starch_iqdata_conv_sc16q11_dispatch; @@ -226,22 +229,25 @@ void starch_iqdata_conv_sc16q11_set_wisdom (const char * const * received_wisdom starch_iqdata_conv_sc16q11_regentry starch_iqdata_conv_sc16q11_registry[] = { #ifdef STARCH_MIX_AARCH64 - { 0, "arith_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_arith_armv8_neon_simd, cpu_supports_armv8_simd }, - { 1, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, + { 0, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_generic_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 2, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_AARCH64 */ #ifdef STARCH_MIX_ARM - { 0, "arith_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_arith_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, - { 1, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, + { 0, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 2, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_ARM */ #ifdef STARCH_MIX_GENERIC - { 0, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, + { 0, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "arith_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_arith_x86_avx2, cpu_supports_avx2 }, - { 1, "arith_generic", "generic", starch_iqdata_conv_sc16q11_arith_generic, NULL }, + { 0, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_generic_x86_avx2, cpu_supports_avx2 }, + { 1, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_avx2_x86_avx2, cpu_supports_avx2 }, + { 2, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } }; @@ -260,13 +266,13 @@ starch_iqdata_conv_uc8_regentry * starch_iqdata_conv_uc8_select() { return NULL; } -static void starch_iqdata_conv_uc8_dispatch ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ) { +static void starch_iqdata_conv_uc8_dispatch ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ) { starch_iqdata_conv_uc8_regentry *entry = starch_iqdata_conv_uc8_select(); if (!entry) abort(); starch_iqdata_conv_uc8 = entry->callable; - starch_iqdata_conv_uc8 ( arg0, arg1, arg2, arg3, arg4 ); + starch_iqdata_conv_uc8 ( arg0, arg1, arg2, arg3, arg4, arg5, arg6 ); } starch_iqdata_conv_uc8_ptr starch_iqdata_conv_uc8 = starch_iqdata_conv_uc8_dispatch; @@ -301,22 +307,25 @@ void starch_iqdata_conv_uc8_set_wisdom (const char * const * received_wisdom) starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[] = { #ifdef STARCH_MIX_AARCH64 - { 0, "arith_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_arith_armv8_neon_simd, cpu_supports_armv8_simd }, - { 1, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, + { 0, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_generic_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 2, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_AARCH64 */ #ifdef STARCH_MIX_ARM - { 0, "arith_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_arith_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, - { 1, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, + { 0, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 2, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_ARM */ #ifdef STARCH_MIX_GENERIC - { 0, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, + { 0, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "arith_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_arith_x86_avx2, cpu_supports_avx2 }, - { 1, "arith_generic", "generic", starch_iqdata_conv_uc8_arith_generic, NULL }, + { 0, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_generic_x86_avx2, cpu_supports_avx2 }, + { 1, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_avx2_x86_avx2, cpu_supports_avx2 }, + { 2, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } }; diff --git a/dsp/generated/starch.h b/dsp/generated/starch.h index dc3980b18..bffbce0df 100644 --- a/dsp/generated/starch.h +++ b/dsp/generated/starch.h @@ -138,7 +138,7 @@ extern starch_count_above_u16_regentry starch_count_above_u16_registry[]; starch_count_above_u16_regentry * starch_count_above_u16_select(); void starch_count_above_u16_set_wisdom( const char * const * received_wisdom ); -typedef void (* starch_iqdata_conv_uc8_ptr) ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +typedef void (* starch_iqdata_conv_uc8_ptr) ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); extern starch_iqdata_conv_uc8_ptr starch_iqdata_conv_uc8; typedef struct { @@ -153,7 +153,7 @@ extern starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[]; starch_iqdata_conv_uc8_regentry * starch_iqdata_conv_uc8_select(); void starch_iqdata_conv_uc8_set_wisdom( const char * const * received_wisdom ); -typedef void (* starch_iqdata_conv_sc16_ptr) ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +typedef void (* starch_iqdata_conv_sc16_ptr) ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); extern starch_iqdata_conv_sc16_ptr starch_iqdata_conv_sc16; typedef struct { @@ -168,7 +168,7 @@ extern starch_iqdata_conv_sc16_regentry starch_iqdata_conv_sc16_registry[]; starch_iqdata_conv_sc16_regentry * starch_iqdata_conv_sc16_select(); void starch_iqdata_conv_sc16_set_wisdom( const char * const * received_wisdom ); -typedef void (* starch_iqdata_conv_sc16q11_ptr) ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); +typedef void (* starch_iqdata_conv_sc16q11_ptr) ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); extern starch_iqdata_conv_sc16q11_ptr starch_iqdata_conv_sc16q11; typedef struct { @@ -221,17 +221,17 @@ int cpu_supports_armv7_neon_vfpv4 (void); void starch_magnitude_power_uc8_twopass_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_iqdata_conv_uc8_arith_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_armv7a_neon_vfpv4 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_generic_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_generic_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_generic_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_magnitude_sc16_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16_arith_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16q11_arith_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); @@ -247,6 +247,9 @@ void starch_magnitude_sc16_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, void starch_magnitude_sc16q11_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); int starch_upsample12_neon_armv7a_neon_vfpv4 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_neon_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_neon_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_neon_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); #endif /* STARCH_FLAVOR_ARMV7A_NEON_VFPV4 */ int starch_read_wisdom (const char * path); @@ -256,17 +259,17 @@ int cpu_supports_armv8_simd (void); void starch_magnitude_power_uc8_twopass_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_iqdata_conv_uc8_arith_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_armv8_neon_simd ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_generic_armv8_neon_simd ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_generic_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_generic_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_generic_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_magnitude_sc16_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16_arith_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16q11_arith_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); @@ -282,6 +285,9 @@ void starch_magnitude_sc16_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, u void starch_magnitude_sc16q11_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); int starch_upsample12_neon_armv8_neon_simd ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_neon_armv8_neon_simd ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_neon_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_neon_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_neon_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); #endif /* STARCH_FLAVOR_ARMV8_NEON_SIMD */ int starch_read_wisdom (const char * path); @@ -290,17 +296,17 @@ int starch_read_wisdom (const char * path); void starch_magnitude_power_uc8_twopass_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_iqdata_conv_uc8_arith_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_generic ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_generic_generic ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_generic_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_generic_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_generic_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_magnitude_sc16_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16_arith_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16q11_arith_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_generic ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_generic ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_generic ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); @@ -317,17 +323,17 @@ int cpu_supports_avx2 (void); void starch_magnitude_power_uc8_twopass_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); -void starch_iqdata_conv_uc8_arith_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_x86_avx2 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_generic_x86_avx2 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_generic_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_generic_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_generic_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_magnitude_sc16_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16_arith_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_sc16q11_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_11bit_table_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_12bit_table_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); -void starch_iqdata_conv_sc16q11_arith_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_mean_power_u16_float_x86_avx2 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u32_x86_avx2 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); void starch_mean_power_u16_u64_x86_avx2 ( const uint16_t * arg0, unsigned arg1, double * arg2, double * arg3 ); @@ -337,6 +343,9 @@ void starch_magnitude_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_ void starch_magnitude_uc8_exact_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); int starch_upsample12_avx2_x86_avx2 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); int starch_locate_preamble_avx2_x86_avx2 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +void starch_iqdata_conv_uc8_avx2_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16_avx2_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_iqdata_conv_sc16q11_avx2_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); #endif /* STARCH_FLAVOR_X86_AVX2 */ int starch_read_wisdom (const char * path); diff --git a/dsp/helpers/demod2400x_iqconv.h b/dsp/helpers/demod2400x_iqconv.h new file mode 100644 index 000000000..009667494 --- /dev/null +++ b/dsp/helpers/demod2400x_iqconv.h @@ -0,0 +1,103 @@ +// Part of dump1090, a Mode S message decoder for RTLSDR devices. +// +// demod2400x_iqconv.h: DC removal and level/power sums shared by the +// iqdata_conv_* starch kernels (dsp/impl/demod2400x.*.c). The SIMD versions +// use these for their tails, so every flavour gives the same bits. +// +// This file is free software: you may copy, redistribute and/or modify it +// under the terms of the GNU General Public License as published by the +// Free Software Foundation, either version 2 of the License, or (at your +// option) any later version. + +#ifndef DEMOD2400X_IQCONV_H +#define DEMOD2400X_IQCONV_H + +#include + +#include "dsp-types.h" +#include "compat/compat.h" +#include "dsp/helpers/demod2400x_coeffs.h" + +// DC filter: offset[] holds the running mean << IQ_DC_SHIFT (time constant 4096 samples). +// The correction is held for a block of IQ_DC_BLOCK samples, then the offset is +// updated with the block's sum, so the SIMD versions need no serial loop. +#define IQ_DC_SHIFT 12 +#define IQ_DC_BLOCK 16 + +// Level is summed as floor(sqrt(magsq << IQ_LEVEL_SHIFT)), i.e. magnitude * 8 +#define IQ_LEVEL_SHIFT 6 + +// Raw samples -> 12-bit signed interleaved I,Q (before DC removal) +static inline void iqconv_raw_uc8(const uc8_t *in, int16_t *raw, unsigned n) +{ + for (unsigned k = 0; k < n; ++k) { + raw[2 * k] = (int16_t) ((2 * in[k].I - 255) * 8); // -2040..+2040 + raw[2 * k + 1] = (int16_t) ((2 * in[k].Q - 255) * 8); + } +} + +static inline void iqconv_raw_sc16(const sc16_t *in, int16_t *raw, unsigned n) +{ + for (unsigned k = 0; k < n; ++k) { + raw[2 * k] = (int16_t) ((int16_t) le16toh(in[k].I) >> 4); + raw[2 * k + 1] = (int16_t) ((int16_t) le16toh(in[k].Q) >> 4); + } +} + +static inline int16_t iqconv_clamp12(int32_t v) +{ + return (int16_t) (v > 2047 ? 2047 : v < -2048 ? -2048 : v); +} + +static inline void iqconv_raw_sc16q11(const sc16_t *in, int16_t *raw, unsigned n) +{ + for (unsigned k = 0; k < n; ++k) { + raw[2 * k] = iqconv_clamp12((int16_t) le16toh(in[k].I)); + raw[2 * k + 1] = iqconv_clamp12((int16_t) le16toh(in[k].Q)); + } +} + +// Remove DC from one block of n <= IQ_DC_BLOCK samples, clamp to 12 bits, +// and add the block to the level/power sums +static inline void iqconv_dc_block(const int16_t *raw, int16_t *out, unsigned n, + int32_t *offset, uint64_t *sum_level, uint64_t *sum_power) +{ + const int32_t corr_i = (offset[0] + (1 << (IQ_DC_SHIFT - 1))) >> IQ_DC_SHIFT; + const int32_t corr_q = (offset[1] + (1 << (IQ_DC_SHIFT - 1))) >> IQ_DC_SHIFT; + int32_t sum_i = 0, sum_q = 0; + uint32_t level = 0, power = 0; + + for (unsigned k = 0; k < n; ++k) { + int32_t ii = raw[2 * k] - corr_i; + int32_t qq = raw[2 * k + 1] - corr_q; + sum_i += ii; + sum_q += qq; + + ii = iqconv_clamp12(ii); + qq = iqconv_clamp12(qq); + out[2 * k] = (int16_t) ii; + out[2 * k + 1] = (int16_t) qq; + + uint32_t magsq = (uint32_t) (ii * ii + qq * qq); // <= 2^23 + power += magsq; + level += demod_isqrt(magsq << IQ_LEVEL_SHIFT); + } + + offset[0] += sum_i; + offset[1] += sum_q; + *sum_level += level; + *sum_power += power; +} + +// Mean level and power, normalised so 12-bit full scale (2048) = 1.0 +static inline void iqconv_stats(uint64_t sum_level, uint64_t sum_power, unsigned len, + double *out_level, double *out_power) +{ + if (out_level != NULL) + *out_level = len ? (double) sum_level / (1 << (IQ_LEVEL_SHIFT / 2)) / 2048.0 / len : 0; + + if (out_power != NULL) + *out_power = len ? (double) sum_power / 2048.0 / 2048.0 / len : 0; +} + +#endif diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index 9e44f5435..e8f2075f4 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -1,6 +1,6 @@ // Part of dump1090, a Mode S message decoder for RTLSDR devices. // -// demod2400x.avx2.c: DEMOD_2400X upsampler and preamble search, x86 AVX2. +// demod2400x.avx2.c: DEMOD_2400X input conversion, upsampler and preamble search, x86 AVX2. // Structured the same way as demod2400x.neon.c. // // This file is free software: you may copy, redistribute and/or modify it @@ -16,6 +16,7 @@ #include "dsp/helpers/demod2400x_dsp.h" #include "dsp/helpers/demod2400x_coeffs.h" +#include "dsp/helpers/demod2400x_iqconv.h" // Two vectors of 8 x int16: val[0] = I, val[1] = Q (same layout as NEON's int16x8x2_t) typedef struct { @@ -321,3 +322,129 @@ int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, ui return METRIC_SIZE(nt); } + + +// Input conversion, as the generic iqdata_conv_* (dsp/helpers/demod2400x_iqconv.h) + +// 16 raw samples as 12-bit interleaved I,Q (samples 0-7 in [0], 8-15 in [1]) +typedef struct { + __m256i v[2]; +} iq16x16_avx2_t; + +static inline __attribute__((always_inline)) iq16x16_avx2_t iqconv_load_uc8_avx2(const uc8_t *in) +{ + const __m256i c = _mm256_set1_epi16(2040); + iq16x16_avx2_t r; + // (2 * v - 255) * 8 = 16 * v - 2040 + r.v[0] = _mm256_sub_epi16(_mm256_slli_epi16(_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *) in)), 4), c); + r.v[1] = _mm256_sub_epi16(_mm256_slli_epi16(_mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i *) (in + 8))), 4), c); + return r; +} + +static inline __attribute__((always_inline)) iq16x16_avx2_t iqconv_load_sc16_avx2(const sc16_t *in) +{ + iq16x16_avx2_t r; + r.v[0] = _mm256_srai_epi16(_mm256_loadu_si256((const __m256i *) in), 4); + r.v[1] = _mm256_srai_epi16(_mm256_loadu_si256((const __m256i *) (in + 8)), 4); + return r; +} + +static inline __attribute__((always_inline)) __m256i clamp12_s16x16_avx2(const __m256i v) +{ + return _mm256_max_epi16(_mm256_min_epi16(v, _mm256_set1_epi16(2047)), _mm256_set1_epi16(-2048)); +} + +static inline __attribute__((always_inline)) iq16x16_avx2_t iqconv_load_sc16q11_avx2(const sc16_t *in) +{ + iq16x16_avx2_t r; + r.v[0] = clamp12_s16x16_avx2(_mm256_loadu_si256((const __m256i *) in)); + r.v[1] = clamp12_s16x16_avx2(_mm256_loadu_si256((const __m256i *) (in + 8))); + return r; +} + +// Add the 8 x u32 lanes of v into the 4 x u64 lanes of acc +static inline __attribute__((always_inline)) __m256i acc_u32x8_u64_avx2(const __m256i acc, const __m256i v) +{ + const __m256i zero = _mm256_setzero_si256(); + return _mm256_add_epi64(acc, _mm256_add_epi64(_mm256_unpacklo_epi32(v, zero), _mm256_unpackhi_epi32(v, zero))); +} + +// Same as iqconv_dc_block() for a full block. The block sum of (raw - corr) is +// sum(raw) - 16 * corr, so the serial part is just a few scalar ops. +static inline __attribute__((always_inline)) +void iqconv_block_avx2(const iq16x16_avx2_t r, int16_t * restrict out, int32_t *off_i, int32_t *off_q, + __m256i *acc_level, __m256i *acc_power) +{ + const int32_t corr_i = (*off_i + (1 << (IQ_DC_SHIFT - 1))) >> IQ_DC_SHIFT; + const int32_t corr_q = (*off_q + (1 << (IQ_DC_SHIFT - 1))) >> IQ_DC_SHIFT; + + // Raw sums: madd with (1, 0) picks I, with (0, 1) picks Q + const __m256i rs = _mm256_add_epi16(r.v[0], r.v[1]); + __m256i s = _mm256_hadd_epi32(_mm256_madd_epi16(rs, _mm256_set1_epi32(0x00000001)), + _mm256_madd_epi16(rs, _mm256_set1_epi32(0x00010000))); + __m128i s4 = _mm_add_epi32(_mm256_castsi256_si128(s), _mm256_extracti128_si256(s, 1)); // [I, I, Q, Q] + s4 = _mm_hadd_epi32(s4, s4); // [I, Q, I, Q] + *off_i += _mm_cvtsi128_si32(s4) - 16 * corr_i; + *off_q += _mm_extract_epi32(s4, 1) - 16 * corr_q; + + const __m256i corr = _mm256_set1_epi32((int32_t) (((uint32_t) corr_q << 16) | (uint16_t) corr_i)); + + __m256i level = _mm256_setzero_si256(), power = _mm256_setzero_si256(); + for (int h = 0; h < 2; ++h) { + const __m256i x = clamp12_s16x16_avx2(_mm256_sub_epi16(r.v[h], corr)); + _mm256_storeu_si256((__m256i *) (out + 16 * h), x); + + const __m256i m = _mm256_madd_epi16(x, x); // I * I + Q * Q per sample + power = _mm256_add_epi32(power, m); + level = _mm256_add_epi32(level, isqrt_u32x8_avx2(_mm256_slli_epi32(m, IQ_LEVEL_SHIFT))); + } + + *acc_level = acc_u32x8_u64_avx2(*acc_level, level); + *acc_power = acc_u32x8_u64_avx2(*acc_power, power); +} + +static inline __attribute__((always_inline)) uint64_t hsum_u64x4_avx2(const __m256i v) +{ + const __m128i s = _mm_add_epi64(_mm256_castsi256_si128(v), _mm256_extracti128_si256(v, 1)); + return (uint64_t) _mm_cvtsi128_si64(s) + (uint64_t) _mm_extract_epi64(s, 1); +} + +#define IQDATA_CONV_AVX2_BODY(fmt) \ + int16_t * restrict out = (int16_t *) out_u; \ + int32_t off_i = dc_in[0], off_q = dc_in[1]; \ + __m256i acc_level = _mm256_setzero_si256(), acc_power = _mm256_setzero_si256(); \ + unsigned n = 0; \ + \ + for (; n + IQ_DC_BLOCK <= len; n += IQ_DC_BLOCK) \ + iqconv_block_avx2(iqconv_load_##fmt##_avx2(in + n), out + 2 * n, &off_i, &off_q, &acc_level, &acc_power); \ + \ + dc_out[0] = off_i; \ + dc_out[1] = off_q; \ + uint64_t sum_level = hsum_u64x4_avx2(acc_level); \ + uint64_t sum_power = hsum_u64x4_avx2(acc_power); \ + \ + if (n < len) { \ + int16_t raw[2 * IQ_DC_BLOCK]; \ + iqconv_raw_##fmt(in + n, raw, len - n); \ + iqconv_dc_block(raw, out + 2 * n, len - n, dc_out, &sum_level, &sum_power); \ + } \ + \ + iqconv_stats(sum_level, sum_power, len, out_level, out_power); + +void STARCH_IMPL(iqdata_conv_uc8, avx2) (const uc8_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + IQDATA_CONV_AVX2_BODY(uc8) +} + +void STARCH_IMPL(iqdata_conv_sc16, avx2) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + IQDATA_CONV_AVX2_BODY(sc16) +} + +void STARCH_IMPL(iqdata_conv_sc16q11, avx2) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + IQDATA_CONV_AVX2_BODY(sc16q11) +} diff --git a/dsp/impl/demod2400x.benchmark.c b/dsp/impl/demod2400x.benchmark.c index 335c0ee86..1abf020ca 100644 --- a/dsp/impl/demod2400x.benchmark.c +++ b/dsp/impl/demod2400x.benchmark.c @@ -1,6 +1,7 @@ // Part of dump1090, a Mode S message decoder for RTLSDR devices. // -// demod2400x.benchmark.c: starch benchmarks for the DEMOD_2400X kernels. +// demod2400x.benchmark.c: starch benchmarks for the DEMOD_2400X kernels +// (input conversion, upsampler and preamble search). // Every implementation is checked to be bit-exact against the plain C // version (generic_generic) before it is timed. // @@ -135,3 +136,187 @@ bool STARCH_BENCHMARK_VERIFY(locate_preamble) (const uint16_t *pp, int nt, uint8 free(ref); return okay; } + + +// Input conversion. Every call starts the DC filter from the same dc_in, so the +// output and the final state (dc_out) don't depend on earlier calls. +// The length isn't a multiple of IQ_DC_BLOCK, so the tail code is checked too. +#define IQCONV_BENCH_LEN (DEMOD2400X_BENCH_LEN + 5) + +static const int32_t iqconv_bench_dc[2] = { 40 * 4096, -25 * 4096 }; +static int32_t iqconv_bench_dc_out[2]; +static double iqconv_bench_level, iqconv_bench_power; + +// Raw I/Q with a DC offset, background noise, bursts and some full-scale samples, +// scaled by 'scale' around 'centre' and limited to lo..hi +static void iqconv_bench_fill(int32_t *v, unsigned n, int scale, int centre, int lo, int hi) +{ + srand(2); + for (unsigned k = 0; k < n; k += 2) { + int amp = (rand() % 50 == 0) ? 2047 : 200; + int i = 40 + (rand() % (2 * amp + 1)) - amp; + int q = -25 + (rand() % (2 * amp + 1)) - amp; + if (rand() % 1000 == 0) { + i = 3000; + q = -3000; + } + i = i * scale + centre; + q = q * scale + centre; + v[k] = i < lo ? lo : i > hi ? hi : i; + v[k + 1] = q < lo ? lo : q > hi ? hi : q; + } +} + +static int32_t *iqconv_bench_raw(int scale, int centre, int lo, int hi) +{ + int32_t *v = malloc(2 * (size_t) IQCONV_BENCH_LEN * sizeof(*v)); + if (v) + iqconv_bench_fill(v, 2 * IQCONV_BENCH_LEN, scale, centre, lo, hi); + return v; +} + +static uint16_t *iqconv_bench_out(void) +{ + uint16_t *out = malloc(2 * (size_t) IQCONV_BENCH_LEN * sizeof(*out)); + if (out) + memset(out, DEMOD2400X_BENCH_POISON, 2 * (size_t) IQCONV_BENCH_LEN * sizeof(*out)); + memset(iqconv_bench_dc_out, DEMOD2400X_BENCH_POISON, sizeof(iqconv_bench_dc_out)); + return out; +} + +// Compare against the reference, then poison the outputs for the next implementation +static bool iqconv_bench_check(uint16_t *out, const uint16_t *ref, unsigned len, int32_t *dc, const int32_t *ref_dc, + double level, double ref_level, double power, double ref_power) +{ + bool okay = true; + for (unsigned k = 0; k < 2 * len; ++k) { + if (out[k] != ref[k]) { + fprintf(stderr, "verification failed: out[%u]=%d, expected %d\n", k, (int16_t) out[k], (int16_t) ref[k]); + okay = false; + break; + } + } + if (okay && (dc[0] != ref_dc[0] || dc[1] != ref_dc[1])) { + fprintf(stderr, "verification failed: dc_offset=(%d,%d), expected (%d,%d)\n", dc[0], dc[1], ref_dc[0], ref_dc[1]); + okay = false; + } + if (okay && (level != ref_level || power != ref_power)) { + fprintf(stderr, "verification failed: level=%.9g power=%.9g, expected %.9g %.9g\n", level, power, ref_level, ref_power); + okay = false; + } + + memset(out, DEMOD2400X_BENCH_POISON, 2 * (size_t) len * sizeof(*out)); + memset(dc, DEMOD2400X_BENCH_POISON, 2 * sizeof(*dc)); + return okay; +} + + +void STARCH_BENCHMARK(iqdata_conv_uc8) (void) +{ + int32_t *raw = iqconv_bench_raw(1, 2048, 0, 4095); // 12-bit offset binary, reduced to 8 bits below + uc8_t *in = malloc((size_t) IQCONV_BENCH_LEN * sizeof(*in)); + uint16_t *out = iqconv_bench_out(); + + if (raw && in && out) { + for (unsigned n = 0; n < IQCONV_BENCH_LEN; ++n) { + in[n].I = (uint8_t) (raw[2 * n] >> 4); + in[n].Q = (uint8_t) (raw[2 * n + 1] >> 4); + } + STARCH_BENCHMARK_RUN( iqdata_conv_uc8, in, out, IQCONV_BENCH_LEN, iqconv_bench_dc, iqconv_bench_dc_out, &iqconv_bench_level, &iqconv_bench_power ); + } + + free(raw); + free(in); + free(out); +} + +bool STARCH_BENCHMARK_VERIFY(iqdata_conv_uc8) (const uc8_t *in, uint16_t *out, unsigned len, const int32_t *dc_in, int32_t *dc, + double *level, double *power) +{ + uint16_t *ref = malloc(2 * (size_t) len * sizeof(*ref)); + if (!ref) + return false; + + int32_t ref_dc[2]; + double ref_level, ref_power; + memset(ref, DEMOD2400X_BENCH_POISON, 2 * (size_t) len * sizeof(*ref)); + starch_iqdata_conv_uc8_generic_generic(in, ref, len, dc_in, ref_dc, &ref_level, &ref_power); + + bool okay = iqconv_bench_check(out, ref, len, dc, ref_dc, *level, ref_level, *power, ref_power); + free(ref); + return okay; +} + + +void STARCH_BENCHMARK(iqdata_conv_sc16) (void) +{ + int32_t *raw = iqconv_bench_raw(16, 0, -32768, 32767); + sc16_t *in = malloc((size_t) IQCONV_BENCH_LEN * sizeof(*in)); + uint16_t *out = iqconv_bench_out(); + + if (raw && in && out) { + for (unsigned n = 0; n < IQCONV_BENCH_LEN; ++n) { + in[n].I = (uint16_t) htole16((uint16_t) (raw[2 * n] + (n & 15))); // low bits dropped by >> 4 + in[n].Q = (uint16_t) htole16((uint16_t) (raw[2 * n + 1] + (n & 15))); + } + STARCH_BENCHMARK_RUN( iqdata_conv_sc16, in, out, IQCONV_BENCH_LEN, iqconv_bench_dc, iqconv_bench_dc_out, &iqconv_bench_level, &iqconv_bench_power ); + } + + free(raw); + free(in); + free(out); +} + +bool STARCH_BENCHMARK_VERIFY(iqdata_conv_sc16) (const sc16_t *in, uint16_t *out, unsigned len, const int32_t *dc_in, int32_t *dc, + double *level, double *power) +{ + uint16_t *ref = malloc(2 * (size_t) len * sizeof(*ref)); + if (!ref) + return false; + + int32_t ref_dc[2]; + double ref_level, ref_power; + memset(ref, DEMOD2400X_BENCH_POISON, 2 * (size_t) len * sizeof(*ref)); + starch_iqdata_conv_sc16_generic_generic(in, ref, len, dc_in, ref_dc, &ref_level, &ref_power); + + bool okay = iqconv_bench_check(out, ref, len, dc, ref_dc, *level, ref_level, *power, ref_power); + free(ref); + return okay; +} + + +void STARCH_BENCHMARK(iqdata_conv_sc16q11) (void) +{ + int32_t *raw = iqconv_bench_raw(1, 0, -4096, 4095); // includes values beyond 12 bits + sc16_t *in = malloc((size_t) IQCONV_BENCH_LEN * sizeof(*in)); + uint16_t *out = iqconv_bench_out(); + + if (raw && in && out) { + for (unsigned n = 0; n < IQCONV_BENCH_LEN; ++n) { + in[n].I = (uint16_t) htole16((uint16_t) raw[2 * n]); + in[n].Q = (uint16_t) htole16((uint16_t) raw[2 * n + 1]); + } + STARCH_BENCHMARK_RUN( iqdata_conv_sc16q11, in, out, IQCONV_BENCH_LEN, iqconv_bench_dc, iqconv_bench_dc_out, &iqconv_bench_level, &iqconv_bench_power ); + } + + free(raw); + free(in); + free(out); +} + +bool STARCH_BENCHMARK_VERIFY(iqdata_conv_sc16q11) (const sc16_t *in, uint16_t *out, unsigned len, const int32_t *dc_in, int32_t *dc, + double *level, double *power) +{ + uint16_t *ref = malloc(2 * (size_t) len * sizeof(*ref)); + if (!ref) + return false; + + int32_t ref_dc[2]; + double ref_level, ref_power; + memset(ref, DEMOD2400X_BENCH_POISON, 2 * (size_t) len * sizeof(*ref)); + starch_iqdata_conv_sc16q11_generic_generic(in, ref, len, dc_in, ref_dc, &ref_level, &ref_power); + + bool okay = iqconv_bench_check(out, ref, len, dc, ref_dc, *level, ref_level, *power, ref_power); + free(ref); + return okay; +} diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c index 9e6817aa0..dd7a2328d 100644 --- a/dsp/impl/demod2400x.generic.c +++ b/dsp/impl/demod2400x.generic.c @@ -1,6 +1,6 @@ // Part of dump1090, a Mode S message decoder for RTLSDR devices. // -// demod2400x.generic.c: DEMOD_2400X upsampler and preamble search, plain C. +// demod2400x.generic.c: DEMOD_2400X input conversion, upsampler and preamble search, plain C. // Also built into every SIMD flavour, where the compiler may auto-vectorise it. // // This file is free software: you may copy, redistribute and/or modify it @@ -10,6 +10,7 @@ #include "dsp/helpers/demod2400x_dsp.h" #include "dsp/helpers/demod2400x_coeffs.h" +#include "dsp/helpers/demod2400x_iqconv.h" // Get power static inline unsigned getpow_generic( @@ -164,3 +165,64 @@ int STARCH_IMPL(locate_preamble, generic) (const uint16_t * restrict pp, int nt, return METRIC_SIZE(nt); } + + +// Input conversion: raw samples -> DC-removed 12-bit signed interleaved I,Q in the +// FIFO buffer, plus mean level and power. The DC filter starts from dc_in[2] (I, Q) +// and leaves its final state in dc_out[2]; they may be the same array. +void STARCH_IMPL(iqdata_conv_uc8, generic) (const uc8_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer + int16_t raw[2 * IQ_DC_BLOCK]; + int32_t offset[2] = { dc_in[0], dc_in[1] }; + uint64_t sum_level = 0, sum_power = 0; + + for (unsigned n = 0; n < len; n += IQ_DC_BLOCK) { + unsigned nb = MIN(IQ_DC_BLOCK, len - n); + iqconv_raw_uc8(in + n, raw, nb); + iqconv_dc_block(raw, out + 2 * n, nb, offset, &sum_level, &sum_power); + } + + dc_out[0] = offset[0]; + dc_out[1] = offset[1]; + iqconv_stats(sum_level, sum_power, len, out_level, out_power); +} + +void STARCH_IMPL(iqdata_conv_sc16, generic) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer + int16_t raw[2 * IQ_DC_BLOCK]; + int32_t offset[2] = { dc_in[0], dc_in[1] }; + uint64_t sum_level = 0, sum_power = 0; + + for (unsigned n = 0; n < len; n += IQ_DC_BLOCK) { + unsigned nb = MIN(IQ_DC_BLOCK, len - n); + iqconv_raw_sc16(in + n, raw, nb); + iqconv_dc_block(raw, out + 2 * n, nb, offset, &sum_level, &sum_power); + } + + dc_out[0] = offset[0]; + dc_out[1] = offset[1]; + iqconv_stats(sum_level, sum_power, len, out_level, out_power); +} + +void STARCH_IMPL(iqdata_conv_sc16q11, generic) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer + int16_t raw[2 * IQ_DC_BLOCK]; + int32_t offset[2] = { dc_in[0], dc_in[1] }; + uint64_t sum_level = 0, sum_power = 0; + + for (unsigned n = 0; n < len; n += IQ_DC_BLOCK) { + unsigned nb = MIN(IQ_DC_BLOCK, len - n); + iqconv_raw_sc16q11(in + n, raw, nb); + iqconv_dc_block(raw, out + 2 * n, nb, offset, &sum_level, &sum_power); + } + + dc_out[0] = offset[0]; + dc_out[1] = offset[1]; + iqconv_stats(sum_level, sum_power, len, out_level, out_power); +} diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 7ced30957..5d2ddc85d 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -1,6 +1,6 @@ // Part of dump1090, a Mode S message decoder for RTLSDR devices. // -// demod2400x.neon.c: DEMOD_2400X upsampler and preamble search, ARM NEON. +// demod2400x.neon.c: DEMOD_2400X input conversion, upsampler and preamble search, ARM NEON. // Built for both 32-bit (armv7a_neon_vfpv4) and 64-bit (armv8_neon_simd) ARM. // // This file is free software: you may copy, redistribute and/or modify it @@ -16,6 +16,7 @@ #include "dsp/helpers/demod2400x_dsp.h" #include "dsp/helpers/demod2400x_coeffs.h" +#include "dsp/helpers/demod2400x_iqconv.h" // Horizontal add of two vectors at once: returns [sum of a, sum of b] static inline __attribute__((always_inline)) @@ -375,3 +376,128 @@ int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, ui return METRIC_SIZE(nt); } + + +// Input conversion, as the generic iqdata_conv_* (dsp/helpers/demod2400x_iqconv.h) + +// 16 raw samples as 12-bit I and Q (samples 0-7 in [0], 8-15 in [1]) +typedef struct { + int16x8_t i[2]; + int16x8_t q[2]; +} iq16x16_neon_t; + +static inline __attribute__((always_inline)) iq16x16_neon_t iqconv_load_uc8_neon(const uc8_t *in) +{ + const uint8x16x2_t v = vld2q_u8((const uint8_t *) in); // I0-15, Q0-15 + const int16x8_t c = vdupq_n_s16(2040); + iq16x16_neon_t r; + // (2 * v - 255) * 8 = 16 * v - 2040 + r.i[0] = vsubq_s16(vreinterpretq_s16_u16(vshll_n_u8(vget_low_u8(v.val[0]), 4)), c); + r.i[1] = vsubq_s16(vreinterpretq_s16_u16(vshll_n_u8(vget_high_u8(v.val[0]), 4)), c); + r.q[0] = vsubq_s16(vreinterpretq_s16_u16(vshll_n_u8(vget_low_u8(v.val[1]), 4)), c); + r.q[1] = vsubq_s16(vreinterpretq_s16_u16(vshll_n_u8(vget_high_u8(v.val[1]), 4)), c); + return r; +} + +static inline __attribute__((always_inline)) iq16x16_neon_t iqconv_load_sc16_neon(const sc16_t *in) +{ + const int16x8x2_t a = vld2q_s16((const int16_t *) in); + const int16x8x2_t b = vld2q_s16((const int16_t *) (in + 8)); + iq16x16_neon_t r; + r.i[0] = vshrq_n_s16(a.val[0], 4); + r.i[1] = vshrq_n_s16(b.val[0], 4); + r.q[0] = vshrq_n_s16(a.val[1], 4); + r.q[1] = vshrq_n_s16(b.val[1], 4); + return r; +} + +static inline __attribute__((always_inline)) int16x8_t clamp12_s16x8_neon(const int16x8_t v) +{ + return vmaxq_s16(vminq_s16(v, vdupq_n_s16(2047)), vdupq_n_s16(-2048)); +} + +static inline __attribute__((always_inline)) iq16x16_neon_t iqconv_load_sc16q11_neon(const sc16_t *in) +{ + const int16x8x2_t a = vld2q_s16((const int16_t *) in); + const int16x8x2_t b = vld2q_s16((const int16_t *) (in + 8)); + iq16x16_neon_t r; + r.i[0] = clamp12_s16x8_neon(a.val[0]); + r.i[1] = clamp12_s16x8_neon(b.val[0]); + r.q[0] = clamp12_s16x8_neon(a.val[1]); + r.q[1] = clamp12_s16x8_neon(b.val[1]); + return r; +} + +// Same as iqconv_dc_block() for a full block. off = [I, Q] offsets; the block sum +// of (raw - corr) is sum(raw) - 16 * corr, so the serial part is just a few ops. +static inline __attribute__((always_inline)) +void iqconv_block_neon(const iq16x16_neon_t r, int16_t * restrict out, int32x2_t *off, + uint64x2_t *acc_level, uint64x2_t *acc_power) +{ + const int32x2_t corr = vrshr_n_s32(*off, IQ_DC_SHIFT); // (off + 2048) >> 12 + const int32x2_t sums = hsum2_s32x4_neon(vpaddlq_s16(vaddq_s16(r.i[0], r.i[1])), + vpaddlq_s16(vaddq_s16(r.q[0], r.q[1]))); + *off = vadd_s32(*off, vsub_s32(sums, vshl_n_s32(corr, 4))); + + const int16x4_t c16 = vmovn_s32(vcombine_s32(corr, corr)); // [ci, cq, ci, cq] + const int16x8_t ci = vdupq_lane_s16(c16, 0); + const int16x8_t cq = vdupq_lane_s16(c16, 1); + + uint32x4_t level = vdupq_n_u32(0), power = vdupq_n_u32(0); + for (int h = 0; h < 2; ++h) { + int16x8x2_t x; + x.val[0] = clamp12_s16x8_neon(vsubq_s16(r.i[h], ci)); + x.val[1] = clamp12_s16x8_neon(vsubq_s16(r.q[h], cq)); + vst2q_s16(out + 16 * h, x); + + const uint32x4_t m0 = vreinterpretq_u32_s32( + vmlal_s16(vmull_s16(vget_low_s16(x.val[0]), vget_low_s16(x.val[0])), vget_low_s16(x.val[1]), vget_low_s16(x.val[1]))); + const uint32x4_t m1 = vreinterpretq_u32_s32( + vmlal_s16(vmull_s16(vget_high_s16(x.val[0]), vget_high_s16(x.val[0])), vget_high_s16(x.val[1]), vget_high_s16(x.val[1]))); + power = vaddq_u32(power, vaddq_u32(m0, m1)); + level = vaddq_u32(level, vaddq_u32(isqrt_u32x4_neon(vshlq_n_u32(m0, IQ_LEVEL_SHIFT)), + isqrt_u32x4_neon(vshlq_n_u32(m1, IQ_LEVEL_SHIFT)))); + } + + *acc_level = vpadalq_u32(*acc_level, level); + *acc_power = vpadalq_u32(*acc_power, power); +} + +#define IQDATA_CONV_NEON_BODY(fmt) \ + int16_t * restrict out = (int16_t *) out_u; \ + int32x2_t off = vld1_s32(dc_in); \ + uint64x2_t acc_level = vdupq_n_u64(0), acc_power = vdupq_n_u64(0); \ + unsigned n = 0; \ + \ + for (; n + IQ_DC_BLOCK <= len; n += IQ_DC_BLOCK) \ + iqconv_block_neon(iqconv_load_##fmt##_neon(in + n), out + 2 * n, &off, &acc_level, &acc_power); \ + \ + vst1_s32(dc_out, off); \ + uint64_t sum_level = vgetq_lane_u64(acc_level, 0) + vgetq_lane_u64(acc_level, 1); \ + uint64_t sum_power = vgetq_lane_u64(acc_power, 0) + vgetq_lane_u64(acc_power, 1); \ + \ + if (n < len) { \ + int16_t raw[2 * IQ_DC_BLOCK]; \ + iqconv_raw_##fmt(in + n, raw, len - n); \ + iqconv_dc_block(raw, out + 2 * n, len - n, dc_out, &sum_level, &sum_power); \ + } \ + \ + iqconv_stats(sum_level, sum_power, len, out_level, out_power); + +void STARCH_IMPL(iqdata_conv_uc8, neon) (const uc8_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + IQDATA_CONV_NEON_BODY(uc8) +} + +void STARCH_IMPL(iqdata_conv_sc16, neon) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + IQDATA_CONV_NEON_BODY(sc16) +} + +void STARCH_IMPL(iqdata_conv_sc16q11, neon) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, + const int32_t *dc_in, int32_t *dc_out, double *out_level, double *out_power) +{ + IQDATA_CONV_NEON_BODY(sc16q11) +} diff --git a/dsp/impl/magnitude_power_uc8.generic.c b/dsp/impl/magnitude_power_uc8.generic.c index 76908bd1f..a6f4c66df 100644 --- a/dsp/impl/magnitude_power_uc8.generic.c +++ b/dsp/impl/magnitude_power_uc8.generic.c @@ -83,38 +83,3 @@ void STARCH_IMPL(magnitude_power_uc8, lookup_unroll_4) (const uc8_t * restrict i *out_level = sum_level / 65536.0 / len; *out_power = sum_power / 65536.0 / 65536.0 / len; } - -/* UC8 (8-bit offset binary, centre 127.5) -> 12-bit signed interleaved I,Q, - * -2040..+2040, plus mean level and power normalised to 12-bit full scale */ - -void STARCH_IMPL(iqdata_conv_uc8, arith) (const uc8_t * restrict in, uint16_t * restrict out, unsigned len, double *out_level, double *out_power) -{ - // 8-bit offset binary (centre 127.5) -> 12-bit signed, -2040..+2040, interleaved I,Q - for (unsigned n = 0; n < len; ++n) { - out[2 * n] = (int16_t) ((2 * in[n].I - 255) * 8); - out[2 * n + 1] = (int16_t) ((2 * in[n].Q - 255) * 8); - } - - // Same normalisation as the stock stats: magnitude scaled so full scale (2048) = 1.0 - double sum_level = 0; - uint64_t sum_power = 0; - - for (unsigned n = 0; n < len; ++n) { - int32_t I = (2 * in[n].I - 255) * 8; - int32_t Q = (2 * in[n].Q - 255) * 8; - - out[2 * n] = (int16_t) I; - out[2 * n + 1] = (int16_t) Q; - - uint32_t magsq = (uint32_t) (I * I + Q * Q); - sum_power += magsq; - sum_level += sqrtf((float) magsq); - } - - if (out_level != NULL) - *out_level = sum_level / 2048.0 / len; - - if (out_power != NULL) - *out_power = (double) sum_power / 2048.0 / 2048.0 / len; -} - diff --git a/dsp/impl/magnitude_sc16.generic.c b/dsp/impl/magnitude_sc16.generic.c index d9314868c..cbf14cdd5 100644 --- a/dsp/impl/magnitude_sc16.generic.c +++ b/dsp/impl/magnitude_sc16.generic.c @@ -37,32 +37,3 @@ void STARCH_IMPL(magnitude_sc16, exact_float) (const sc16_t * restrict in, uint1 in += 1; } } - -/* SC16 (full-scale 16-bit, little-endian) -> 12-bit signed interleaved I,Q, - * plus mean level and power normalised to 12-bit full scale */ - -void STARCH_IMPL(iqdata_conv_sc16, arith) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, double *out_level, double *out_power) -{ - int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer - - double sum_level = 0; - uint64_t sum_power = 0; - - for (unsigned n = 0; n < len; ++n) { - int32_t I = (int16_t) le16toh(in[n].I) >> 4; - int32_t Q = (int16_t) le16toh(in[n].Q) >> 4; - - out[2 * n] = (int16_t) I; - out[2 * n + 1] = (int16_t) Q; - - uint32_t magsq = (uint32_t) (I * I + Q * Q); - sum_power += magsq; - sum_level += sqrtf((float) magsq); - } - - if (out_level != NULL) - *out_level = sum_level / 2048.0 / len; - - if (out_power != NULL) - *out_power = (double) sum_power / 2048.0 / 2048.0 / len; -} diff --git a/dsp/impl/magnitude_sc16q11.generic.c b/dsp/impl/magnitude_sc16q11.generic.c index 924b5c6e5..ca99bd950 100644 --- a/dsp/impl/magnitude_sc16q11.generic.c +++ b/dsp/impl/magnitude_sc16q11.generic.c @@ -70,38 +70,3 @@ void STARCH_IMPL(magnitude_sc16q11, 12bit_table) (const sc16_t * restrict in, ui out += 1; } } - - -/* SC16Q11 (already 12-bit, little-endian) -> 12-bit signed interleaved I,Q, - * plus mean level and power normalised to 12-bit full scale */ - -void STARCH_IMPL(iqdata_conv_sc16q11, arith) (const sc16_t * restrict in, uint16_t * restrict out_u, unsigned len, double *out_level, double *out_power) -{ - int16_t * restrict out = (int16_t *) out_u; // signed I/Q stored in the uint16_t FIFO buffer - - double sum_level = 0; - uint64_t sum_power = 0; - - for (unsigned n = 0; n < len; ++n) { - int32_t I = (int16_t) le16toh(in[n].I); - int32_t Q = (int16_t) le16toh(in[n].Q); - - // keep to 12-bit like the stock 11-bit table path - if (I > 2047) I = 2047; else if (I < -2048) I = -2048; - if (Q > 2047) Q = 2047; else if (Q < -2048) Q = -2048; - - out[2 * n] = (int16_t) I; - out[2 * n + 1] = (int16_t) Q; - - uint32_t magsq = (uint32_t) (I * I + Q * Q); - sum_power += magsq; - sum_level += sqrtf((float) magsq); - } - - if (out_level != NULL) - *out_level = sum_level / 2048.0 / len; - - if (out_power != NULL) - *out_power = (double) sum_power / 2048.0 / 2048.0 / len; -} - diff --git a/dsp/starchgen.py b/dsp/starchgen.py index f619d90da..a78ff25c1 100755 --- a/dsp/starchgen.py +++ b/dsp/starchgen.py @@ -35,13 +35,13 @@ argtypes = ['const uint16_t *', 'unsigned', 'uint16_t', 'unsigned *']) gen.add_function( name = 'iqdata_conv_uc8', - argtypes = ['const uc8_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) + argtypes = ['const uc8_t *', 'uint16_t *', 'unsigned', 'const int32_t *', 'int32_t *', 'double *', 'double *']) gen.add_function( name = 'iqdata_conv_sc16', - argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) + argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'const int32_t *', 'int32_t *', 'double *', 'double *']) gen.add_function( name = 'iqdata_conv_sc16q11', - argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'double *', 'double *']) + argtypes = ['const sc16_t *', 'uint16_t *', 'unsigned', 'const int32_t *', 'int32_t *', 'double *', 'double *']) gen.add_function( name = 'upsample12', argtypes = ['const int16_t *', 'int', 'int', 'uint16_t *'], diff --git a/fifo.h b/fifo.h index 05b79ea6d..414ab8e0e 100644 --- a/fifo.h +++ b/fifo.h @@ -68,6 +68,19 @@ struct mag_buf { struct mag_buf *next; // linked list forward link }; +// Values stored per sample in mag_buf.data: one magnitude, or interleaved I,Q in the DEMOD_2400X build +#ifdef DEMOD_2400X +#define MAGBUF_VALUES_PER_SAMPLE 2 +#else +#define MAGBUF_VALUES_PER_SAMPLE 1 +#endif + +// Address of sample n in a buffer (n counts samples, not uint16_t values) +static inline uint16_t *magbuf_sample(struct mag_buf *buf, unsigned n) +{ + return &buf->data[n * MAGBUF_VALUES_PER_SAMPLE]; +} + // Create the queue structures. Not threadsafe. Returns true on success. // // buffer_count - the number of buffers to preallocate diff --git a/sdr_bladerf.c b/sdr_bladerf.c index 5f7c726e3..1e37599bf 100644 --- a/sdr_bladerf.c +++ b/sdr_bladerf.c @@ -396,7 +396,7 @@ static void *handle_bladerf_samples(struct bladerf *dev, // Convert one block of sample data double mean_level, mean_power; - BladeRF.converter(sample_data, &outbuf->data[outbuf->validLength], samples_per_block, BladeRF.converter_state, &mean_level, &mean_power); + BladeRF.converter(sample_data, magbuf_sample(outbuf, outbuf->validLength), samples_per_block, BladeRF.converter_state, &mean_level, &mean_power); outbuf->validLength += samples_per_block; outbuf->mean_level += mean_level; outbuf->mean_power += mean_power; diff --git a/sdr_hackrf.c b/sdr_hackrf.c index a21ef5b03..51b6a19a2 100644 --- a/sdr_hackrf.c +++ b/sdr_hackrf.c @@ -264,7 +264,7 @@ static int handle_hackrf_samples(hackrf_transfer *transfer) dropped = samples_read - to_convert; } - HackRF.converter(buf, &outbuf->data[outbuf->overlap], to_convert, HackRF.converter_state, &outbuf->mean_level, &outbuf->mean_power); + HackRF.converter(buf, magbuf_sample(outbuf, outbuf->overlap), to_convert, HackRF.converter_state, &outbuf->mean_level, &outbuf->mean_power); outbuf->validLength = outbuf->overlap + to_convert; // Push to the demodulation thread diff --git a/sdr_ifile.c b/sdr_ifile.c index fb49a42bd..70e780b0f 100644 --- a/sdr_ifile.c +++ b/sdr_ifile.c @@ -221,7 +221,7 @@ void ifileRun() unsigned samples_read = bytes_read / ifile.bytes_per_sample; // Convert the new data - ifile.converter(ifile.readbuf, &outbuf->data[outbuf->overlap], samples_read, ifile.converter_state, &outbuf->mean_level, &outbuf->mean_power); + ifile.converter(ifile.readbuf, magbuf_sample(outbuf, outbuf->overlap), samples_read, ifile.converter_state, &outbuf->mean_level, &outbuf->mean_power); outbuf->validLength = outbuf->overlap + samples_read; outbuf->flags = 0; diff --git a/sdr_limesdr.c b/sdr_limesdr.c index e010e7378..92f69cea5 100644 --- a/sdr_limesdr.c +++ b/sdr_limesdr.c @@ -379,7 +379,7 @@ static void limesdrCallback(unsigned char *buf, uint32_t len, void *ctx) dropped = samples_read - to_convert; } - LimeSDR.converter(buf, &outbuf->data[outbuf->overlap], to_convert, LimeSDR.converter_state, &outbuf->mean_level, &outbuf->mean_power); + LimeSDR.converter(buf, magbuf_sample(outbuf, outbuf->overlap), to_convert, LimeSDR.converter_state, &outbuf->mean_level, &outbuf->mean_power); outbuf->validLength = outbuf->overlap + to_convert; // Push to the demodulation thread diff --git a/sdr_rtlsdr.c b/sdr_rtlsdr.c index 61e276013..614b1b32f 100644 --- a/sdr_rtlsdr.c +++ b/sdr_rtlsdr.c @@ -362,7 +362,7 @@ static void rtlsdrCallback(unsigned char *buf, uint32_t len, void *ctx) buf = RTLSDR.bounce_buffer; #endif - RTLSDR.converter(buf, &outbuf->data[outbuf->overlap], to_convert, RTLSDR.converter_state, &outbuf->mean_level, &outbuf->mean_power); + RTLSDR.converter(buf, magbuf_sample(outbuf, outbuf->overlap), to_convert, RTLSDR.converter_state, &outbuf->mean_level, &outbuf->mean_power); outbuf->validLength = outbuf->overlap + to_convert; // Push to the demodulation thread diff --git a/sdr_soapy.c b/sdr_soapy.c index 396ece74e..5e0b018f8 100644 --- a/sdr_soapy.c +++ b/sdr_soapy.c @@ -516,7 +516,7 @@ void soapyRun() } // Convert the new data - SOAPY.converter(buf, &outbuf->data[outbuf->overlap], to_convert, SOAPY.converter_state, &outbuf->mean_level, &outbuf->mean_power); + SOAPY.converter(buf, magbuf_sample(outbuf, outbuf->overlap), to_convert, SOAPY.converter_state, &outbuf->mean_level, &outbuf->mean_power); outbuf->validLength = outbuf->overlap + to_convert; // Push to the demodulation thread From 0f244bc91631ad6ba3405a69cbc31441f2fbddfe Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 05:18:50 +0100 Subject: [PATCH 09/25] more speedups --- demod_2400x.c | 105 ++++++++++++++++++++----------------- dsp/generated/dispatcher.c | 36 ++++++------- wisdom.aarch64 | 13 +++++ wisdom.arm | 13 +++++ wisdom.x86 | 13 +++++ 5 files changed, 115 insertions(+), 65 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index d25013cd3..fe471e1ff 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -55,6 +55,7 @@ struct staticdata_t { struct candidate { int pos; // metric index = 12 MHz sample index from mag->data[0] unsigned quality; // data quality of the first 56 bits (0..256) + unsigned quality2; // data quality of the second 56 bits (0..256) uint8_t msg[MODES_LONG_MSG_BYTES]; // demodulated frame }; @@ -142,23 +143,12 @@ static void checkMetric(const uint8_t *metric, int mlen) { #endif -// Decode 1 bit -#define DECODE_BIT(pp, ofs, bt, qq, pw) ({ \ - int aa = (pp)[2*(ofs)+0]; \ - int bb = (pp)[2*(ofs)+1]; \ - int dd = bb - aa; \ - if (dd < 0) { \ - dd = -dd; \ - bt |= (1<<(7-ofs)); \ - } \ - qq += dd; \ - pw += aa+bb; \ -}) - - -// Decode 56 bits starting at metric position 'pos'; returns a quality score (0..256) -static unsigned decodeShortAdbsData(int pos, const uint16_t * restrict udata, int nt, uint8_t * restrict msg) { - +// Decode 56 bits starting at metric position 'pos'; returns a quality score (0..256). +// Each bit is two chips: bit = 1 if the first is larger. Branch-free: on noise the +// bit values are random, and a branch on them mispredicts half the time. +static inline __attribute__((always_inline)) +unsigned decodeShortAdbsData(int pos, const uint16_t * restrict udata, int nt, uint8_t * restrict msg) +{ int stride = NU_STRIDE(nt); int phase = pos % 6; int offs = (pos / 6) + 16; // Start of data @@ -168,18 +158,17 @@ static unsigned decodeShortAdbsData(int pos, const uint16_t * restrict udata, in const uint16_t * restrict p = &udata[phase * stride + offs]; for (int i = 0; i < SHORT_FRAME / 8; i++) { // 7 bytes = 56 bits, 16 chips per byte - unsigned bit = 0; - - DECODE_BIT(p, 0, bit, qual, pow); - DECODE_BIT(p, 1, bit, qual, pow); - DECODE_BIT(p, 2, bit, qual, pow); - DECODE_BIT(p, 3, bit, qual, pow); - DECODE_BIT(p, 4, bit, qual, pow); - DECODE_BIT(p, 5, bit, qual, pow); - DECODE_BIT(p, 6, bit, qual, pow); - DECODE_BIT(p, 7, bit, qual, pow); + unsigned bits = 0; + for (int k = 0; k < 8; k++) { + int a = p[2 * k]; + int b = p[2 * k + 1]; + int d = b - a; + bits = (bits << 1) | ((unsigned) d >> 31); // 1 if a > b + qual += (uint32_t) (d < 0 ? -d : d); + pow += (uint32_t) (a + b); + } - msg[i] = bit; + msg[i] = (uint8_t) bits; p += 16; } unsigned score = (256 * qual) / (pow + 1); @@ -188,7 +177,7 @@ static unsigned decodeShortAdbsData(int pos, const uint16_t * restrict udata, in // Is metric position 'p' a preamble peak? -static inline int isPeak(const uint8_t *metric, int p) +static inline __attribute__((always_inline)) int isPeak(const uint8_t *metric, int p) { unsigned m0 = metric[p - 1], m1 = metric[p], m2 = metric[p + 1]; @@ -196,6 +185,20 @@ static inline int isPeak(const uint8_t *metric, int p) } +// Does any of the 8 metric bytes at 'm' reach PEAMBLE_THRES? (exact: bytes below +// 128 get 128 - THRES added without carrying into the next byte; bytes from 128 up +// have their top bit set already) +_Static_assert(PEAMBLE_THRES >= 1 && PEAMBLE_THRES <= 128, "anyAboveThres8() needs 1 <= PEAMBLE_THRES <= 128"); +static inline __attribute__((always_inline)) int anyAboveThres8(const uint8_t *m) +{ + const uint64_t ones = 0x0101010101010101ULL; + uint64_t x; + memcpy(&x, m, sizeof(x)); + uint64_t y = (x & (0x7f * ones)) + (128 - PEAMBLE_THRES) * ones; + return ((x | y) & (0x80 * ones)) != 0; +} + + // Try the peak and its two neighbours, keep the one whose first 56 bits look best static void bestPhase(int peak, const uint16_t *udata, int nt, struct candidate *c) { @@ -212,17 +215,12 @@ static void bestPhase(int peak, const uint16_t *udata, int nt, struct candidate } -// Demodulate the rest of the frame if the DF says it is a long one. -// Returns the frame length in bits, or 0 if the second half doesn't hold up. -static int demodFrame(struct candidate *c, const uint16_t *udata, int nt) +// Demodulate the second 56 bits whatever the DF says: scoreModesMessage() can +// correct the DF field, including the bit that selects 56 or 112 bits, so the +// frame length is only known after scoring +static void demodFrame(struct candidate *c, const uint16_t *udata, int nt) { - int bits = modesMessageLenByType(c->msg[0] >> 3); - - if (bits == MODES_LONG_MSG_BITS && - decodeShortAdbsData(c->pos + 12 * SHORT_FRAME, udata, nt, c->msg + MODES_SHORT_MSG_BYTES) < DATA2_THRES) - return 0; - - return bits; + c->quality2 = decodeShortAdbsData(c->pos + 12 * SHORT_FRAME, udata, nt, c->msg + MODES_SHORT_MSG_BYTES); } @@ -248,8 +246,8 @@ static double signalPower(const int16_t *iq, int pos, int bits, double *energy) // Score, decode and pass one frame on to the rest of dump1090. -// Returns 1 if it was accepted; *energy gets its signal energy. -static int emitFrame(const struct mag_buf *mag, const struct candidate *c, int bits, +// Returns its length in bits if it was accepted, else 0; *energy gets its signal energy. +static int emitFrame(const struct mag_buf *mag, const struct candidate *c, const int16_t *iq, double *energy) { static const struct modesMessage zeroMessage; @@ -275,16 +273,24 @@ static int emitFrame(const struct mag_buf *mag, const struct candidate *c, int b Modes.stats_current.demod_rejected_bad++; return 0; } + + // A long frame without a CRC to check (DF16/20/21/24: address/parity only) + // also needs a clean second half + if (mm.msgbits == MODES_LONG_MSG_BITS && mm.msgtype != 17 && mm.msgtype != 18 && + c->quality2 < DATA2_THRES) { + Modes.stats_current.demod_rejected_bad++; + return 0; + } Modes.stats_current.demod_accepted[mm.correctedbits]++; - mm.signalLevel = signalPower(iq, c->pos, bits, energy); + mm.signalLevel = signalPower(iq, c->pos, mm.msgbits, energy); if (mm.signalLevel > Modes.stats_current.peak_signal_power) Modes.stats_current.peak_signal_power = mm.signalLevel; if (mm.signalLevel > 0.50119) Modes.stats_current.strong_signal_count++; // above -3 dBFS useModesMessage(&mm); - return 1; + return mm.msgbits; } @@ -301,6 +307,12 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i for (p = MAX(1, st->next_start); p <= last; p++) { + // Most of the metric is noise below the threshold: skip it 8 at a time + while (p + 7 <= last && !anyAboveThres8(st->metric + p)) + p += 8; + if (p > last) + break; + if (!isPeak(st->metric, p)) continue; Modes.stats_current.demod_preambles++; @@ -310,12 +322,11 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i if (c.quality < DATA_THRES) continue; - int bits = demodFrame(&c, st->udata, nt); - if (!bits) - continue; + demodFrame(&c, st->udata, nt); double energy; - if (emitFrame(mag, &c, bits, (const int16_t *) mag->data, &energy)) { + int bits = emitFrame(mag, &c, (const int16_t *) mag->data, &energy); + if (bits) { energy_sum += energy; // Skip to 8 bits before the end of the frame, as demod_2400.c does p = c.pos + bits * 12 - 1; diff --git a/dsp/generated/dispatcher.c b/dsp/generated/dispatcher.c index 536a9246e..1a4a65a2f 100644 --- a/dsp/generated/dispatcher.c +++ b/dsp/generated/dispatcher.c @@ -151,14 +151,14 @@ void starch_iqdata_conv_sc16_set_wisdom (const char * const * received_wisdom) starch_iqdata_conv_sc16_regentry starch_iqdata_conv_sc16_registry[] = { #ifdef STARCH_MIX_AARCH64 - { 0, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_generic_armv8_neon_simd, cpu_supports_armv8_simd }, - { 1, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 0, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16_generic_armv8_neon_simd, cpu_supports_armv8_simd }, { 2, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_AARCH64 */ #ifdef STARCH_MIX_ARM - { 0, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, - { 1, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 0, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, { 2, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_ARM */ @@ -167,8 +167,8 @@ starch_iqdata_conv_sc16_regentry starch_iqdata_conv_sc16_registry[] = { #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_generic_x86_avx2, cpu_supports_avx2 }, - { 1, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_avx2_x86_avx2, cpu_supports_avx2 }, + { 0, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_avx2_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16_generic_x86_avx2, cpu_supports_avx2 }, { 2, "generic_generic", "generic", starch_iqdata_conv_sc16_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } @@ -229,14 +229,14 @@ void starch_iqdata_conv_sc16q11_set_wisdom (const char * const * received_wisdom starch_iqdata_conv_sc16q11_regentry starch_iqdata_conv_sc16q11_registry[] = { #ifdef STARCH_MIX_AARCH64 - { 0, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_generic_armv8_neon_simd, cpu_supports_armv8_simd }, - { 1, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 0, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_sc16q11_generic_armv8_neon_simd, cpu_supports_armv8_simd }, { 2, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_AARCH64 */ #ifdef STARCH_MIX_ARM - { 0, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, - { 1, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 0, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_sc16q11_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, { 2, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_ARM */ @@ -245,8 +245,8 @@ starch_iqdata_conv_sc16q11_regentry starch_iqdata_conv_sc16q11_registry[] = { #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_generic_x86_avx2, cpu_supports_avx2 }, - { 1, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_avx2_x86_avx2, cpu_supports_avx2 }, + { 0, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_avx2_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_sc16q11_generic_x86_avx2, cpu_supports_avx2 }, { 2, "generic_generic", "generic", starch_iqdata_conv_sc16q11_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } @@ -307,14 +307,14 @@ void starch_iqdata_conv_uc8_set_wisdom (const char * const * received_wisdom) starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[] = { #ifdef STARCH_MIX_AARCH64 - { 0, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_generic_armv8_neon_simd, cpu_supports_armv8_simd }, - { 1, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 0, "neon_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "generic_armv8_neon_simd", "armv8_neon_simd", starch_iqdata_conv_uc8_generic_armv8_neon_simd, cpu_supports_armv8_simd }, { 2, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_AARCH64 */ #ifdef STARCH_MIX_ARM - { 0, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, - { 1, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 0, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_iqdata_conv_uc8_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, { 2, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_ARM */ @@ -323,8 +323,8 @@ starch_iqdata_conv_uc8_regentry starch_iqdata_conv_uc8_registry[] = { #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_generic_x86_avx2, cpu_supports_avx2 }, - { 1, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_avx2_x86_avx2, cpu_supports_avx2 }, + { 0, "avx2_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_avx2_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_x86_avx2", "x86_avx2", starch_iqdata_conv_uc8_generic_x86_avx2, cpu_supports_avx2 }, { 2, "generic_generic", "generic", starch_iqdata_conv_uc8_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } diff --git a/wisdom.aarch64 b/wisdom.aarch64 index 3077a1376..0641ff098 100644 --- a/wisdom.aarch64 +++ b/wisdom.aarch64 @@ -29,3 +29,16 @@ upsample12 generic_generic locate_preamble neon_armv8_neon_simd locate_preamble generic_armv8_neon_simd locate_preamble generic_generic + +# DEMOD_2400X input conversion, measured with starch-benchmark (Pi 5, Cortex-A76) +iqdata_conv_uc8 neon_armv8_neon_simd # 217643 ns/call +iqdata_conv_uc8 generic_armv8_neon_simd # 423520 ns/call +iqdata_conv_uc8 generic_generic # 771133 ns/call + +iqdata_conv_sc16 neon_armv8_neon_simd # 239234 ns/call +iqdata_conv_sc16 generic_armv8_neon_simd # 392460 ns/call +iqdata_conv_sc16 generic_generic # 761970 ns/call + +iqdata_conv_sc16q11 neon_armv8_neon_simd # 227178 ns/call +iqdata_conv_sc16q11 generic_armv8_neon_simd # 423654 ns/call +iqdata_conv_sc16q11 generic_generic # 772901 ns/call diff --git a/wisdom.arm b/wisdom.arm index 8ffe8615d..276be313f 100644 --- a/wisdom.arm +++ b/wisdom.arm @@ -33,3 +33,16 @@ upsample12 generic_generic locate_preamble neon_armv7a_neon_vfpv4 locate_preamble generic_armv7a_neon_vfpv4 locate_preamble generic_generic + +# DEMOD_2400X input conversion, measured with starch-benchmark (32-bit build on a Cortex-A76) +iqdata_conv_uc8 neon_armv7a_neon_vfpv4 # 330265 ns/call +iqdata_conv_uc8 generic_armv7a_neon_vfpv4 # 761068 ns/call +iqdata_conv_uc8 generic_generic # 937124 ns/call + +iqdata_conv_sc16 neon_armv7a_neon_vfpv4 # 345799 ns/call +iqdata_conv_sc16 generic_armv7a_neon_vfpv4 # 760711 ns/call +iqdata_conv_sc16 generic_generic # 891093 ns/call + +iqdata_conv_sc16q11 neon_armv7a_neon_vfpv4 # 336462 ns/call +iqdata_conv_sc16q11 generic_armv7a_neon_vfpv4 # 760099 ns/call +iqdata_conv_sc16q11 generic_generic # 1022069 ns/call diff --git a/wisdom.x86 b/wisdom.x86 index 4d3d41e4c..5f2a451d8 100644 --- a/wisdom.x86 +++ b/wisdom.x86 @@ -32,3 +32,16 @@ upsample12 generic_x86_avx2 locate_preamble generic_x86_avx2 # 2140715 ns/call locate_preamble generic_generic # 2293071 ns/call locate_preamble avx2_x86_avx2 # 2808931 ns/call + +# DEMOD_2400X input conversion: provisional order (hand-written AVX2 first, not timed on x86 hardware) +iqdata_conv_uc8 avx2_x86_avx2 +iqdata_conv_uc8 generic_x86_avx2 +iqdata_conv_uc8 generic_generic + +iqdata_conv_sc16 avx2_x86_avx2 +iqdata_conv_sc16 generic_x86_avx2 +iqdata_conv_sc16 generic_generic + +iqdata_conv_sc16q11 avx2_x86_avx2 +iqdata_conv_sc16q11 generic_x86_avx2 +iqdata_conv_sc16q11 generic_generic From dbd34b43e9bf72679f251c4aa4f017a6d24c187b Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 12:02:00 +0100 Subject: [PATCH 10/25] locate_preamble: zero metrics below a threshold without dividing Pass the preamble threshold to locate_preamble(); a metric below it is stored as 0, tested exactly as n < thres * d, so the divide is skipped. The peak search gives the same results (a neighbour below the threshold is below the peak either way). NEON/AVX2 skip the divide when all 8 lanes are below (96% of vectors on noise). thres = 0 gives the old metric. aarch64 NEON 1.64 -> 1.22 ms, ARMv7 NEON 2.66 -> 1.34 ms per buffer; dump1090 demod ~1574 -> ~1495 ms on the test file, identical frames. Co-Authored-By: Claude Opus 5.5 --- demod_2400x.c | 2 +- dsp/generated/benchmark.c | 16 +++++------ dsp/generated/dispatcher.c | 4 +-- dsp/generated/starch.h | 16 +++++------ dsp/helpers/demod2400x_dsp.h | 4 +++ dsp/impl/demod2400x.avx2.c | 30 ++++++++++++++------ dsp/impl/demod2400x.benchmark.c | 8 ++++-- dsp/impl/demod2400x.generic.c | 7 +++-- dsp/impl/demod2400x.neon.c | 50 ++++++++++++++++++++++----------- dsp/starchgen.py | 2 +- 10 files changed, 88 insertions(+), 51 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index fe471e1ff..2a5251e9a 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -402,7 +402,7 @@ void demodulate2400x(struct mag_buf *mag) #endif // Locate possible preambles - int mlen = starch_locate_preamble(stdata.udata, ulen, stdata.metric); + int mlen = starch_locate_preamble(stdata.udata, ulen, PEAMBLE_THRES, stdata.metric); // metric < PEAMBLE_THRES is 0 #ifdef CHECK_METRIC checkMetric(stdata.metric, mlen); #endif diff --git a/dsp/generated/benchmark.c b/dsp/generated/benchmark.c index 95fb4a6d3..1b6709c85 100644 --- a/dsp/generated/benchmark.c +++ b/dsp/generated/benchmark.c @@ -525,12 +525,12 @@ static void starch_benchmark_run_iqdata_conv_uc8( const uc8_t * arg0, uint16_t * /* prototypes for benchmark helpers provided by user code */ void starch_locate_preamble_benchmark (void); -bool starch_locate_preamble_benchmark_verify ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +bool starch_locate_preamble_benchmark_verify ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); /* prototype the benchmarking function so that we can build with -Wmissing-declarations */ void starch_locate_preamble_benchmark(void); -static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentry * _entry, const uint16_t * arg0, int arg1, uint8_t * arg2 ) +static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentry * _entry, const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ) { fprintf(stderr, " %-40s ", _entry->name); @@ -557,10 +557,10 @@ static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentr /* initial warmup */ for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) - _entry->callable ( arg0, arg1, arg2 ); + _entry->callable ( arg0, arg1, arg2, arg3 ); /* verify correctness of the output */ - if (! starch_locate_preamble_benchmark_verify ( arg0, arg1, arg2 )) { + if (! starch_locate_preamble_benchmark_verify ( arg0, arg1, arg2, arg3 )) { fprintf(stderr, "skipped (verification failed)\n"); starch_benchmark_validation_failed = true; return; @@ -578,7 +578,7 @@ static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentr _loops *= 2; starch_benchmark_get_time(&_start); for (uint64_t _loop = 0; _loop < _loops; ++_loop) - _entry->callable ( arg0, arg1, arg2 ); + _entry->callable ( arg0, arg1, arg2, arg3 ); starch_benchmark_get_time(&_end); _elapsed = starch_benchmark_elapsed(&_start, &_end); } @@ -592,7 +592,7 @@ static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentr for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { starch_benchmark_get_time(&_start); for (uint64_t _loop = 0; _loop < _loops; ++_loop) - _entry->callable ( arg0, arg1, arg2 ); + _entry->callable ( arg0, arg1, arg2, arg3 ); starch_benchmark_get_time(&_end); uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); if (_elapsed_one < _elapsed_min) @@ -629,10 +629,10 @@ static void starch_benchmark_one_locate_preamble( starch_locate_preamble_regentr ++starch_benchmark_result_count; } -static void starch_benchmark_run_locate_preamble( const uint16_t * arg0, int arg1, uint8_t * arg2 ) +static void starch_benchmark_run_locate_preamble( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ) { for (starch_locate_preamble_regentry *_entry = starch_locate_preamble_registry; _entry->name; ++_entry) { - starch_benchmark_one_locate_preamble( _entry, arg0, arg1, arg2 ); + starch_benchmark_one_locate_preamble( _entry, arg0, arg1, arg2, arg3 ); } } diff --git a/dsp/generated/dispatcher.c b/dsp/generated/dispatcher.c index 1a4a65a2f..184d6c603 100644 --- a/dsp/generated/dispatcher.c +++ b/dsp/generated/dispatcher.c @@ -344,13 +344,13 @@ starch_locate_preamble_regentry * starch_locate_preamble_select() { return NULL; } -static int starch_locate_preamble_dispatch ( const uint16_t * arg0, int arg1, uint8_t * arg2 ) { +static int starch_locate_preamble_dispatch ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ) { starch_locate_preamble_regentry *entry = starch_locate_preamble_select(); if (!entry) abort(); starch_locate_preamble = entry->callable; - return starch_locate_preamble ( arg0, arg1, arg2 ); + return starch_locate_preamble ( arg0, arg1, arg2, arg3 ); } starch_locate_preamble_ptr starch_locate_preamble = starch_locate_preamble_dispatch; diff --git a/dsp/generated/starch.h b/dsp/generated/starch.h index bffbce0df..544a92b0e 100644 --- a/dsp/generated/starch.h +++ b/dsp/generated/starch.h @@ -198,7 +198,7 @@ extern starch_upsample12_regentry starch_upsample12_registry[]; starch_upsample12_regentry * starch_upsample12_select(); void starch_upsample12_set_wisdom( const char * const * received_wisdom ); -typedef int (* starch_locate_preamble_ptr) ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +typedef int (* starch_locate_preamble_ptr) ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); extern starch_locate_preamble_ptr starch_locate_preamble; typedef struct { @@ -222,7 +222,7 @@ void starch_magnitude_power_uc8_twopass_armv7a_neon_vfpv4 ( const uc8_t * arg0, void starch_magnitude_power_uc8_lookup_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_armv7a_neon_vfpv4 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_generic_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -246,7 +246,7 @@ void starch_magnitude_power_uc8_neon_vrsqrte_armv7a_neon_vfpv4 ( const uc8_t * a void starch_magnitude_sc16_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_neon_vrsqrte_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); int starch_upsample12_neon_armv7a_neon_vfpv4 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_neon_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_neon_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_neon_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -260,7 +260,7 @@ void starch_magnitude_power_uc8_twopass_armv8_neon_simd ( const uc8_t * arg0, ui void starch_magnitude_power_uc8_lookup_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_armv8_neon_simd ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_generic_armv8_neon_simd ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_generic_armv8_neon_simd ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_generic_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -284,7 +284,7 @@ void starch_magnitude_power_uc8_neon_vrsqrte_armv8_neon_simd ( const uc8_t * arg void starch_magnitude_sc16_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_neon_vrsqrte_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); int starch_upsample12_neon_armv8_neon_simd ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_neon_armv8_neon_simd ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_neon_armv8_neon_simd ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_neon_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_neon_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_neon_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -297,7 +297,7 @@ void starch_magnitude_power_uc8_twopass_generic ( const uc8_t * arg0, uint16_t * void starch_magnitude_power_uc8_lookup_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_generic ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_generic_generic ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_generic_generic ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_generic_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -324,7 +324,7 @@ void starch_magnitude_power_uc8_twopass_x86_avx2 ( const uc8_t * arg0, uint16_t void starch_magnitude_power_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); void starch_magnitude_power_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, double * arg3, double * arg4 ); int starch_upsample12_generic_x86_avx2 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_generic_x86_avx2 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_generic_x86_avx2 ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_generic_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -342,7 +342,7 @@ void starch_magnitude_uc8_lookup_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, void starch_magnitude_uc8_lookup_unroll_4_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_uc8_exact_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2 ); int starch_upsample12_avx2_x86_avx2 ( const int16_t * arg0, int arg1, int arg2, uint16_t * arg3 ); -int starch_locate_preamble_avx2_x86_avx2 ( const uint16_t * arg0, int arg1, uint8_t * arg2 ); +int starch_locate_preamble_avx2_x86_avx2 ( const uint16_t * arg0, int arg1, unsigned arg2, uint8_t * arg3 ); void starch_iqdata_conv_uc8_avx2_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_avx2_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_avx2_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); diff --git a/dsp/helpers/demod2400x_dsp.h b/dsp/helpers/demod2400x_dsp.h index 9b5bb14fe..49af40d42 100644 --- a/dsp/helpers/demod2400x_dsp.h +++ b/dsp/helpers/demod2400x_dsp.h @@ -28,6 +28,10 @@ #define METRIC_SIZE(t) ((t)-6*15-12*LONG_FRAME) #define PREAMBLE_KK 2 + +// locate_preamble() sets metrics below its 'thres' argument to 0. That is exact +// without dividing (floor(n / d) >= t <=> n >= t * d), and the peak search can't +// tell the difference as long as thres <= its own threshold. #define PREAMBLE_MIN_PULSE #endif diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index e8f2075f4..fcaab868b 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -182,9 +182,9 @@ __m256i div_u32x8_avx2(const __m256i n, const __m256i d) return _mm256_add_epi32(q, _mm256_cmpgt_epi32(_mm256_mullo_epi32(q, d), n)); } -// Scalar preamble metric at p[0] +// Scalar preamble metric at p[0], 0 if below thres static inline __attribute__((always_inline)) -uint8_t preamble_metric1(const uint16_t * restrict p) +uint8_t preamble_metric1(const uint16_t * restrict p, unsigned thres) { #ifdef PREAMBLE_MIN_PULSE uint32_t minp = MIN(MIN(p[0], p[2]), MIN(p[7], p[9])); @@ -195,7 +195,9 @@ uint8_t preamble_metric1(const uint16_t * restrict p) uint32_t gap = p[1] + p[3] + p[4] + p[5] + p[6] + p[8] + p[10] + p[11] + p[12] + p[13] + p[14] + p[15]; - return (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); + uint32_t n = 256 * PREAMBLE_KK * minp; + uint32_t d = PREAMBLE_KK * minp + gap + 1; + return n < thres * d ? 0 : n / d; } #define PREAMBLE_BLOCK 64 // Positions per phase per block, multiple of 8 @@ -217,7 +219,7 @@ __m256i sum4_u16x8_avx2(const uint16_t * restrict p) // Preamble metrics for 8*n8 consecutive positions of one phase static inline __attribute__((always_inline)) -void preamble_metric_block_avx2(const uint16_t * restrict p, int n8, uint8_t * restrict out) +void preamble_metric_block_avx2(const uint16_t * restrict p, int n8, unsigned thres, uint8_t * restrict out) { // Preamble detection // P G P G, G G G P, G P G G, G G G G @@ -225,6 +227,7 @@ void preamble_metric_block_avx2(const uint16_t * restrict p, int n8, uint8_t * r // s4[11] for these 8 positions is s4[3] for the next 8 const __m256i one = _mm256_set1_epi32(1); + const __m256i vthres = _mm256_set1_epi32((int) thres); __m256i s4a = sum4_u16x8_avx2(p + 3); for (int k = 0; k < 8 * n8; k += 8, p += 8) { @@ -247,8 +250,17 @@ void preamble_metric_block_avx2(const uint16_t * restrict p, int n8, uint8_t * r const __m256i minp = _mm256_add_epi32(_mm256_add_epi32(load_u16x8_u32_avx2(p + 0), load_u16x8_u32_avx2(p + 2)), _mm256_add_epi32(load_u16x8_u32_avx2(p + 7), load_u16x8_u32_avx2(p + 9))); #endif - const __m256i q = div_u32x8_avx2(_mm256_mullo_epi32(minp, _mm256_set1_epi32(256 * PREAMBLE_KK)), - _mm256_add_epi32(gap, _mm256_mullo_epi32(minp, _mm256_set1_epi32(PREAMBLE_KK)))); + const __m256i n = _mm256_mullo_epi32(minp, _mm256_set1_epi32(256 * PREAMBLE_KK)); + const __m256i d = _mm256_add_epi32(gap, _mm256_mullo_epi32(minp, _mm256_set1_epi32(PREAMBLE_KK))); + + // Most positions are noise below the threshold: skip the divide when all 8 are + // (values < 2^31, so the signed compare is fine) + const __m256i below = _mm256_cmpgt_epi32(_mm256_mullo_epi32(d, vthres), n); + if (_mm256_movemask_epi8(below) == -1) { + _mm_storel_epi64((__m128i *) (out + k), _mm_setzero_si128()); + continue; + } + const __m256i q = _mm256_andnot_si256(below, div_u32x8_avx2(n, d)); // Metric < 256 const __m128i q16 = _mm_packus_epi32(_mm256_castsi256_si128(q), _mm256_extracti128_si256(q, 1)); @@ -257,7 +269,7 @@ void preamble_metric_block_avx2(const uint16_t * restrict p, int n8, uint8_t * r } // Locate preamble -int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, unsigned thres, uint8_t * restrict mm) { int stride = NU_STRIDE(nt); // Stride int count[6]; // Number of positions to evaluate in each phase @@ -289,7 +301,7 @@ int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, ui uint8_t t[6][PREAMBLE_BLOCK]; for (int j = 0; j < 6; j++) - preamble_metric_block_avx2(&pp[j*stride + i], n8, t[j]); + preamble_metric_block_avx2(&pp[j*stride + i], n8, thres, t[j]); // mm[6*(i+k) + j] = t[j][k]: interleave phases in pairs, then 3-way as 16-bit uint8_t * restrict mp = &mm[6*i]; @@ -318,7 +330,7 @@ int STARCH_IMPL(locate_preamble, avx2) (const uint16_t * restrict pp, int nt, ui // Remainder for (int j = 0; j < 6; j++) for (int k = i; k < count[j]; k++) - mm[6*k + j] = preamble_metric1(&pp[j*stride + k]); + mm[6*k + j] = preamble_metric1(&pp[j*stride + k], thres); return METRIC_SIZE(nt); } diff --git a/dsp/impl/demod2400x.benchmark.c b/dsp/impl/demod2400x.benchmark.c index 1abf020ca..97f6cd6c0 100644 --- a/dsp/impl/demod2400x.benchmark.c +++ b/dsp/impl/demod2400x.benchmark.c @@ -104,8 +104,10 @@ void STARCH_BENCHMARK(locate_preamble) (void) if (iq && pp) { int nt = starch_upsample12_generic_generic(iq, len, DEMOD2400X_BENCH_SHIFT, pp); if ((mm = malloc((size_t) nt))) { + // thres 0 gives every metric; 50 zeroes most of them (noise) without dividing memset(mm, DEMOD2400X_BENCH_POISON, (size_t) nt); - STARCH_BENCHMARK_RUN( locate_preamble, pp, nt, mm ); + STARCH_BENCHMARK_RUN( locate_preamble, pp, nt, 0, mm ); + STARCH_BENCHMARK_RUN( locate_preamble, pp, nt, 50, mm ); } } @@ -114,14 +116,14 @@ void STARCH_BENCHMARK(locate_preamble) (void) free(mm); } -bool STARCH_BENCHMARK_VERIFY(locate_preamble) (const uint16_t *pp, int nt, uint8_t *mm) +bool STARCH_BENCHMARK_VERIFY(locate_preamble) (const uint16_t *pp, int nt, unsigned thres, uint8_t *mm) { uint8_t *ref = malloc((size_t) nt); if (!ref) return false; memset(ref, DEMOD2400X_BENCH_POISON, (size_t) nt); - starch_locate_preamble_generic_generic(pp, nt, ref); + starch_locate_preamble_generic_generic(pp, nt, thres, ref); bool okay = true; for (int k = 0; k < nt; ++k) { diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c index dd7a2328d..dc92bc312 100644 --- a/dsp/impl/demod2400x.generic.c +++ b/dsp/impl/demod2400x.generic.c @@ -116,7 +116,7 @@ int STARCH_IMPL(upsample12, generic) (const int16_t * restrict in, int len, int // Locate preamble -int STARCH_IMPL(locate_preamble, generic) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +int STARCH_IMPL(locate_preamble, generic) (const uint16_t * restrict pp, int nt, unsigned thres, uint8_t * restrict mm) { int stride = NU_STRIDE(nt); @@ -155,8 +155,9 @@ int STARCH_IMPL(locate_preamble, generic) (const uint16_t * restrict pp, int nt, p[14] + p[15]; - *mp = (256 * PREAMBLE_KK * minp) - / (PREAMBLE_KK * minp + gap + 1); + uint32_t n = 256 * PREAMBLE_KK * minp; + uint32_t d = PREAMBLE_KK * minp + gap + 1; + *mp = n < thres * d ? 0 : n / d; // below threshold: 0, no divide p++; mp += 6; diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 5d2ddc85d..76ec00fcd 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -246,9 +246,9 @@ uint32x4_t div_u32x4_neon(const uint32x4_t n, const uint32x4_t d) #endif } -// Scalar preamble metric at p[0] +// Scalar preamble metric at p[0], 0 if below thres static inline __attribute__((always_inline)) -uint8_t preamble_metric1(const uint16_t * restrict p) +uint8_t preamble_metric1(const uint16_t * restrict p, unsigned thres) { #ifdef PREAMBLE_MIN_PULSE uint32_t minp = MIN(MIN(p[0], p[2]), MIN(p[7], p[9])); @@ -259,7 +259,9 @@ uint8_t preamble_metric1(const uint16_t * restrict p) uint32_t gap = p[1] + p[3] + p[4] + p[5] + p[6] + p[8] + p[10] + p[11] + p[12] + p[13] + p[14] + p[15]; - return (256 * PREAMBLE_KK * minp) / (PREAMBLE_KK * minp + gap + 1); + uint32_t n = 256 * PREAMBLE_KK * minp; + uint32_t d = PREAMBLE_KK * minp + gap + 1; + return n < thres * d ? 0 : n / d; } #define PREAMBLE_BLOCK 64 // Positions per phase per block, multiple of 8 @@ -280,7 +282,7 @@ uint32x4x2_t sum4_u16x8_neon(const uint16_t * restrict p) // Preamble metrics for 8*n8 consecutive positions of one phase static inline __attribute__((always_inline)) -void preamble_metric_block_neon(const uint16_t * restrict p, int n8, uint8_t * restrict out) +void preamble_metric_block_neon(const uint16_t * restrict p, int n8, unsigned thres, uint8_t * restrict out) { // Preamble detection // P G P G, G G G P, G P G G, G G G G @@ -288,6 +290,7 @@ void preamble_metric_block_neon(const uint16_t * restrict p, int n8, uint8_t * r // s4[11] for these 8 positions is s4[3] for the next 8 const uint32x4_t one = vdupq_n_u32(1); + const uint32x4_t vthres = vdupq_n_u32(thres); uint32x4x2_t s4a = sum4_u16x8_neon(p + 3); for (int k = 0; k < 8 * n8; k += 8, p += 8) { @@ -304,14 +307,14 @@ void preamble_metric_block_neon(const uint16_t * restrict p, int n8, uint8_t * r gap_hi = vaddq_u32(vaddq_u32(gap_hi, one), vaddq_u32(s4a.val[1], s4b.val[1])); s4a = s4b; - // Metric = (256 * KK * minp) / (KK * minp + gap + 1) + // Metric = n / d = (256 * KK * minp) / (KK * minp + gap + 1), or 0 if below thres #ifdef PREAMBLE_MIN_PULSE const uint16x8_t m = vminq_u16(vminq_u16(vld1q_u16(p + 0), vld1q_u16(p + 2)), vminq_u16(vld1q_u16(p + 7), vld1q_u16(p + 9))); - const uint32x4_t q_lo = div_u32x4_neon(vmull_n_u16(vget_low_u16(m), 256 * PREAMBLE_KK), - vmlal_n_u16(gap_lo, vget_low_u16(m), PREAMBLE_KK)); - const uint32x4_t q_hi = div_u32x4_neon(vmull_n_u16(vget_high_u16(m), 256 * PREAMBLE_KK), - vmlal_n_u16(gap_hi, vget_high_u16(m), PREAMBLE_KK)); + const uint32x4_t n_lo = vmull_n_u16(vget_low_u16(m), 256 * PREAMBLE_KK); + const uint32x4_t n_hi = vmull_n_u16(vget_high_u16(m), 256 * PREAMBLE_KK); + const uint32x4_t d_lo = vmlal_n_u16(gap_lo, vget_low_u16(m), PREAMBLE_KK); + const uint32x4_t d_hi = vmlal_n_u16(gap_hi, vget_high_u16(m), PREAMBLE_KK); #endif #ifdef PREAMBLE_SUM_PULSE const uint16x8_t v0 = vld1q_u16(p + 0), v2 = vld1q_u16(p + 2); @@ -320,19 +323,34 @@ void preamble_metric_block_neon(const uint16_t * restrict p, int n8, uint8_t * r vaddl_u16(vget_low_u16(v7), vget_low_u16(v9))); const uint32x4_t m_hi = vaddq_u32(vaddl_u16(vget_high_u16(v0), vget_high_u16(v2)), vaddl_u16(vget_high_u16(v7), vget_high_u16(v9))); - const uint32x4_t q_lo = div_u32x4_neon(vmulq_n_u32(m_lo, 256 * PREAMBLE_KK), - vmlaq_n_u32(gap_lo, m_lo, PREAMBLE_KK)); - const uint32x4_t q_hi = div_u32x4_neon(vmulq_n_u32(m_hi, 256 * PREAMBLE_KK), - vmlaq_n_u32(gap_hi, m_hi, PREAMBLE_KK)); + const uint32x4_t n_lo = vmulq_n_u32(m_lo, 256 * PREAMBLE_KK); + const uint32x4_t n_hi = vmulq_n_u32(m_hi, 256 * PREAMBLE_KK); + const uint32x4_t d_lo = vmlaq_n_u32(gap_lo, m_lo, PREAMBLE_KK); + const uint32x4_t d_hi = vmlaq_n_u32(gap_hi, m_hi, PREAMBLE_KK); #endif + // Most positions are noise below the threshold: skip the divide when all 8 are + const uint32x4_t ok_lo = vcgeq_u32(n_lo, vmulq_u32(d_lo, vthres)); + const uint32x4_t ok_hi = vcgeq_u32(n_hi, vmulq_u32(d_hi, vthres)); + const uint32x4_t any = vorrq_u32(ok_lo, ok_hi); +#ifdef __aarch64__ + if (vmaxvq_u32(any) == 0) { +#else + if (vget_lane_u64(vreinterpret_u64_u32(vorr_u32(vget_low_u32(any), vget_high_u32(any))), 0) == 0) { +#endif + vst1_u8(out + k, vdup_n_u8(0)); + continue; + } + const uint32x4_t q_lo = vandq_u32(div_u32x4_neon(n_lo, d_lo), ok_lo); + const uint32x4_t q_hi = vandq_u32(div_u32x4_neon(n_hi, d_hi), ok_hi); + // Metric < 256 vst1_u8(out + k, vmovn_u16(vcombine_u16(vmovn_u32(q_lo), vmovn_u32(q_hi)))); } } // Locate preamble -int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, uint8_t * restrict mm) +int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, unsigned thres, uint8_t * restrict mm) { int stride = NU_STRIDE(nt); // Stride int count[6]; // Number of positions to evaluate in each phase @@ -352,7 +370,7 @@ int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, ui uint8_t t[6][PREAMBLE_BLOCK]; for (int j = 0; j < 6; j++) - preamble_metric_block_neon(&pp[j*stride + i], n8, t[j]); + preamble_metric_block_neon(&pp[j*stride + i], n8, thres, t[j]); // mm[6*(i+k) + j] = t[j][k]: interleave phases in pairs, then 3-way as 16-bit uint8_t * restrict mp = &mm[6*i]; @@ -372,7 +390,7 @@ int STARCH_IMPL(locate_preamble, neon) (const uint16_t * restrict pp, int nt, ui // Remainder for (int j = 0; j < 6; j++) for (int k = i; k < count[j]; k++) - mm[6*k + j] = preamble_metric1(&pp[j*stride + k]); + mm[6*k + j] = preamble_metric1(&pp[j*stride + k], thres); return METRIC_SIZE(nt); } diff --git a/dsp/starchgen.py b/dsp/starchgen.py index a78ff25c1..b38c77f6a 100755 --- a/dsp/starchgen.py +++ b/dsp/starchgen.py @@ -48,7 +48,7 @@ returntype = 'int') gen.add_function( name = 'locate_preamble', - argtypes = ['const uint16_t *', 'int', 'uint8_t *'], + argtypes = ['const uint16_t *', 'int', 'unsigned', 'uint8_t *'], returntype = 'int') gen.add_flavor( From 5b485331adb8e35fb7b7e2421f6762bb91571f40 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 13:32:46 +0100 Subject: [PATCH 11/25] AVX2: correct divide/sqrt in both directions The x86_avx2 flavour is built with -ffast-math, so the compiler may replace _mm256_div_ps / sqrt with reciprocal estimates. The truncated result can then be one too small, which the one-sided check missed (locate_preamble failed verification on real Intel hardware). Use two-sided checks in the AVX2 divide and sqrt and in the scalar demod_isqrt, as the ARMv7 code already does. Co-Authored-By: Claude Opus 5.5 --- dsp/helpers/demod2400x_coeffs.h | 9 ++++++--- dsp/impl/demod2400x.avx2.c | 20 +++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h index 272036ef2..9c9b9da56 100644 --- a/dsp/helpers/demod2400x_coeffs.h +++ b/dsp/helpers/demod2400x_coeffs.h @@ -36,13 +36,16 @@ static const int8_t upsample_coeffs[5][8] = { }; // Exact floor(sqrt(x)) for x < 2^31, for USE_MAGNITUDE. -// sqrtf is correctly rounded, so after adding 2^-6 the truncated value is the exact -// result or one more, fixed with an integer check. Exact, so generic, NEON and AVX2 +// After adding 2^-6 the truncated sqrtf is the exact result or within one of it +// (sqrtf may be an estimate under -ffast-math), fixed with integer checks. Exact, +// so generic, NEON and AVX2 // (which use the same steps in vector form) give bit-identical results. static inline __attribute__((always_inline)) uint32_t demod_isqrt(uint32_t x) { uint32_t q = (uint32_t) (__builtin_sqrtf((float) x) + 1.0f / 64); - return q - (q * q > x); + // Correct in both directions: under -ffast-math the sqrt may be an estimate + q -= (q * q > x); + return q + ((q + 1) * (q + 1) <= x); } // Square root used by getpow() in USE_MAGNITUDE mode. Override on the command line diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index fcaab868b..f49e1108e 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -72,13 +72,18 @@ static inline __attribute__((always_inline)) unsigned getpow_avx2(const __m128i static inline __attribute__((always_inline)) __m256i isqrt_u32x8_avx2(const __m256i x) { - // sqrt is correctly rounded, so after adding 2^-6 truncation is exact or one too large + // -ffast-math lets the compiler use a reciprocal sqrt estimate, so truncation + // is within one either way: correct in both directions const __m256 f = _mm256_add_ps(_mm256_sqrt_ps(_mm256_cvtepi32_ps(x)), _mm256_set1_ps(1.0f / 64)); - const __m256i q = _mm256_cvttps_epi32(f); + __m256i q = _mm256_cvttps_epi32(f); // q * q > x (unsigned, q * q may reach 2^31): q - 1 unless max(q * q, x) == x const __m256i qq = _mm256_mullo_epi32(q, q); const __m256i ok = _mm256_cmpeq_epi32(_mm256_max_epu32(qq, x), x); - return _mm256_sub_epi32(_mm256_sub_epi32(q, _mm256_set1_epi32(1)), ok); + q = _mm256_sub_epi32(_mm256_sub_epi32(q, _mm256_set1_epi32(1)), ok); + // (q + 1)^2 <= x (unsigned, < 2^32): q + 1 + const __m256i q1 = _mm256_add_epi32(q, _mm256_set1_epi32(1)); + const __m256i q1q1 = _mm256_mullo_epi32(q1, q1); + return _mm256_sub_epi32(q, _mm256_cmpeq_epi32(_mm256_max_epu32(q1q1, x), x)); } // Power (or magnitude) of 8 filter outputs, as getpow_avx2() @@ -177,9 +182,14 @@ static inline __attribute__((always_inline)) __m256i div_u32x8_avx2(const __m256i n, const __m256i d) { // Values are < 2^31, so the signed conversions and compares are fine - // Correctly rounded quotient: truncation is at most one too large + // -ffast-math lets the compiler use a reciprocal estimate for the divide, so + // truncation is within one either way: correct in both directions const __m256i q = _mm256_cvttps_epi32(_mm256_div_ps(_mm256_cvtepi32_ps(n), _mm256_cvtepi32_ps(d))); - return _mm256_add_epi32(q, _mm256_cmpgt_epi32(_mm256_mullo_epi32(q, d), n)); + const __m256i qd = _mm256_mullo_epi32(q, d); + // q + 1, -1 if q * d > n (too large), -1 if (q + 1) * d > n (not too small) + return _mm256_add_epi32(_mm256_add_epi32(q, _mm256_set1_epi32(1)), + _mm256_add_epi32(_mm256_cmpgt_epi32(qd, n), + _mm256_cmpgt_epi32(_mm256_add_epi32(qd, d), n))); } // Scalar preamble metric at p[0], 0 if below thres From cce3ecd945a1dc8c3ed553852b27c436cd965cf7 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 14:58:29 +0100 Subject: [PATCH 12/25] wisdom.x86: measured DEMOD_2400X kernel timings on Intel AVX2 is now fastest for locate_preamble too (it was last in the old, pre-vectorisation timings), so production x86 builds use it. Co-Authored-By: Claude Opus 5.5 --- wisdom.x86 | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/wisdom.x86 b/wisdom.x86 index 5f2a451d8..b2b70c101 100644 --- a/wisdom.x86 +++ b/wisdom.x86 @@ -25,23 +25,23 @@ count_above_u16 generic_generic # DEMOD_2400X kernels, measured with starch-benchmark -i 5 (x86-64 with AVX2) -upsample12 avx2_x86_avx2 # 1877779 ns/call -upsample12 generic_generic # 3290448 ns/call -upsample12 generic_x86_avx2 # 3709713 ns/call - -locate_preamble generic_x86_avx2 # 2140715 ns/call -locate_preamble generic_generic # 2293071 ns/call -locate_preamble avx2_x86_avx2 # 2808931 ns/call - -# DEMOD_2400X input conversion: provisional order (hand-written AVX2 first, not timed on x86 hardware) -iqdata_conv_uc8 avx2_x86_avx2 -iqdata_conv_uc8 generic_x86_avx2 -iqdata_conv_uc8 generic_generic - -iqdata_conv_sc16 avx2_x86_avx2 -iqdata_conv_sc16 generic_x86_avx2 -iqdata_conv_sc16 generic_generic - -iqdata_conv_sc16q11 avx2_x86_avx2 -iqdata_conv_sc16q11 generic_x86_avx2 -iqdata_conv_sc16q11 generic_generic +upsample12 avx2_x86_avx2 # 1434127 ns/call +upsample12 generic_x86_avx2 # 4728847 ns/call +upsample12 generic_generic # 4868776 ns/call + +locate_preamble avx2_x86_avx2 # 375300 ns/call +locate_preamble generic_generic # 1879389 ns/call +locate_preamble generic_x86_avx2 # 1943528 ns/call + +# DEMOD_2400X input conversion, measured with starch-benchmark -i 5 (x86-64 with AVX2) +iqdata_conv_uc8 avx2_x86_avx2 # 96348 ns/call +iqdata_conv_uc8 generic_x86_avx2 # 326970 ns/call +iqdata_conv_uc8 generic_generic # 501021 ns/call + +iqdata_conv_sc16 avx2_x86_avx2 # 96116 ns/call +iqdata_conv_sc16 generic_x86_avx2 # 319438 ns/call +iqdata_conv_sc16 generic_generic # 490116 ns/call + +iqdata_conv_sc16q11 avx2_x86_avx2 # 93840 ns/call +iqdata_conv_sc16q11 generic_x86_avx2 # 328977 ns/call +iqdata_conv_sc16q11 generic_generic # 493939 ns/call From fbb2f7b4d07a0931b1cb83422f777f5dfb382d85 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 16:14:20 +0100 Subject: [PATCH 13/25] demod_2400x: fine arrival time for MLAT bestPhase() picks the frame position by bit quality, which is flat across about 3 x 12 MHz samples on a clean pulse, with ties going to the earliest. Used as the timestamp, that gave a 0..2 sample bias depending on the sub-sample phase and a bimodal +-1 sample jitter in noise. After a frame is accepted, fineTiming() correlates its data chips (from the corrected message) with the upsampled signal at pos-2..pos+2 and timestamps the best offset. On synthetic frames at known 12 MHz ticks, all 5 phases, the timestamp is now exact (stock: 0..2 ticks bias, sigma 0.41-0.48 ticks). Decoded messages are unchanged. Co-Authored-By: Claude Opus 5.5 --- demod_2400x.c | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index 2a5251e9a..b991a7263 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -245,10 +245,42 @@ static double signalPower(const int16_t *iq, int pos, int bits, double *energy) } +// Arrival time: the 12 MHz sample within +-2 of 'pos' where the frame's data chips +// (from the corrected message) correlate best with the upsampled signal, i.e. the +// chip centres. bestPhase() picks 'pos' by bit quality, which is flat across about +// 3 samples on a clean pulse, so 'pos' alone is a poor time estimate for MLAT. +static int fineTiming(int pos, const uint8_t *msg, int bits, const uint16_t *udata, int nt) +{ + static const int order[5] = { 0, -1, 1, -2, 2 }; // ties go to the nearest to 'pos' + int stride = NU_STRIDE(nt); + int best = pos; + int32_t best_corr = INT32_MIN; + + if (pos < 2 || pos + 2 + 6 * (16 + 2 * bits) > nt) + return pos; + + for (int k = 0; k < 5; k++) { + int t = pos + order[k]; + const uint16_t *u = &udata[(t % 6) * stride + t / 6 + 16]; // first data chip + int32_t corr = 0; + + for (int i = 0; i < bits; i++) { + int d = u[2 * i] - u[2 * i + 1]; // first chip - second + corr += ((msg[i >> 3] >> (7 - (i & 7))) & 1) ? d : -d; + } + if (corr > best_corr) { + best_corr = corr; + best = t; + } + } + return best; +} + + // Score, decode and pass one frame on to the rest of dump1090. // Returns its length in bits if it was accepted, else 0; *energy gets its signal energy. static int emitFrame(const struct mag_buf *mag, const struct candidate *c, - const int16_t *iq, double *energy) + const int16_t *iq, const uint16_t *udata, int nt, double *energy) { static const struct modesMessage zeroMessage; struct modesMessage mm; @@ -263,10 +295,6 @@ static int emitFrame(const struct mag_buf *mag, const struct candidate *c, } mm = zeroMessage; - - // Timestamp at the end of bit 56, even for 112-bit frames (Beast / Radarcape convention) - mm.timestampMsg = mag->sampleTimestamp + c->pos + UPSAMPLE_DELAY + (8 + 56) * 12; - mm.sysTimestampMsg = mag->sysTimestamp + receiveclock_ms_elapsed(mag->sampleTimestamp, mm.timestampMsg); mm.score = score; if (decodeModesMessage(&mm, c->msg) < 0) { @@ -283,6 +311,11 @@ static int emitFrame(const struct mag_buf *mag, const struct candidate *c, } Modes.stats_current.demod_accepted[mm.correctedbits]++; + // Timestamp at the end of bit 56, even for 112-bit frames (Beast / Radarcape convention) + int tpos = fineTiming(c->pos, mm.msg, mm.msgbits, udata, nt); + mm.timestampMsg = mag->sampleTimestamp + tpos + UPSAMPLE_DELAY + (8 + 56) * 12; + mm.sysTimestampMsg = mag->sysTimestamp + receiveclock_ms_elapsed(mag->sampleTimestamp, mm.timestampMsg); + mm.signalLevel = signalPower(iq, c->pos, mm.msgbits, energy); if (mm.signalLevel > Modes.stats_current.peak_signal_power) Modes.stats_current.peak_signal_power = mm.signalLevel; @@ -325,7 +358,7 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i demodFrame(&c, st->udata, nt); double energy; - int bits = emitFrame(mag, &c, (const int16_t *) mag->data, &energy); + int bits = emitFrame(mag, &c, (const int16_t *) mag->data, st->udata, nt, &energy); if (bits) { energy_sum += energy; // Skip to 8 bits before the end of the frame, as demod_2400.c does From 1bc983ac4428453a01a9f3c03543a42292e5c1e6 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 16:34:46 +0100 Subject: [PATCH 14/25] DEMOD_2400X: least-squares upsampler coefficients, DATA_THRES 65 Upsampler: each of the 5 phases is a least-squares fractional-delay fit (delay 3.1 + 0.2p input samples, flat over 0..1.0 MHz), rounded with a common offset so every phase sums to exactly 112. Against the windowed sinc it matches delay and gain across phases (group delay error <= 0.046 samples to 0.8 MHz) with lower noise gain (0.94 vs 1.04). DATA_THRES 80 -> 65: more messages at no CPU cost and no extra tracks. 60 s Pi 4 RTL-SDR capture (--fix): 83910 -> 84366 usable messages. Weak synthetic frames (+-100 kHz offset) decoded out of 2000: noise 300: 1205 -> 1405, noise 350: 454 -> 585. Timestamps stay exact. Co-Authored-By: Claude Opus 5.5 --- demod_2400x.c | 2 +- dsp/helpers/demod2400x_coeffs.h | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index b991a7263..d63261e2e 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -29,7 +29,7 @@ #ifdef USE_MAGNITUDE #define PEAMBLE_THRES 50 -#define DATA_THRES 80 +#define DATA_THRES 65 #else #define PEAMBLE_THRES 80 #define DATA_THRES 150 diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h index 9c9b9da56..c10c83ba4 100644 --- a/dsp/helpers/demod2400x_coeffs.h +++ b/dsp/helpers/demod2400x_coeffs.h @@ -27,12 +27,24 @@ // sinc fc=1.2 win=12 gain=126 // gain 126 per phase, noise gain 1.035 -static const int8_t upsample_coeffs[5][8] = { +/*static const int8_t upsample_coeffs[5][8] = { { -4, 6, -11, 126, 14, -6, 4, -3 }, // phase=0 sum=126 { -9, 14, -25, 113, 48, -19, 11, -7 }, // phase=1 sum=126 { -10, 15, -27, 85, 85, -27, 15, -10 }, // phase=2 sum=126 { -7, 11, -19, 48, 113, -25, 14, -9 }, // phase=3 sum=126 { -3, 4, -6, 14, 126, -11, 6, -4 }, // phase=4 sum=126 +};*/ + +// Least-squares fractional delay: each phase fitted to delay 3.1 + 0.2p input +// samples with flat gain over 0..1.0 MHz, then rounded with a common offset +// (binary search) so every phase sums to exactly 112. +// Group delay error between phases <= 0.046 samples up to 0.8 MHz, noise gain 0.94. +static const int8_t upsample_coeffs[5][8] = { + { -2, 4, -9, 110, 12, -5, 3, -1 }, // phase=0 sum=112 + { -4, 9, -19, 95, 41, -15, 8, -3 }, // phase=1 sum=112 + { -4, 10, -21, 71, 71, -21, 10, -4 }, // phase=2 sum=112 + { -3, 8, -15, 41, 95, -19, 9, -4 }, // phase=3 sum=112 + { -1, 3, -5, 12, 110, -9, 4, -2 }, // phase=4 sum=112 }; // Exact floor(sqrt(x)) for x < 2^31, for USE_MAGNITUDE. From 6abfb6d67aabb4ac9ed480a49b5772bdbc661cbe Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 16:41:16 +0100 Subject: [PATCH 15/25] demod_2400x: feed adaptive gain, cap signal power at full scale demodulate2400x() never called adaptive_update(), so --adaptive-range and --adaptive-burst got no noise floor or loud-burst input. When either is on, compute 2.4 MHz magnitudes (stock magnitude_sc16q11 kernel, stock scale) and feed them as demod_2400.c does: gaps between frames, then each frame with its message, carrying a frame that runs into the overlap to the next buffer. Signal power per sample is capped at full scale, as the stock magnitude kernels clamp, so message power can't exceed 0 dBFS (was up to +1.1 dBFS on clipped frames). 60 s Pi 4 capture, fake gain table on --ifile: same adaptive decisions as stock (range+burst: same 4 gain steps), noise floor -34.1 vs -34.5 dBFS; per-message RSSI vs stock: mean +0.01 dB, sigma 0.20 dB. Co-Authored-By: Claude Opus 5.5 --- demod_2400x.c | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index d63261e2e..6552e8737 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -49,6 +49,13 @@ struct staticdata_t { unsigned prev_mlen; // new samples in the previous buffer (0 = no previous buffer) int next_start; // first peak position to search in this buffer (skips the rest of a // frame decoded at the end of the previous buffer) + + // Adaptive gain input, as demod_2400.c: 2.4 MHz magnitudes (0..65535) of this buffer + // and the first sample not yet passed to adaptive_update() + int asize; + uint16_t *abuf; + uint16_t *amag; // abuf, or NULL when adaptive gain is off + unsigned adaptive_end; }; // One preamble candidate @@ -91,6 +98,7 @@ static int allocateBuffers(struct staticdata_t *stdata, int totalLength) { if (allocBuffer(6*stride*sizeof(uint16_t), &stdata->usize, (void **)&stdata->udata) < 0) return -1; if (allocBuffer(mt*sizeof(uint8_t), &stdata->msize, (void **)&stdata->metric) < 0) return -1; + if (allocBuffer(totalLength*sizeof(uint16_t), &stdata->asize, (void **)&stdata->abuf) < 0) return -1; return 0; } @@ -234,7 +242,8 @@ static double signalPower(const int16_t *iq, int pos, int bits, double *energy) for (int k = start; k < start + len; k++) { int32_t I = iq[2 * k], Q = iq[2 * k + 1]; - sum += (uint32_t) (I * I + Q * Q); + uint32_t p = (uint32_t) (I * I + Q * Q); + sum += p < 2048 * 2048 ? p : 2048 * 2048; // full scale, as the stock magnitude clamps } *energy = sum / 2048.0 / 2048.0; @@ -280,7 +289,7 @@ static int fineTiming(int pos, const uint8_t *msg, int bits, const uint16_t *uda // Score, decode and pass one frame on to the rest of dump1090. // Returns its length in bits if it was accepted, else 0; *energy gets its signal energy. static int emitFrame(const struct mag_buf *mag, const struct candidate *c, - const int16_t *iq, const uint16_t *udata, int nt, double *energy) + const int16_t *iq, struct staticdata_t *st, int nt, double *energy) { static const struct modesMessage zeroMessage; struct modesMessage mm; @@ -312,7 +321,7 @@ static int emitFrame(const struct mag_buf *mag, const struct candidate *c, Modes.stats_current.demod_accepted[mm.correctedbits]++; // Timestamp at the end of bit 56, even for 112-bit frames (Beast / Radarcape convention) - int tpos = fineTiming(c->pos, mm.msg, mm.msgbits, udata, nt); + int tpos = fineTiming(c->pos, mm.msg, mm.msgbits, st->udata, nt); mm.timestampMsg = mag->sampleTimestamp + tpos + UPSAMPLE_DELAY + (8 + 56) * 12; mm.sysTimestampMsg = mag->sysTimestamp + receiveclock_ms_elapsed(mag->sampleTimestamp, mm.timestampMsg); @@ -322,6 +331,15 @@ static int emitFrame(const struct mag_buf *mag, const struct candidate *c, if (mm.signalLevel > 0.50119) Modes.stats_current.strong_signal_count++; // above -3 dBFS + // Adaptive gain: the samples since the last frame, then this frame's (preamble included) + if (st->amag) { + unsigned j = (unsigned) (c->pos + UPSAMPLE_DELAY) / 5; + if (j > st->adaptive_end) + adaptive_update(&st->amag[st->adaptive_end], j - st->adaptive_end, NULL); + st->adaptive_end = j + (mm.msgbits + 8) * 12 / 5; + adaptive_update(&st->amag[j], st->adaptive_end - j, &mm); + } + useModesMessage(&mm); return mm.msgbits; } @@ -358,7 +376,7 @@ static double searchFrames(const struct mag_buf *mag, struct staticdata_t *st, i demodFrame(&c, st->udata, nt); double energy; - int bits = emitFrame(mag, &c, (const int16_t *) mag->data, st->udata, nt, &energy); + int bits = emitFrame(mag, &c, (const int16_t *) mag->data, st, nt, &energy); if (bits) { energy_sum += energy; // Skip to 8 bits before the end of the frame, as demod_2400.c does @@ -440,9 +458,29 @@ void demodulate2400x(struct mag_buf *mag) checkMetric(stdata.metric, mlen); #endif + // Magnitudes for adaptive gain, on the stock scale + stdata.amag = NULL; + if (Modes.adaptive_burst_control || Modes.adaptive_range_control) { + stdata.amag = stdata.abuf; + starch_magnitude_sc16q11((const sc16_t *) mag->data, stdata.amag, dlen); + if (stdata.adaptive_end > newlen || (mag->flags & MAGBUF_DISCONTINUOUS)) + stdata.adaptive_end = 0; + } + // Decode the frames this buffer owns double signal_energy = searchFrames(mag, &stdata, mlen, ulen); + // Adaptive gain: the rest of the new samples; a frame running into the overlap + // carries on into the next buffer, as demod_2400.c + if (stdata.amag) { + if (stdata.adaptive_end < newlen) { + adaptive_update(&stdata.amag[stdata.adaptive_end], newlen - stdata.adaptive_end, NULL); + stdata.adaptive_end = 0; + } else { + stdata.adaptive_end -= newlen; + } + } + // Noise power = everything in the new samples that wasn't a decoded frame Modes.stats_current.noise_power_sum += mag->mean_power * newlen - signal_energy; Modes.stats_current.noise_power_count += newlen; From 34c85566fb761cfe6bc569825e2cbffa21aceca8 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 17:06:00 +0100 Subject: [PATCH 16/25] Makefile: default version 11.1x for the demod-2400x build Shows in receiver.json (and so on the map) and --help, to tell this build from the packaged dump1090-fa 11.1. Co-Authored-By: Claude Opus 5.5 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7bcbd18d5..d88893cc6 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PROGNAME=dump1090 -DUMP1090_VERSION ?= unknown +DUMP1090_VERSION ?= 11.1x CFLAGS ?= -O3 -g DUMP1090_CFLAGS := -std=c11 -fno-common -Wall -Wmissing-declarations -Wformat-signedness -W From 36c6815de77d2746b2c648f76abb99a3ff0fef43 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 17:45:17 +0100 Subject: [PATCH 17/25] upsample12: mirror-symmetric phases in NEON/AVX2, UPSAMPLE_SHIFT 4 The upsampler phases are mirror images (phase 4-c is phase c reversed, phase 2 symmetric), so NEON and AVX2 form u[k] = x[k] + x[7-k] and v[k] = x[k] - x[7-k] and compute each phase pair from shared sums: half the multiplies. Exact integer arithmetic, so the output is identical to the direct form; both fall back to it if the table isn't mirrored. Generic keeps the direct form as the reference. Rounding shift 5 -> UPSAMPLE_SHIFT 4 in all implementations, one more bit for weak signals. UPSAMPLE_ABS_SUM (largest sum of |taps|, 212) and a _Static_assert keep the power I^2 + Q^2 below 2^31 (max 1.47e9). starch-benchmark: upsample12 also runs on full-range data (uniform -2048..2047 and per-phase worst-case windows, magnitude up to 38371 of 38376), and the generic reference is checked in 64-bit arithmetic. Pi 5: upsample12 NEON 2.48 -> 1.54 ms per call; 60 s capture demod 5.9 -> 4.8 s, 84369 messages (84366 before). Validated on aarch64, ARMv7 and x86 (QEMU). Co-Authored-By: Claude Opus 5.5 --- dsp/helpers/demod2400x_coeffs.h | 36 +++++++++++ dsp/impl/demod2400x.avx2.c | 105 ++++++++++++++++++++++++++------ dsp/impl/demod2400x.benchmark.c | 67 +++++++++++++++++++- dsp/impl/demod2400x.generic.c | 4 +- dsp/impl/demod2400x.neon.c | 89 ++++++++++++++++++++++++--- 5 files changed, 273 insertions(+), 28 deletions(-) diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h index c10c83ba4..7d0c23534 100644 --- a/dsp/helpers/demod2400x_coeffs.h +++ b/dsp/helpers/demod2400x_coeffs.h @@ -39,6 +39,7 @@ // samples with flat gain over 0..1.0 MHz, then rounded with a common offset // (binary search) so every phase sums to exactly 112. // Group delay error between phases <= 0.046 samples up to 0.8 MHz, noise gain 0.94. +#define UPSAMPLE_ABS_SUM 212 // largest sum of |taps| over the phases (phase 2) static const int8_t upsample_coeffs[5][8] = { { -2, 4, -9, 110, 12, -5, 3, -1 }, // phase=0 sum=112 { -4, 9, -19, 95, 41, -15, 8, -3 }, // phase=1 sum=112 @@ -47,6 +48,41 @@ static const int8_t upsample_coeffs[5][8] = { { -1, 3, -5, 12, 110, -9, 4, -2 }, // phase=4 sum=112 }; +// Filter outputs are rounded by (x + 2^(UPSAMPLE_SHIFT-1)) >> UPSAMPLE_SHIFT. With 12-bit input +// (|x| <= 2048) each component is at most 2048 * UPSAMPLE_ABS_SUM >> UPSAMPLE_SHIFT, and the +// power I^2 + Q^2 must stay below 2^31: signed 32-bit arithmetic and demod_isqrt()'s range. +#define UPSAMPLE_SHIFT 4 +#define UPSAMPLE_ROUND (1 << (UPSAMPLE_SHIFT - 1)) +#define UPSAMPLE_MAX_COMPONENT ((2048LL * UPSAMPLE_ABS_SUM + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT) +_Static_assert(2 * UPSAMPLE_MAX_COMPONENT * UPSAMPLE_MAX_COMPONENT < (1LL << 31), + "upsampler power can overflow: raise UPSAMPLE_SHIFT"); + +// The phases are mirror images: phase 4-c is phase c reversed and phase 2 is symmetric. +// With u[k] = x[k] + x[7-k] and v[k] = x[k] - x[7-k], k = 0..3: +// 2 * out[c] = sum pc[c][k] u[k] + sum mc[c][k] v[k] +// 2 * out[4-c] = sum pc[c][k] u[k] - sum mc[c][k] v[k] +// with pc[c][k] = a[c][k] + a[c][7-k] and mc[c][k] = a[c][k] - a[c][7-k], which halves the +// multiplies. Exact integers (u, v fit in 16 bits), so the result is identical to the direct +// form; the doubled sums are rounded by UPSAMPLE_SHIFT + 1. Used by the SIMD versions, which +// fall back to the direct form if the table isn't mirrored. +static inline int upsample_coeffs_mirrored(void) +{ + for (int c = 0; c < 5; c++) + for (int k = 0; k < 8; k++) + if (upsample_coeffs[c][k] != upsample_coeffs[4 - c][7 - k]) + return 0; + return 1; +} + +static inline void upsample_mirror_coeffs(int16_t pc[3][4], int16_t mc[3][4]) +{ + for (int c = 0; c < 3; c++) + for (int k = 0; k < 4; k++) { + pc[c][k] = (int16_t) (upsample_coeffs[c][k] + upsample_coeffs[c][7 - k]); + mc[c][k] = (int16_t) (upsample_coeffs[c][k] - upsample_coeffs[c][7 - k]); + } +} + // Exact floor(sqrt(x)) for x < 2^31, for USE_MAGNITUDE. // After adding 2^-6 the truncated sqrtf is the exact result or within one of it // (sqrtf may be an estimate under -ffast-math), fixed with integer checks. Exact, diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index f49e1108e..543960b2e 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -51,8 +51,8 @@ static inline __attribute__((always_inline)) unsigned getpow_avx2(const __m128i const __m128i cv = _mm_cvtepi8_epi16(cv8); // widen coefficients to 16-bit // madd: multiply 8 pairs and add neighbours -> 4 x int32 partial sums - int32_t resi = (hsum_s32x4_avx2(_mm_madd_epi16(iq.val[0], cv)) + 16) >> 5; - int32_t resq = (hsum_s32x4_avx2(_mm_madd_epi16(iq.val[1], cv)) + 16) >> 5; + int32_t resi = (hsum_s32x4_avx2(_mm_madd_epi16(iq.val[0], cv)) + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT; + int32_t resq = (hsum_s32x4_avx2(_mm_madd_epi16(iq.val[1], cv)) + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT; uint32_t pwr = (uint32_t)(resi*resi) + (uint32_t)(resq*resq); @@ -86,13 +86,10 @@ __m256i isqrt_u32x8_avx2(const __m256i x) return _mm256_sub_epi32(q, _mm256_cmpeq_epi32(_mm256_max_epu32(q1q1, x), x)); } -// Power (or magnitude) of 8 filter outputs, as getpow_avx2() +// Power (or magnitude) of 8 filter outputs already rounded by UPSAMPLE_SHIFT static inline __attribute__((always_inline)) -__m128i getpow8_avx2(__m256i si, __m256i sq, int shift) +__m128i pow8_rounded_avx2(__m256i si, __m256i sq, int shift) { - const __m256i round = _mm256_set1_epi32(16); - si = _mm256_srai_epi32(_mm256_add_epi32(si, round), 5); // (x+16)>>5 - sq = _mm256_srai_epi32(_mm256_add_epi32(sq, round), 5); __m256i pwr = _mm256_add_epi32(_mm256_mullo_epi32(si, si), _mm256_mullo_epi32(sq, sq)); #ifdef USE_MAGNITUDE @@ -106,6 +103,36 @@ __m128i getpow8_avx2(__m256i si, __m256i sq, int shift) return _mm_packus_epi32(_mm256_castsi256_si128(pwr), _mm256_extracti128_si256(pwr, 1)); } +// Power (or magnitude) of 8 filter outputs, as getpow_avx2() +static inline __attribute__((always_inline)) +__m128i getpow8_avx2(__m256i si, __m256i sq, int shift) +{ + const __m256i round = _mm256_set1_epi32(UPSAMPLE_ROUND); + return pow8_rounded_avx2(_mm256_srai_epi32(_mm256_add_epi32(si, round), UPSAMPLE_SHIFT), + _mm256_srai_epi32(_mm256_add_epi32(sq, round), UPSAMPLE_SHIFT), shift); +} + +// The same for doubled filter outputs: rounded (2s) >> (UPSAMPLE_SHIFT + 1) == rounded s >> UPSAMPLE_SHIFT +static inline __attribute__((always_inline)) +__m128i getpow8_x2_avx2(__m256i si2, __m256i sq2, int shift) +{ + const __m256i round = _mm256_set1_epi32(2 * UPSAMPLE_ROUND); + return pow8_rounded_avx2(_mm256_srai_epi32(_mm256_add_epi32(si2, round), UPSAMPLE_SHIFT + 1), + _mm256_srai_epi32(_mm256_add_epi32(sq2, round), UPSAMPLE_SHIFT + 1), shift); +} + +// 16-bit pairs (a[w], b[w]) of 8 windows as 8 x int32 lanes, windows 0..7 in order, for madd +static inline __attribute__((always_inline)) __m256i pairs8_avx2(const __m128i a, const __m128i b) +{ + return _mm256_inserti128_si256(_mm256_castsi128_si256(_mm_unpacklo_epi16(a, b)), _mm_unpackhi_epi16(a, b), 1); +} + +// Two coefficients as a madd pair +static inline __attribute__((always_inline)) __m256i coeff_pair_avx2(int16_t c0, int16_t c1) +{ + return _mm256_set1_epi32((int32_t) ((uint16_t) c0 | ((uint32_t) (uint16_t) c1 << 16))); +} + // 16-bit pairs (v[k], v[k+1]) for 8 consecutive windows, from v[0..15] = lo, hi #define PAIRS_AVX2(lo, hi, k) \ _mm256_inserti128_si256( \ @@ -132,23 +159,65 @@ int STARCH_IMPL(upsample12, avx2) (const int16_t * restrict in, int len, int shi cp[c][j] = _mm256_set1_epi32((int32_t) ((uint16_t) upsample_coeffs[c][2 * j] | ((uint32_t) (uint16_t) upsample_coeffs[c][2 * j + 1] << 16))); + // Mirror-symmetric phases (see demod2400x_coeffs.h): half the multiplies + const int mirrored = upsample_coeffs_mirrored(); + int16_t pc[3][4], mc[3][4]; + upsample_mirror_coeffs(pc, mc); + __m256i pcp[3][2], mcp[3][2]; + for (int c = 0; c < 3; c++) + for (int j = 0; j < 2; j++) { + pcp[c][j] = coeff_pair_avx2(pc[c][2 * j], pc[c][2 * j + 1]); + mcp[c][j] = coeff_pair_avx2(mc[c][2 * j], mc[c][2 * j + 1]); + } + // 8 windows at a time; reads samples n .. n+15 for (; n + 9 <= nw; n += 8) { const iq16x8x2_avx2_t a = load2_s16_avx2(in + 2 * n); // samples n .. n+7 const iq16x8x2_avx2_t b = load2_s16_avx2(in + 2 * n + 16); // samples n+8 .. n+15 + uint16_t t[5][8]; - const __m256i pi0 = PAIRS_AVX2(a.val[0], b.val[0], 0), pq0 = PAIRS_AVX2(a.val[1], b.val[1], 0); - const __m256i pi2 = PAIRS_AVX2(a.val[0], b.val[0], 2), pq2 = PAIRS_AVX2(a.val[1], b.val[1], 2); - const __m256i pi4 = PAIRS_AVX2(a.val[0], b.val[0], 4), pq4 = PAIRS_AVX2(a.val[1], b.val[1], 4); - const __m256i pi6 = PAIRS_AVX2(a.val[0], b.val[0], 6), pq6 = PAIRS_AVX2(a.val[1], b.val[1], 6); + if (mirrored) { + // x[k] = tap k of the 8 windows; u[k] = x[k] + x[7-k], v[k] = x[k] - x[7-k] as madd pairs + __m128i xi[8], xq[8]; + xi[0] = a.val[0]; + xq[0] = a.val[1]; +#define TAPS_AVX2(k) xi[k] = _mm_alignr_epi8(b.val[0], a.val[0], 2 * (k)); xq[k] = _mm_alignr_epi8(b.val[1], a.val[1], 2 * (k)) + TAPS_AVX2(1); TAPS_AVX2(2); TAPS_AVX2(3); TAPS_AVX2(4); TAPS_AVX2(5); TAPS_AVX2(6); TAPS_AVX2(7); +#undef TAPS_AVX2 + __m256i ui[2], vi[2], uq[2], vq[2]; + for (int j = 0; j < 2; j++) { + const int k = 2 * j; + ui[j] = pairs8_avx2(_mm_add_epi16(xi[k], xi[7 - k]), _mm_add_epi16(xi[k + 1], xi[6 - k])); + vi[j] = pairs8_avx2(_mm_sub_epi16(xi[k], xi[7 - k]), _mm_sub_epi16(xi[k + 1], xi[6 - k])); + uq[j] = pairs8_avx2(_mm_add_epi16(xq[k], xq[7 - k]), _mm_add_epi16(xq[k + 1], xq[6 - k])); + vq[j] = pairs8_avx2(_mm_sub_epi16(xq[k], xq[7 - k]), _mm_sub_epi16(xq[k + 1], xq[6 - k])); + } - uint16_t t[5][8]; - for (int c = 0; c < 5; c++) { - const __m256i si = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(pi0, cp[c][0]), _mm256_madd_epi16(pi2, cp[c][1])), - _mm256_add_epi32(_mm256_madd_epi16(pi4, cp[c][2]), _mm256_madd_epi16(pi6, cp[c][3]))); - const __m256i sq = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(pq0, cp[c][0]), _mm256_madd_epi16(pq2, cp[c][1])), - _mm256_add_epi32(_mm256_madd_epi16(pq4, cp[c][2]), _mm256_madd_epi16(pq6, cp[c][3]))); - _mm_storeu_si128((__m128i *) t[c], getpow8_avx2(si, sq, shift)); + for (int c = 0; c < 3; c++) { + const __m256i pi = _mm256_add_epi32(_mm256_madd_epi16(ui[0], pcp[c][0]), _mm256_madd_epi16(ui[1], pcp[c][1])); + const __m256i pq = _mm256_add_epi32(_mm256_madd_epi16(uq[0], pcp[c][0]), _mm256_madd_epi16(uq[1], pcp[c][1])); + if (c == 2) { // symmetric: no v terms + _mm_storeu_si128((__m128i *) t[2], getpow8_x2_avx2(pi, pq, shift)); + break; + } + const __m256i mi = _mm256_add_epi32(_mm256_madd_epi16(vi[0], mcp[c][0]), _mm256_madd_epi16(vi[1], mcp[c][1])); + const __m256i mq = _mm256_add_epi32(_mm256_madd_epi16(vq[0], mcp[c][0]), _mm256_madd_epi16(vq[1], mcp[c][1])); + _mm_storeu_si128((__m128i *) t[c], getpow8_x2_avx2(_mm256_add_epi32(pi, mi), _mm256_add_epi32(pq, mq), shift)); + _mm_storeu_si128((__m128i *) t[4 - c], getpow8_x2_avx2(_mm256_sub_epi32(pi, mi), _mm256_sub_epi32(pq, mq), shift)); + } + } else { + const __m256i pi0 = PAIRS_AVX2(a.val[0], b.val[0], 0), pq0 = PAIRS_AVX2(a.val[1], b.val[1], 0); + const __m256i pi2 = PAIRS_AVX2(a.val[0], b.val[0], 2), pq2 = PAIRS_AVX2(a.val[1], b.val[1], 2); + const __m256i pi4 = PAIRS_AVX2(a.val[0], b.val[0], 4), pq4 = PAIRS_AVX2(a.val[1], b.val[1], 4); + const __m256i pi6 = PAIRS_AVX2(a.val[0], b.val[0], 6), pq6 = PAIRS_AVX2(a.val[1], b.val[1], 6); + + for (int c = 0; c < 5; c++) { + const __m256i si = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(pi0, cp[c][0]), _mm256_madd_epi16(pi2, cp[c][1])), + _mm256_add_epi32(_mm256_madd_epi16(pi4, cp[c][2]), _mm256_madd_epi16(pi6, cp[c][3]))); + const __m256i sq = _mm256_add_epi32(_mm256_add_epi32(_mm256_madd_epi16(pq0, cp[c][0]), _mm256_madd_epi16(pq2, cp[c][1])), + _mm256_add_epi32(_mm256_madd_epi16(pq4, cp[c][2]), _mm256_madd_epi16(pq6, cp[c][3]))); + _mm_storeu_si128((__m128i *) t[c], getpow8_avx2(si, sq, shift)); + } } // Rearrange output diff --git a/dsp/impl/demod2400x.benchmark.c b/dsp/impl/demod2400x.benchmark.c index 97f6cd6c0..c6c6f9e86 100644 --- a/dsp/impl/demod2400x.benchmark.c +++ b/dsp/impl/demod2400x.benchmark.c @@ -13,8 +13,11 @@ #include #include #include +#include +#include #include "dsp/helpers/demod2400x_dsp.h" +#include "dsp/helpers/demod2400x_coeffs.h" #define DEMOD2400X_BENCH_LEN 131072 // one FIFO buffer of I/Q samples #define DEMOD2400X_BENCH_SHIFT 8 @@ -53,17 +56,48 @@ static int16_t *demod2400x_make_iq(int len) return iq; } +// Full-range 12-bit I/Q, to check the upsampler can't overflow: the first half uniform over +// -2048..2047, the second half worst-case windows where every sample has the sign of its +// tap in one phase, at full scale (that phase's largest output, 2048 * sum |tap|), cycling +// through the phases, both signs, and Q equal to or opposite I +static int16_t *demod2400x_make_iq_full(int len) +{ + int16_t *iq = malloc(2 * (size_t) len * sizeof(*iq)); + if (!iq) + return NULL; + + srand(2); + int n = 0; + for (; n < len / 2; ++n) { + iq[2 * n] = (int16_t) ((rand() % 4096) - 2048); + iq[2 * n + 1] = (int16_t) ((rand() % 4096) - 2048); + } + for (int w = 0; n < len; ++n) { + int k = n % 8, c = (w / 4) % 5; + int sign_i = (w & 1) ? -1 : 1, sign_q = (w & 2) ? -sign_i : sign_i; + int t = upsample_coeffs[c][k] < 0 ? -1 : 1; + iq[2 * n] = (int16_t) (sign_i * t > 0 ? 2047 : -2048); + iq[2 * n + 1] = (int16_t) (sign_q * t > 0 ? 2047 : -2048); + if (k == 7) + ++w; + } + return iq; +} + void STARCH_BENCHMARK(upsample12) (void) { const int len = DEMOD2400X_BENCH_LEN; int16_t *in = demod2400x_make_iq(len); + int16_t *in_full = demod2400x_make_iq_full(len); uint16_t *out = calloc(demod2400x_upsample_size(len), sizeof(*out)); - if (in && out) { + if (in && in_full && out) { memset(out, DEMOD2400X_BENCH_POISON, demod2400x_upsample_size(len) * sizeof(*out)); STARCH_BENCHMARK_RUN( upsample12, in, len, DEMOD2400X_BENCH_SHIFT, out ); + STARCH_BENCHMARK_RUN( upsample12, in_full, len, DEMOD2400X_BENCH_SHIFT, out ); } + free(in_full); free(in); free(out); @@ -80,6 +114,37 @@ bool STARCH_BENCHMARK_VERIFY(upsample12) (const int16_t *in, int len, int shift, starch_upsample12_generic_generic(in, len, shift, ref); bool okay = true; + +#ifdef USE_MAGNITUDE + // Check the reference itself in 64-bit arithmetic, so an overflow can't hide + // by being the same in every implementation + int stride = NU_STRIDE(NU_SAMP(len)); + for (int n = 0; n + 8 <= len && okay; ++n) { + for (int c = 0; c < 5; ++c) { + int64_t si = 0, sq = 0; + for (int k = 0; k < 8; ++k) { + si += (int64_t) upsample_coeffs[c][k] * in[2 * (n + k)]; + sq += (int64_t) upsample_coeffs[c][k] * in[2 * (n + k) + 1]; + } + si = (si + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT; + sq = (sq + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT; + int64_t pwr = si * si + sq * sq; + int64_t mag = (int64_t) sqrt((double) pwr); // exact floor(sqrt) after the fix-ups + while (mag * mag > pwr) + --mag; + while ((mag + 1) * (mag + 1) <= pwr) + ++mag; + int o = 5 * n + c; + if (ref[(o % 6) * stride + o / 6] != mag) { + fprintf(stderr, "reference check failed: window %d phase %d: %u, expected %lld\n", + n, c, ref[(o % 6) * stride + o / 6], (long long) mag); + okay = false; + break; + } + } + } +#endif + for (size_t k = 0; k < size; ++k) { if (out[k] != ref[k]) { fprintf(stderr, "verification failed: out[%zu]=%u, expected %u\n", k, out[k], ref[k]); diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c index dc92bc312..2d928731e 100644 --- a/dsp/impl/demod2400x.generic.c +++ b/dsp/impl/demod2400x.generic.c @@ -39,8 +39,8 @@ static inline unsigned getpow_generic( + cv[6] * qq[6] + cv[7] * qq[7]; - si = (si+16)>>5; - sq = (sq+16)>>5; + si = (si + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT; + sq = (sq + UPSAMPLE_ROUND) >> UPSAMPLE_SHIFT; uint32_t pwr = (uint32_t)(si*si) + (uint32_t)(sq*sq); #ifdef USE_MAGNITUDE diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index 76ec00fcd..d33f86800 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -44,7 +44,7 @@ static inline __attribute__((always_inline)) unsigned getpow_neon(const int8x8_t sq = vmlal_s16(sq, vget_high_s16(iq.val[1]), vget_high_s16(cv)); int32x2_t res = hsum2_s32x4_neon(si, sq); // [resi, resq] - res = vrshr_n_s32(res, 5); // (x+16)>>5 on both + res = vrshr_n_s32(res, UPSAMPLE_SHIFT); // rounded shift on both uint32x2_t sqr = vreinterpret_u32_s32(vmul_s32(res, res)); // [resi^2, resq^2] uint32_t pwr = vget_lane_u32(sqr, 0) + vget_lane_u32(sqr, 1); @@ -81,12 +81,10 @@ uint32x4_t isqrt_u32x4_neon(const uint32x4_t x) #endif } -// Power (or magnitude) of 4 filter outputs, as getpow_neon() +// Power (or magnitude) of 4 filter outputs already rounded by UPSAMPLE_SHIFT static inline __attribute__((always_inline)) -uint16x4_t getpow4_neon(int32x4_t si, int32x4_t sq, int shift) +uint16x4_t pow4_rounded_neon(int32x4_t si, int32x4_t sq, int shift) { - si = vrshrq_n_s32(si, 5); // (x+16)>>5 - sq = vrshrq_n_s32(sq, 5); const uint32x4_t pwr = vreinterpretq_u32_s32(vmlaq_s32(vmulq_s32(si, si), sq, sq)); #ifdef USE_MAGNITUDE @@ -97,6 +95,13 @@ uint16x4_t getpow4_neon(int32x4_t si, int32x4_t sq, int shift) #endif } +// Power (or magnitude) of 4 filter outputs, as getpow_neon() +static inline __attribute__((always_inline)) +uint16x4_t getpow4_neon(int32x4_t si, int32x4_t sq, int shift) +{ + return pow4_rounded_neon(vrshrq_n_s32(si, UPSAMPLE_SHIFT), vrshrq_n_s32(sq, UPSAMPLE_SHIFT), shift); +} + // One phase of the upsampler for 8 consecutive windows; s[k] = I/Q at window + k static inline __attribute__((always_inline)) uint16x8_t upsample_phase8_neon(const int16x8x2_t * restrict s, const int8_t * restrict coeffs, int shift) @@ -123,6 +128,62 @@ uint16x8_t upsample_phase8_neon(const int16x8x2_t * restrict s, const int8_t * r return vcombine_u16(getpow4_neon(si_lo, sq_lo, shift), getpow4_neon(si_hi, sq_hi, shift)); } +// Doubled filter outputs -> power (or magnitude) of 4 outputs, as getpow4_neon() +static inline __attribute__((always_inline)) +uint16x4_t getpow4_x2_neon(int32x4_t si2, int32x4_t sq2, int shift) +{ + // rounded (2s) >> (UPSAMPLE_SHIFT + 1) == rounded s >> UPSAMPLE_SHIFT + return pow4_rounded_neon(vrshrq_n_s32(si2, UPSAMPLE_SHIFT + 1), vrshrq_n_s32(sq2, UPSAMPLE_SHIFT + 1), shift); +} + +// sum over k = 0..3 of x[k] * c[k], 8 windows -> lo and hi halves +#define MAC4(x, c, lo, hi) \ + do { \ + lo = vmull_lane_s16(vget_low_s16((x)[0]), c, 0); \ + hi = vmull_lane_s16(vget_high_s16((x)[0]), c, 0); \ + lo = vmlal_lane_s16(lo, vget_low_s16((x)[1]), c, 1); \ + hi = vmlal_lane_s16(hi, vget_high_s16((x)[1]), c, 1); \ + lo = vmlal_lane_s16(lo, vget_low_s16((x)[2]), c, 2); \ + hi = vmlal_lane_s16(hi, vget_high_s16((x)[2]), c, 2); \ + lo = vmlal_lane_s16(lo, vget_low_s16((x)[3]), c, 3); \ + hi = vmlal_lane_s16(hi, vget_high_s16((x)[3]), c, 3); \ + } while (0) + +// All 5 phases for 8 consecutive windows using the mirror symmetry (see demod2400x_coeffs.h); +// s[k] = I/Q at window + k, pc/mc from upsample_mirror_coeffs() +static inline __attribute__((always_inline)) +void upsample_mirror8_neon(const int16x8x2_t * restrict s, const int16x4_t * restrict pc, + const int16x4_t * restrict mc, int shift, uint16x8_t * restrict ph) +{ + int16x8_t ui[4], vi[4], uq[4], vq[4]; + for (int k = 0; k < 4; k++) { + ui[k] = vaddq_s16(s[k].val[0], s[7 - k].val[0]); + vi[k] = vsubq_s16(s[k].val[0], s[7 - k].val[0]); + uq[k] = vaddq_s16(s[k].val[1], s[7 - k].val[1]); + vq[k] = vsubq_s16(s[k].val[1], s[7 - k].val[1]); + } + + int32x4_t pi_lo, pi_hi, pq_lo, pq_hi, mi_lo, mi_hi, mq_lo, mq_hi; + + // Phases 0 and 4, then 1 and 3 + for (int c = 0; c < 2; c++) { + MAC4(ui, pc[c], pi_lo, pi_hi); + MAC4(uq, pc[c], pq_lo, pq_hi); + MAC4(vi, mc[c], mi_lo, mi_hi); + MAC4(vq, mc[c], mq_lo, mq_hi); + ph[c] = vcombine_u16(getpow4_x2_neon(vaddq_s32(pi_lo, mi_lo), vaddq_s32(pq_lo, mq_lo), shift), + getpow4_x2_neon(vaddq_s32(pi_hi, mi_hi), vaddq_s32(pq_hi, mq_hi), shift)); + ph[4 - c] = vcombine_u16(getpow4_x2_neon(vsubq_s32(pi_lo, mi_lo), vsubq_s32(pq_lo, mq_lo), shift), + getpow4_x2_neon(vsubq_s32(pi_hi, mi_hi), vsubq_s32(pq_hi, mq_hi), shift)); + } + + // Phase 2 (symmetric) + MAC4(ui, pc[2], pi_lo, pi_hi); + MAC4(uq, pc[2], pq_lo, pq_hi); + ph[2] = vcombine_u16(getpow4_x2_neon(pi_lo, pq_lo, shift), getpow4_x2_neon(pi_hi, pq_hi, shift)); +} +#undef MAC4 + #ifdef __aarch64__ // 5-way interleave of 16-bit lanes: bytes from ph[0..3] (tbl4), then ph[4] (tbx1) static const uint8_t interleave5_tbl4[5][16] = { @@ -179,6 +240,16 @@ int STARCH_IMPL(upsample12, neon) (const int16_t * restrict in, int len, int shi int nw = len - 7; // Windows: 5 outputs each, output o goes to oo[(o % 6) * stride + o / 6] int n = 0; + // Mirror-symmetric phases (see demod2400x_coeffs.h): half the multiplies + const int mirrored = upsample_coeffs_mirrored(); + int16_t pc4[3][4], mc4[3][4]; + upsample_mirror_coeffs(pc4, mc4); + int16x4_t pc[3], mc[3]; + for (int c = 0; c < 3; c++) { + pc[c] = vld1_s16(pc4[c]); + mc[c] = vld1_s16(mc4[c]); + } + // 8 windows at a time for each phase, a block at a time so the outputs // can be put in order then split into the 6 phases with vector loads/stores for (; n + UPSAMPLE_BLOCK <= nw; n += UPSAMPLE_BLOCK) { @@ -191,8 +262,12 @@ int STARCH_IMPL(upsample12, neon) (const int16_t * restrict in, int len, int shi s[k] = vld2q_s16(p + 2 * k); uint16x8_t ph[5]; - for (int c = 0; c < 5; c++) - ph[c] = upsample_phase8_neon(s, upsample_coeffs[c], shift); + if (mirrored) { + upsample_mirror8_neon(s, pc, mc, shift, ph); + } else { + for (int c = 0; c < 5; c++) + ph[c] = upsample_phase8_neon(s, upsample_coeffs[c], shift); + } interleave5_u16x8_neon(ph, tmp + 5 * w); } From 4159fdf9e503651f3bd37c94405ed7043dae3ec8 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 18:02:56 +0100 Subject: [PATCH 18/25] decode_bits starch kernel: NEON and AVX2 bit decoding decodeShortAdbsData() (5 calls per preamble candidate) becomes a wrapper around a new starch kernel, decode_bits(p, nbytes, msg, quality): generic (the old loop), NEON (vld2 splits the chips, vcgt + weighted sum packs a byte, vabd/vpadal for quality and power) and AVX2 (one 256-bit load per byte as 32-bit (a, b) lanes, cmpgt + lane reverse + movemask). Quality goes out through a pointer so starch-benchmark can verify it. Benchmark: 7 bytes on ordinary data and 2048 bytes on full-range data. 7 bytes: Pi 5 NEON 26 ns vs 82 ns generic; ARMv7 NEON 65 vs 102 ns. 60 s capture demod 4.8 -> 4.3 s; raw output with timestamps identical. Validated on aarch64, ARMv7 and x86 (QEMU); x86 wisdom provisional. Co-Authored-By: Claude Opus 5.5 --- demod_2400x.c | 31 +------- dsp/generated/benchmark.c | 125 ++++++++++++++++++++++++++++++++ dsp/generated/dispatcher.c | 110 +++++++++++++++++++++++++++- dsp/generated/starch.h | 22 ++++++ dsp/helpers/demod2400x_dsp.h | 5 ++ dsp/impl/demod2400x.avx2.c | 31 ++++++++ dsp/impl/demod2400x.benchmark.c | 53 ++++++++++++++ dsp/impl/demod2400x.generic.c | 27 +++++++ dsp/impl/demod2400x.neon.c | 35 +++++++++ dsp/starchgen.py | 3 + wisdom.aarch64 | 5 ++ wisdom.arm | 5 ++ wisdom.x86 | 5 ++ 13 files changed, 426 insertions(+), 31 deletions(-) diff --git a/demod_2400x.c b/demod_2400x.c index 6552e8737..1c0e7b32c 100644 --- a/demod_2400x.c +++ b/demod_2400x.c @@ -151,36 +151,13 @@ static void checkMetric(const uint8_t *metric, int mlen) { #endif -// Decode 56 bits starting at metric position 'pos'; returns a quality score (0..256). -// Each bit is two chips: bit = 1 if the first is larger. Branch-free: on noise the -// bit values are random, and a branch on them mispredicts half the time. +// Decode 56 bits starting at metric position 'pos'; returns a quality score (0..256) static inline __attribute__((always_inline)) unsigned decodeShortAdbsData(int pos, const uint16_t * restrict udata, int nt, uint8_t * restrict msg) { - int stride = NU_STRIDE(nt); - int phase = pos % 6; - int offs = (pos / 6) + 16; // Start of data - uint32_t qual = 0; - uint32_t pow = 0; - - const uint16_t * restrict p = &udata[phase * stride + offs]; - for (int i = 0; i < SHORT_FRAME / 8; i++) { // 7 bytes = 56 bits, 16 chips per byte - - unsigned bits = 0; - for (int k = 0; k < 8; k++) { - int a = p[2 * k]; - int b = p[2 * k + 1]; - int d = b - a; - bits = (bits << 1) | ((unsigned) d >> 31); // 1 if a > b - qual += (uint32_t) (d < 0 ? -d : d); - pow += (uint32_t) (a + b); - } - - msg[i] = (uint8_t) bits; - p += 16; - } - unsigned score = (256 * qual) / (pow + 1); - return score; + unsigned quality; + starch_decode_bits(&udata[(pos % 6) * NU_STRIDE(nt) + pos / 6 + 16], SHORT_FRAME / 8, msg, &quality); + return quality; } diff --git a/dsp/generated/benchmark.c b/dsp/generated/benchmark.c index 1b6709c85..1ddd68627 100644 --- a/dsp/generated/benchmark.c +++ b/dsp/generated/benchmark.c @@ -184,6 +184,119 @@ static void starch_benchmark_run_count_above_u16( const uint16_t * arg0, unsigne } } +/* prototypes for benchmark helpers provided by user code */ +void starch_decode_bits_benchmark (void); +bool starch_decode_bits_benchmark_verify ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); + +/* prototype the benchmarking function so that we can build with -Wmissing-declarations */ +void starch_decode_bits_benchmark(void); + +static void starch_benchmark_one_decode_bits( starch_decode_bits_regentry * _entry, const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ) +{ + fprintf(stderr, " %-40s ", _entry->name); + + /* test for support */ + if (_entry->flavor_supported && !(_entry->flavor_supported())) { + fprintf(stderr, "unsupported\n"); + return; + } + + if (starch_benchmark_flavor_whitelist && !starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_whitelist)) { + fprintf(stderr, "skipped (not whitelisted)\n"); + return; + } + + if (starch_benchmark_flavor_blacklist && starch_benchmark_flavor_in_list(_entry->flavor, starch_benchmark_flavor_blacklist)) { + fprintf(stderr, "skipped (blacklisted)\n"); + return; + } + + if (starch_benchmark_list_only) { + fprintf(stderr, "supported\n"); + return; + } + + /* initial warmup */ + for (unsigned _loop = 0; _loop < starch_benchmark_warmup_loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3 ); + + /* verify correctness of the output */ + if (! starch_decode_bits_benchmark_verify ( arg0, arg1, arg2, arg3 )) { + fprintf(stderr, "skipped (verification failed)\n"); + starch_benchmark_validation_failed = true; + return; + } + if (starch_benchmark_validate_only) { + fprintf(stderr, "validation ok\n"); + return; + } + + /* pre-benchmark, find a loop count that takes at least 100ms */ + starch_benchmark_time _start, _end; + uint64_t _elapsed = 0; + uint64_t _loops = 127; + while (_elapsed < 100000000) { + _loops *= 2; + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3 ); + starch_benchmark_get_time(&_end); + _elapsed = starch_benchmark_elapsed(&_start, &_end); + } + + /* real benchmark, run for approx 1 second */ + _loops = _loops * 1000000000 / _elapsed; + + _elapsed = 0; + uint64_t _elapsed_min = UINT64_MAX; + uint64_t _elapsed_max = 0; + for (unsigned _iter = 0; _iter < starch_benchmark_iterations; ++_iter) { + starch_benchmark_get_time(&_start); + for (uint64_t _loop = 0; _loop < _loops; ++_loop) + _entry->callable ( arg0, arg1, arg2, arg3 ); + starch_benchmark_get_time(&_end); + uint64_t _elapsed_one = starch_benchmark_elapsed(&_start, &_end); + if (_elapsed_one < _elapsed_min) + _elapsed_min = _elapsed_one; + if (_elapsed_one > _elapsed_max) + _elapsed_max = _elapsed_one; + _elapsed += _elapsed_one; + } + + uint64_t _per_loop; + if (starch_benchmark_iterations > 2) + _per_loop = (_elapsed - _elapsed_min - _elapsed_max) / _loops / (starch_benchmark_iterations - 2); + else + _per_loop = _elapsed / _loops / starch_benchmark_iterations; + + fprintf(stderr, "%" PRIu64 " ns/call\n", _per_loop); + + if (starch_benchmark_result_count >= starch_benchmark_result_size) { + if (!starch_benchmark_result_size) + starch_benchmark_result_size = 64; + else + starch_benchmark_result_size *= 2; + starch_benchmark_results = realloc(starch_benchmark_results, starch_benchmark_result_size * sizeof(*starch_benchmark_results)); + if (!starch_benchmark_results) { + fprintf(stderr, "realloc: %s\n", strerror(errno)); + exit(1); + } + } + + starch_benchmark_results[starch_benchmark_result_count].name = "decode_bits"; + starch_benchmark_results[starch_benchmark_result_count].impl = _entry->name; + starch_benchmark_results[starch_benchmark_result_count].ns = _per_loop; + starch_benchmark_results[starch_benchmark_result_count].flavor = _entry->flavor; + ++starch_benchmark_result_count; +} + +static void starch_benchmark_run_decode_bits( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ) +{ + for (starch_decode_bits_regentry *_entry = starch_decode_bits_registry; _entry->name; ++_entry) { + starch_benchmark_one_decode_bits( _entry, arg0, arg1, arg2, arg3 ); + } +} + /* prototypes for benchmark helpers provided by user code */ void starch_iqdata_conv_sc16_benchmark (void); bool starch_iqdata_conv_sc16_benchmark_verify ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); @@ -1338,6 +1451,11 @@ static void starch_benchmark_all_count_above_u16(void) fprintf(stderr, "==== count_above_u16 ===\n"); starch_count_above_u16_benchmark (); } +static void starch_benchmark_all_decode_bits(void) +{ + fprintf(stderr, "==== decode_bits ===\n"); + starch_decode_bits_benchmark (); +} static void starch_benchmark_all_iqdata_conv_sc16(void) { fprintf(stderr, "==== iqdata_conv_sc16 ===\n"); @@ -1458,6 +1576,7 @@ static void starch_benchmark_usage(const char *argv0) fprintf(stderr, "\nFunctions: " "count_above_u16 " + "decode_bits " "iqdata_conv_sc16 " "iqdata_conv_sc16q11 " "iqdata_conv_uc8 " @@ -1558,6 +1677,11 @@ int main(int argc, char **argv) starch_benchmark_all_count_above_u16(); continue; } + if (!strcmp(argv[i], "decode_bits")) { + specific = 1; + starch_benchmark_all_decode_bits(); + continue; + } if (!strcmp(argv[i], "iqdata_conv_sc16")) { specific = 1; starch_benchmark_all_iqdata_conv_sc16(); @@ -1615,6 +1739,7 @@ int main(int argc, char **argv) if (!specific) { starch_benchmark_all_count_above_u16(); + starch_benchmark_all_decode_bits(); starch_benchmark_all_iqdata_conv_sc16(); starch_benchmark_all_iqdata_conv_sc16q11(); starch_benchmark_all_iqdata_conv_uc8(); diff --git a/dsp/generated/dispatcher.c b/dsp/generated/dispatcher.c index 184d6c603..4e7660ea8 100644 --- a/dsp/generated/dispatcher.c +++ b/dsp/generated/dispatcher.c @@ -96,6 +96,84 @@ starch_count_above_u16_regentry starch_count_above_u16_registry[] = { { 0, NULL, NULL, NULL, NULL } }; +/* dispatcher / registry for decode_bits */ + +starch_decode_bits_regentry * starch_decode_bits_select() { + for (starch_decode_bits_regentry *entry = starch_decode_bits_registry; + entry->name; + ++entry) + { + if (entry->flavor_supported && !(entry->flavor_supported())) + continue; + return entry; + } + return NULL; +} + +static void starch_decode_bits_dispatch ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ) { + starch_decode_bits_regentry *entry = starch_decode_bits_select(); + if (!entry) + abort(); + + starch_decode_bits = entry->callable; + starch_decode_bits ( arg0, arg1, arg2, arg3 ); +} + +starch_decode_bits_ptr starch_decode_bits = starch_decode_bits_dispatch; + +void starch_decode_bits_set_wisdom (const char * const * received_wisdom) +{ + /* re-rank the registry based on received wisdom */ + starch_decode_bits_regentry *entry; + for (entry = starch_decode_bits_registry; entry->name; ++entry) { + const char * const *search; + for (search = received_wisdom; *search; ++search) { + if (!strcmp(*search, entry->name)) { + break; + } + } + if (*search) { + /* matches an entry in the wisdom list, order by position in the list */ + entry->rank = search - received_wisdom; + } else { + /* no match, rank after all possible matches, retaining existing order */ + entry->rank = (search - received_wisdom) + (entry - starch_decode_bits_registry); + } + } + + /* re-sort based on the new ranking */ + qsort(starch_decode_bits_registry, entry - starch_decode_bits_registry, sizeof(starch_decode_bits_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_decode_bits = starch_decode_bits_dispatch; +} + +starch_decode_bits_regentry starch_decode_bits_registry[] = { + +#ifdef STARCH_MIX_AARCH64 + { 0, "neon_armv8_neon_simd", "armv8_neon_simd", starch_decode_bits_neon_armv8_neon_simd, cpu_supports_armv8_simd }, + { 1, "generic_armv8_neon_simd", "armv8_neon_simd", starch_decode_bits_generic_armv8_neon_simd, cpu_supports_armv8_simd }, + { 2, "generic_generic", "generic", starch_decode_bits_generic_generic, NULL }, +#endif /* STARCH_MIX_AARCH64 */ + +#ifdef STARCH_MIX_ARM + { 0, "neon_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_decode_bits_neon_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, + { 1, "generic_generic", "generic", starch_decode_bits_generic_generic, NULL }, + { 2, "generic_armv7a_neon_vfpv4", "armv7a_neon_vfpv4", starch_decode_bits_generic_armv7a_neon_vfpv4, cpu_supports_armv7_neon_vfpv4 }, +#endif /* STARCH_MIX_ARM */ + +#ifdef STARCH_MIX_GENERIC + { 0, "generic_generic", "generic", starch_decode_bits_generic_generic, NULL }, +#endif /* STARCH_MIX_GENERIC */ + +#ifdef STARCH_MIX_X86 + { 0, "avx2_x86_avx2", "x86_avx2", starch_decode_bits_avx2_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_x86_avx2", "x86_avx2", starch_decode_bits_generic_x86_avx2, cpu_supports_avx2 }, + { 2, "generic_generic", "generic", starch_decode_bits_generic_generic, NULL }, +#endif /* STARCH_MIX_X86 */ + { 0, NULL, NULL, NULL, NULL } +}; + /* dispatcher / registry for iqdata_conv_sc16 */ starch_iqdata_conv_sc16_regentry * starch_iqdata_conv_sc16_select() { @@ -401,9 +479,9 @@ starch_locate_preamble_regentry starch_locate_preamble_registry[] = { #endif /* STARCH_MIX_GENERIC */ #ifdef STARCH_MIX_X86 - { 0, "generic_x86_avx2", "x86_avx2", starch_locate_preamble_generic_x86_avx2, cpu_supports_avx2 }, + { 0, "avx2_x86_avx2", "x86_avx2", starch_locate_preamble_avx2_x86_avx2, cpu_supports_avx2 }, { 1, "generic_generic", "generic", starch_locate_preamble_generic_generic, NULL }, - { 2, "avx2_x86_avx2", "x86_avx2", starch_locate_preamble_avx2_x86_avx2, cpu_supports_avx2 }, + { 2, "generic_x86_avx2", "x86_avx2", starch_locate_preamble_generic_x86_avx2, cpu_supports_avx2 }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } }; @@ -935,8 +1013,8 @@ starch_upsample12_regentry starch_upsample12_registry[] = { #ifdef STARCH_MIX_X86 { 0, "avx2_x86_avx2", "x86_avx2", starch_upsample12_avx2_x86_avx2, cpu_supports_avx2 }, - { 1, "generic_generic", "generic", starch_upsample12_generic_generic, NULL }, - { 2, "generic_x86_avx2", "x86_avx2", starch_upsample12_generic_x86_avx2, cpu_supports_avx2 }, + { 1, "generic_x86_avx2", "x86_avx2", starch_upsample12_generic_x86_avx2, cpu_supports_avx2 }, + { 2, "generic_generic", "generic", starch_upsample12_generic_generic, NULL }, #endif /* STARCH_MIX_X86 */ { 0, NULL, NULL, NULL, NULL } }; @@ -953,6 +1031,10 @@ int starch_read_wisdom (const char * path) for (starch_count_above_u16_regentry *entry = starch_count_above_u16_registry; entry->name; ++entry) { entry->rank = 0; } + int rank_decode_bits = 0; + for (starch_decode_bits_regentry *entry = starch_decode_bits_registry; entry->name; ++entry) { + entry->rank = 0; + } int rank_iqdata_conv_sc16 = 0; for (starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { entry->rank = 0; @@ -1035,6 +1117,15 @@ int starch_read_wisdom (const char * path) } continue; } + if (!strcmp(name, "decode_bits")) { + for (starch_decode_bits_regentry *entry = starch_decode_bits_registry; entry->name; ++entry) { + if (!strcmp(impl, entry->name)) { + entry->rank = ++rank_decode_bits; + break; + } + } + continue; + } if (!strcmp(name, "iqdata_conv_sc16")) { for (starch_iqdata_conv_sc16_regentry *entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { if (!strcmp(impl, entry->name)) { @@ -1146,6 +1237,17 @@ int starch_read_wisdom (const char * path) /* reset the implementation pointer so the next call will re-select */ starch_count_above_u16 = starch_count_above_u16_dispatch; } + { + starch_decode_bits_regentry *entry; + for (entry = starch_decode_bits_registry; entry->name; ++entry) { + if (!entry->rank) + entry->rank = ++rank_decode_bits; + } + qsort(starch_decode_bits_registry, entry - starch_decode_bits_registry, sizeof(starch_decode_bits_regentry), starch_regentry_rank_compare); + + /* reset the implementation pointer so the next call will re-select */ + starch_decode_bits = starch_decode_bits_dispatch; + } { starch_iqdata_conv_sc16_regentry *entry; for (entry = starch_iqdata_conv_sc16_registry; entry->name; ++entry) { diff --git a/dsp/generated/starch.h b/dsp/generated/starch.h index 544a92b0e..bd1466e6d 100644 --- a/dsp/generated/starch.h +++ b/dsp/generated/starch.h @@ -213,6 +213,21 @@ extern starch_locate_preamble_regentry starch_locate_preamble_registry[]; starch_locate_preamble_regentry * starch_locate_preamble_select(); void starch_locate_preamble_set_wisdom( const char * const * received_wisdom ); +typedef void (* starch_decode_bits_ptr) ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); +extern starch_decode_bits_ptr starch_decode_bits; + +typedef struct { + int rank; + const char *name; + const char *flavor; + starch_decode_bits_ptr callable; + int (*flavor_supported)(); +} starch_decode_bits_regentry; + +extern starch_decode_bits_regentry starch_decode_bits_registry[]; +starch_decode_bits_regentry * starch_decode_bits_select(); +void starch_decode_bits_set_wisdom( const char * const * received_wisdom ); + /* flavors and prototypes */ @@ -226,6 +241,7 @@ int starch_locate_preamble_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, in void starch_iqdata_conv_uc8_generic_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_generic_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); void starch_magnitude_sc16_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_u32_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); @@ -250,6 +266,7 @@ int starch_locate_preamble_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, int a void starch_iqdata_conv_uc8_neon_armv7a_neon_vfpv4 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_neon_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_neon_armv7a_neon_vfpv4 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_neon_armv7a_neon_vfpv4 ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); #endif /* STARCH_FLAVOR_ARMV7A_NEON_VFPV4 */ int starch_read_wisdom (const char * path); @@ -264,6 +281,7 @@ int starch_locate_preamble_generic_armv8_neon_simd ( const uint16_t * arg0, int void starch_iqdata_conv_uc8_generic_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_generic_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); void starch_magnitude_sc16_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_u32_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); @@ -288,6 +306,7 @@ int starch_locate_preamble_neon_armv8_neon_simd ( const uint16_t * arg0, int arg void starch_iqdata_conv_uc8_neon_armv8_neon_simd ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_neon_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_neon_armv8_neon_simd ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_neon_armv8_neon_simd ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); #endif /* STARCH_FLAVOR_ARMV8_NEON_SIMD */ int starch_read_wisdom (const char * path); @@ -301,6 +320,7 @@ int starch_locate_preamble_generic_generic ( const uint16_t * arg0, int arg1, un void starch_iqdata_conv_uc8_generic_generic ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_generic_generic ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); void starch_magnitude_sc16_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_u32_generic ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); @@ -328,6 +348,7 @@ int starch_locate_preamble_generic_x86_avx2 ( const uint16_t * arg0, int arg1, u void starch_iqdata_conv_uc8_generic_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_generic_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_generic_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_generic_x86_avx2 ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); void starch_magnitude_sc16_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16_exact_float_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); void starch_magnitude_sc16q11_exact_u32_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2 ); @@ -346,6 +367,7 @@ int starch_locate_preamble_avx2_x86_avx2 ( const uint16_t * arg0, int arg1, unsi void starch_iqdata_conv_uc8_avx2_x86_avx2 ( const uc8_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16_avx2_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); void starch_iqdata_conv_sc16q11_avx2_x86_avx2 ( const sc16_t * arg0, uint16_t * arg1, unsigned arg2, const int32_t * arg3, int32_t * arg4, double * arg5, double * arg6 ); +void starch_decode_bits_avx2_x86_avx2 ( const uint16_t * arg0, unsigned arg1, uint8_t * arg2, unsigned * arg3 ); #endif /* STARCH_FLAVOR_X86_AVX2 */ int starch_read_wisdom (const char * path); diff --git a/dsp/helpers/demod2400x_dsp.h b/dsp/helpers/demod2400x_dsp.h index 49af40d42..21de6f20f 100644 --- a/dsp/helpers/demod2400x_dsp.h +++ b/dsp/helpers/demod2400x_dsp.h @@ -27,6 +27,11 @@ #define NU_STRIDE(t) ((((t)+5)/6+7)&~7) #define METRIC_SIZE(t) ((t)-6*15-12*LONG_FRAME) +// decode_bits(p, nbytes, msg, quality): bits from chip pairs p[2k], p[2k+1] (bit = 1 if the +// first is larger), 8 bits per byte, most significant first; *quality gets +// 256 * sum |p[2k] - p[2k+1]| / (sum (p[2k] + p[2k+1]) + 1). nbytes <= 2048. +#define DECODE_BITS_MAX_BYTES 2048 + #define PREAMBLE_KK 2 // locate_preamble() sets metrics below its 'thres' argument to 0. That is exact diff --git a/dsp/impl/demod2400x.avx2.c b/dsp/impl/demod2400x.avx2.c index 543960b2e..f9cee0390 100644 --- a/dsp/impl/demod2400x.avx2.c +++ b/dsp/impl/demod2400x.avx2.c @@ -539,3 +539,34 @@ void STARCH_IMPL(iqdata_conv_sc16q11, avx2) (const sc16_t * restrict in, uint16_ { IQDATA_CONV_AVX2_BODY(sc16q11) } + + +// Decode bits from chip pairs (see demod2400x_dsp.h): one byte = 8 pairs per 256-bit load, +// as 32-bit lanes with the first chip in the low half +void STARCH_IMPL(decode_bits, avx2) (const uint16_t * restrict p, unsigned nbytes, uint8_t * restrict msg, + unsigned * restrict quality) +{ + const __m256i lo16 = _mm256_set1_epi32(0xFFFF); + const __m256i reverse = _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0); // lane 0 -> bit 7 + __m256i qual = _mm256_setzero_si256(); + __m256i pow = _mm256_setzero_si256(); + + for (unsigned i = 0; i < nbytes; i++) { + const __m256i x = _mm256_loadu_si256((const __m256i *) (p + 16 * i)); + const __m256i a = _mm256_and_si256(x, lo16); // values < 2^16: signed ops are fine + const __m256i b = _mm256_srli_epi32(x, 16); + const __m256i gt = _mm256_permutevar8x32_epi32(_mm256_cmpgt_epi32(a, b), reverse); + msg[i] = (uint8_t) _mm256_movemask_ps(_mm256_castsi256_ps(gt)); + qual = _mm256_add_epi32(qual, _mm256_abs_epi32(_mm256_sub_epi32(a, b))); + pow = _mm256_add_epi32(pow, _mm256_add_epi32(a, b)); + } + + // Horizontal sums + __m128i q = _mm_add_epi32(_mm256_castsi256_si128(qual), _mm256_extracti128_si256(qual, 1)); + __m128i s = _mm_add_epi32(_mm256_castsi256_si128(pow), _mm256_extracti128_si256(pow, 1)); + __m128i qs = _mm_hadd_epi32(q, s); // [q01, q23, s01, s23] + qs = _mm_hadd_epi32(qs, qs); // [q, s, q, s] + const uint32_t qsum = (uint32_t) _mm_cvtsi128_si32(qs); + const uint32_t ssum = (uint32_t) _mm_extract_epi32(qs, 1); + *quality = (unsigned) ((256 * (uint64_t) qsum) / (ssum + 1)); +} diff --git a/dsp/impl/demod2400x.benchmark.c b/dsp/impl/demod2400x.benchmark.c index c6c6f9e86..5e68adbfc 100644 --- a/dsp/impl/demod2400x.benchmark.c +++ b/dsp/impl/demod2400x.benchmark.c @@ -387,3 +387,56 @@ bool STARCH_BENCHMARK_VERIFY(iqdata_conv_sc16q11) (const sc16_t *in, uint16_t *o free(ref); return okay; } + + +// Bit decoding: 7 bytes (one 56-bit half frame, as used) on ordinary data, and the +// maximum length on upsampled full-range data +void STARCH_BENCHMARK(decode_bits) (void) +{ + const int len = DEMOD2400X_BENCH_LEN; + int16_t *in = demod2400x_make_iq(len); + int16_t *in_full = demod2400x_make_iq_full(len); + uint16_t *pp = calloc(demod2400x_upsample_size(len), sizeof(*pp)); + uint16_t *pp_full = calloc(demod2400x_upsample_size(len), sizeof(*pp_full)); + uint8_t msg[DECODE_BITS_MAX_BYTES]; + unsigned quality; + + if (in && in_full && pp && pp_full) { + starch_upsample12_generic_generic(in, len, DEMOD2400X_BENCH_SHIFT, pp); + starch_upsample12_generic_generic(in_full, len, DEMOD2400X_BENCH_SHIFT, pp_full); + memset(msg, DEMOD2400X_BENCH_POISON, sizeof(msg)); + STARCH_BENCHMARK_RUN( decode_bits, pp + 1001, 7, msg, &quality ); + STARCH_BENCHMARK_RUN( decode_bits, pp_full + 3, DECODE_BITS_MAX_BYTES, msg, &quality ); + } + + free(in); + free(in_full); + free(pp); + free(pp_full); +} + +bool STARCH_BENCHMARK_VERIFY(decode_bits) (const uint16_t *p, unsigned nbytes, uint8_t *msg, unsigned *quality) +{ + uint8_t ref[DECODE_BITS_MAX_BYTES]; + unsigned ref_quality = ~0U; + + memset(ref, DEMOD2400X_BENCH_POISON, sizeof(ref)); + starch_decode_bits_generic_generic(p, nbytes, ref, &ref_quality); + + bool okay = true; + for (unsigned k = 0; k < DECODE_BITS_MAX_BYTES; ++k) { + if (msg[k] != ref[k]) { + fprintf(stderr, "verification failed: msg[%u]=%02x, expected %02x\n", k, msg[k], ref[k]); + okay = false; + break; + } + } + if (okay && *quality != ref_quality) { + fprintf(stderr, "verification failed: quality=%u, expected %u\n", *quality, ref_quality); + okay = false; + } + + memset(msg, DEMOD2400X_BENCH_POISON, DECODE_BITS_MAX_BYTES); + *quality = ~0U; + return okay; +} diff --git a/dsp/impl/demod2400x.generic.c b/dsp/impl/demod2400x.generic.c index 2d928731e..1e3693c2a 100644 --- a/dsp/impl/demod2400x.generic.c +++ b/dsp/impl/demod2400x.generic.c @@ -227,3 +227,30 @@ void STARCH_IMPL(iqdata_conv_sc16q11, generic) (const sc16_t * restrict in, uint dc_out[1] = offset[1]; iqconv_stats(sum_level, sum_power, len, out_level, out_power); } + + +// Decode bits from chip pairs (see demod2400x_dsp.h). Branch-free: on noise the bit +// values are random, and a branch on them mispredicts half the time. +void STARCH_IMPL(decode_bits, generic) (const uint16_t * restrict p, unsigned nbytes, uint8_t * restrict msg, + unsigned * restrict quality) +{ + uint32_t qual = 0; + uint32_t pow = 0; + + for (unsigned i = 0; i < nbytes; i++) { // 16 chips per byte + + unsigned bits = 0; + for (int k = 0; k < 8; k++) { + int a = p[2 * k]; + int b = p[2 * k + 1]; + int d = b - a; + bits = (bits << 1) | ((unsigned) d >> 31); // 1 if a > b + qual += (uint32_t) (d < 0 ? -d : d); + pow += (uint32_t) (a + b); + } + + msg[i] = (uint8_t) bits; + p += 16; + } + *quality = (unsigned) ((256 * (uint64_t) qual) / (pow + 1)); +} diff --git a/dsp/impl/demod2400x.neon.c b/dsp/impl/demod2400x.neon.c index d33f86800..420d920b9 100644 --- a/dsp/impl/demod2400x.neon.c +++ b/dsp/impl/demod2400x.neon.c @@ -594,3 +594,38 @@ void STARCH_IMPL(iqdata_conv_sc16q11, neon) (const sc16_t * restrict in, uint16_ { IQDATA_CONV_NEON_BODY(sc16q11) } + + +// Decode bits from chip pairs (see demod2400x_dsp.h): one byte = 8 pairs per vld2 +void STARCH_IMPL(decode_bits, neon) (const uint16_t * restrict p, unsigned nbytes, uint8_t * restrict msg, + unsigned * restrict quality) +{ + static const uint16_t weights[8] = { 128, 64, 32, 16, 8, 4, 2, 1 }; + const uint16x8_t w = vld1q_u16(weights); + uint32x4_t qual = vdupq_n_u32(0); + uint32x4_t pow = vdupq_n_u32(0); + + for (unsigned i = 0; i < nbytes; i++) { + const uint16x8x2_t ab = vld2q_u16(p + 16 * i); // val[0] = first chips, val[1] = second + const uint16x8_t bits = vandq_u16(vcgtq_u16(ab.val[0], ab.val[1]), w); +#ifdef __aarch64__ + msg[i] = (uint8_t) vaddvq_u16(bits); +#else + uint16x4_t b4 = vadd_u16(vget_low_u16(bits), vget_high_u16(bits)); + b4 = vpadd_u16(b4, b4); + msg[i] = (uint8_t) vget_lane_u16(vpadd_u16(b4, b4), 0); +#endif + qual = vpadalq_u16(qual, vabdq_u16(ab.val[0], ab.val[1])); + pow = vpadalq_u16(pow, ab.val[0]); + pow = vpadalq_u16(pow, ab.val[1]); + } + +#ifdef __aarch64__ + const uint32_t q = vaddvq_u32(qual), s = vaddvq_u32(pow); +#else + const uint32x2_t qs = vpadd_u32(vadd_u32(vget_low_u32(qual), vget_high_u32(qual)), + vadd_u32(vget_low_u32(pow), vget_high_u32(pow))); + const uint32_t q = vget_lane_u32(qs, 0), s = vget_lane_u32(qs, 1); +#endif + *quality = (unsigned) ((256 * (uint64_t) q) / (s + 1)); +} diff --git a/dsp/starchgen.py b/dsp/starchgen.py index b38c77f6a..3576c68c5 100755 --- a/dsp/starchgen.py +++ b/dsp/starchgen.py @@ -50,6 +50,9 @@ name = 'locate_preamble', argtypes = ['const uint16_t *', 'int', 'unsigned', 'uint8_t *'], returntype = 'int') +gen.add_function( + name = 'decode_bits', + argtypes = ['const uint16_t *', 'unsigned', 'uint8_t *', 'unsigned *']) gen.add_flavor( name = 'generic', diff --git a/wisdom.aarch64 b/wisdom.aarch64 index 0641ff098..2c87aeae9 100644 --- a/wisdom.aarch64 +++ b/wisdom.aarch64 @@ -42,3 +42,8 @@ iqdata_conv_sc16 generic_generic iqdata_conv_sc16q11 neon_armv8_neon_simd # 227178 ns/call iqdata_conv_sc16q11 generic_armv8_neon_simd # 423654 ns/call iqdata_conv_sc16q11 generic_generic # 772901 ns/call + +# DEMOD_2400X bit decoding, measured with starch-benchmark -i 10 (Pi 5, Cortex-A76), 7 bytes +decode_bits neon_armv8_neon_simd # 26 ns/call +decode_bits generic_armv8_neon_simd # 82 ns/call +decode_bits generic_generic # 82 ns/call diff --git a/wisdom.arm b/wisdom.arm index 276be313f..a93fd9b89 100644 --- a/wisdom.arm +++ b/wisdom.arm @@ -46,3 +46,8 @@ iqdata_conv_sc16 generic_generic iqdata_conv_sc16q11 neon_armv7a_neon_vfpv4 # 336462 ns/call iqdata_conv_sc16q11 generic_armv7a_neon_vfpv4 # 760099 ns/call iqdata_conv_sc16q11 generic_generic # 1022069 ns/call + +# DEMOD_2400X bit decoding, measured with starch-benchmark -i 5 (32-bit build on a Cortex-A76), 7 bytes +decode_bits neon_armv7a_neon_vfpv4 # 65 ns/call +decode_bits generic_generic # 102 ns/call +decode_bits generic_armv7a_neon_vfpv4 # 110 ns/call diff --git a/wisdom.x86 b/wisdom.x86 index b2b70c101..a3af35c29 100644 --- a/wisdom.x86 +++ b/wisdom.x86 @@ -45,3 +45,8 @@ iqdata_conv_sc16 generic_generic iqdata_conv_sc16q11 avx2_x86_avx2 # 93840 ns/call iqdata_conv_sc16q11 generic_x86_avx2 # 328977 ns/call iqdata_conv_sc16q11 generic_generic # 493939 ns/call + +# DEMOD_2400X bit decoding: provisional order (hand-written AVX2 first) +decode_bits avx2_x86_avx2 +decode_bits generic_x86_avx2 +decode_bits generic_generic From 5c0b3b4898d460efeabe1d7d3c63254f8eeb7eb0 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 18:17:06 +0100 Subject: [PATCH 19/25] README: DEMOD_2400X summary and results What the branch's demodulator does, and stock vs DEMOD_2400X results on a 60 s Pi 4 RTL-SDR capture: messages overall, by signal strength and by distance, MLAT timing on synthetic frames, and CPU on Pi 4 and Pi 5. Co-Authored-By: Claude Opus 5.5 --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/README.md b/README.md index 9e82dd898..a006a2b6a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,95 @@ it can be used to contribute crowd-sourced flight tracking data to FlightAware. It is designed to build as a Debian package, but should also be buildable on many other Linux or Unix-like systems. +## DEMOD_2400X demodulator (this branch) + +This branch adds a new Mode S demodulator for 2.4 MHz SDR input +(`demod_2400x.c`, enabled with `-DDEMOD_2400X -DUSE_MAGNITUDE` in the +Makefile; the build reports version `11.1x`). In outline: + +* DC removal and conversion to 12-bit I/Q in the input converter, so the + FIFO keeps I/Q samples for later processing. +* A 5-phase polyphase FIR upsamples to 12 MHz (least-squares fractional-delay + coefficients, all integer arithmetic), then takes the magnitude. +* A preamble metric at every 12 MHz position, and the 112 bits are decoded + from the best of the neighbouring positions. +* The arrival time is refined by correlating the decoded chips with the + signal, so MLAT timestamps are unbiased across sub-sample phases. +* Adaptive gain and signal level statistics behave as in the stock + demodulator. + +The heavy kernels (`iqdata_conv_*`, `upsample12`, `locate_preamble`, +`decode_bits`) are starch kernels with generic, ARM NEON (ARMv7 and AArch64) +and x86 AVX2 versions. All versions give bit-identical results, which +`starch-benchmark -V` checks, including on full-range worst-case input. + +### Results + +A 60 s capture from a Raspberry Pi 4 with a FlightAware Pro Stick Plus +(gain 49.6 dB), with an outdoor antenna on a 3 m pole and 12 m of LMR400, +decoded by stock dump1090-fa 11.1 and by this branch with the same options +(`--fix`, receiver position set): + +| | Stock | DEMOD_2400X | Change | +|---|---|---|---| +| Usable messages | 73,146 | 84,369 | +15% | +| DF17 (ADS-B) | 10,352 | 11,789 | +14% | +| Global CPR positions | 2,807 | 3,773 | +34% | +| Aircraft | 136 | 141 | +5 | + +The gain is concentrated in weak signals. Messages by signal strength: + +| RSSI (dBFS) | Stock | DEMOD_2400X | Change | +|---|---|---|---| +| −30 to −27 | 131 | 307 | +134% | +| −27 to −24 | 2,347 | 4,199 | +79% | +| −24 to −21 | 7,111 | 9,409 | +32% | +| −21 to −18 | 9,010 | 10,732 | +19% | +| −18 to −15 | 12,213 | 13,707 | +12% | +| −15 to −10 | 18,226 | 19,924 | +9% | +| −10 to −5 | 9,952 | 10,707 | +8% | +| above −5 | 14,151 | 15,378 | +9% | + +Position messages by distance from the receiver: + +| Distance | Stock | DEMOD_2400X | Change | Aircraft (stock / new) | +|---|---|---|---|---| +| 0–50 km | 643 | 799 | +24% | 12 / 13 | +| 50–100 km | 1,225 | 1,523 | +24% | 30 / 30 | +| 100–150 km | 746 | 948 | +27% | 33 / 33 | +| 150–200 km | 356 | 473 | +33% | 22 / 22 | +| 200–250 km | 93 | 133 | +43% | 8 / 14 | +| 250–300 km | 11 | 23 | +109% | 2 / 5 | + +Weak synthetic frames (2,000 DF17 frames with a random ±100 kHz carrier +offset, decoded out of 2,000): 689 (stock) against 1,406 (DEMOD_2400X). + +### Timing (MLAT) + +On synthetic frames at known 12 MHz ticks, covering all 5 sub-sample +phases: + +| | Stock | DEMOD_2400X | +|---|---|---| +| Bias across phases (clean) | 0 to +2 ticks | none | +| Jitter, noisy frames (σ) | 0.41–0.48 ticks | < 0.2 ticks | + +One tick is 83.3 ns. Running live on the Pi 4, mlat-client stays synchronized +with about 260 nearby receivers, as with stock. + +### CPU + +Demodulation time for the 60 s capture: + +| | Stock | DEMOD_2400X | +|---|---|---| +| Raspberry Pi 5 (Cortex-A76) | 2.0 s | 4.3 s | +| Raspberry Pi 4 (Cortex-A72) | 3.9 s | 10.6 s | + +Live on the Pi 4 this is about 31% of one core. These results come from +one receiver and a one-minute capture; longer side-by-side runs are in +progress. + ## Building under bullseye, buster, or stretch ```bash From 5a58fa4f6b92cbf495b3537dc866734c75cd618e Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 18:17:58 +0100 Subject: [PATCH 20/25] README: note Pi 4 timings were taken alongside the live receiver Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a006a2b6a..5bf1a5ab6 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,9 @@ Demodulation time for the 60 s capture: | Raspberry Pi 5 (Cortex-A76) | 2.0 s | 4.3 s | | Raspberry Pi 4 (Cortex-A72) | 3.9 s | 10.6 s | -Live on the Pi 4 this is about 31% of one core. These results come from +The Pi 4 times were measured while the live receiver was also running on +the same Pi, so both are probably slightly high. Live on the Pi 4 this is +about 31% of one core. These results come from one receiver and a one-minute capture; longer side-by-side runs are in progress. From 6ee303f12a8b54c4126c2ee30d4bd579d2450106 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 18:58:47 +0100 Subject: [PATCH 21/25] upsample12 coefficients: every phase sums to 128 Same least-squares design (gen/ls_coeffs.py 1.0 128) at gain 128 instead of 112: less rounding error, group delay error between phases 0.046 -> 0.028 samples. Largest tap 126; UPSAMPLE_ABS_SUM 244 keeps the worst-case power at 1.95e9 < 2^31 with UPSAMPLE_SHIFT 4 (checked by the static assert and the full-range benchmark). 60 s capture: 84389 usable messages (84369 at 112); timestamps exact. Validated on aarch64, ARMv7 and x86 (QEMU). Co-Authored-By: Claude Opus 5.5 --- dsp/helpers/demod2400x_coeffs.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dsp/helpers/demod2400x_coeffs.h b/dsp/helpers/demod2400x_coeffs.h index 7d0c23534..bb8dca2fe 100644 --- a/dsp/helpers/demod2400x_coeffs.h +++ b/dsp/helpers/demod2400x_coeffs.h @@ -37,15 +37,15 @@ // Least-squares fractional delay: each phase fitted to delay 3.1 + 0.2p input // samples with flat gain over 0..1.0 MHz, then rounded with a common offset -// (binary search) so every phase sums to exactly 112. -// Group delay error between phases <= 0.046 samples up to 0.8 MHz, noise gain 0.94. -#define UPSAMPLE_ABS_SUM 212 // largest sum of |taps| over the phases (phase 2) +// (binary search) so every phase sums to exactly 128 (gen/ls_coeffs.py 1.0 128). +// Group delay error between phases <= 0.028 samples up to 0.8 MHz, noise gain 0.94. +#define UPSAMPLE_ABS_SUM 244 // largest sum of |taps| over the phases static const int8_t upsample_coeffs[5][8] = { - { -2, 4, -9, 110, 12, -5, 3, -1 }, // phase=0 sum=112 - { -4, 9, -19, 95, 41, -15, 8, -3 }, // phase=1 sum=112 - { -4, 10, -21, 71, 71, -21, 10, -4 }, // phase=2 sum=112 - { -3, 8, -15, 41, 95, -19, 9, -4 }, // phase=3 sum=112 - { -1, 3, -5, 12, 110, -9, 4, -2 }, // phase=4 sum=112 + { -2, 4, -10, 126, 14, -6, 3, -1 }, // phase=0 sum=128 + { -4, 10, -22, 109, 47, -17, 9, -4 }, // phase=1 sum=128 + { -5, 12, -24, 81, 81, -24, 12, -5 }, // phase=2 sum=128 + { -4, 9, -17, 47, 109, -22, 10, -4 }, // phase=3 sum=128 + { -1, 3, -6, 14, 126, -10, 4, -2 }, // phase=4 sum=128 }; // Filter outputs are rounded by (x + 2^(UPSAMPLE_SHIFT-1)) >> UPSAMPLE_SHIFT. With 12-bit input From aead77c414c74567973c1f00ef372b3cab7dd296 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 19:08:58 +0100 Subject: [PATCH 22/25] Docs: DEMOD_2400X results, notes, AI disclosure; sim/ test tools README.md: short DEMOD_2400X section linking to: * README-demod2400x.md: design outline and results (figures from the current build) * NOTES-demod2400x.md: development notes, decisions with measurements, what was tried and dropped, testing and gotchas * AI-DISCLOSURE.md: use of generative AI on this branch * sim/: upsampler coefficient generator (ls_coeffs.py), synthetic test signal generator (gen_timing.c, needs liquid-dsp) and timestamp checker Co-Authored-By: Claude Opus 5.5 --- AI-DISCLOSURE.md | 37 ++++++++++++ NOTES-demod2400x.md | 130 +++++++++++++++++++++++++++++++++++++++++++ README-demod2400x.md | 91 ++++++++++++++++++++++++++++++ README.md | 103 ++++++---------------------------- sim/README.md | 19 +++++++ sim/gen_timing.c | 118 +++++++++++++++++++++++++++++++++++++++ sim/ls_coeffs.py | 34 +++++++++++ sim/tscheck.py | 20 +++++++ 8 files changed, 465 insertions(+), 87 deletions(-) create mode 100644 AI-DISCLOSURE.md create mode 100644 NOTES-demod2400x.md create mode 100644 README-demod2400x.md create mode 100644 sim/README.md create mode 100644 sim/gen_timing.c create mode 100644 sim/ls_coeffs.py create mode 100644 sim/tscheck.py diff --git a/AI-DISCLOSURE.md b/AI-DISCLOSURE.md new file mode 100644 index 000000000..9b4685f74 --- /dev/null +++ b/AI-DISCLOSURE.md @@ -0,0 +1,37 @@ +# Use of generative AI on this branch + +This branch (`demod-2400x`) contains code and text written with Claude Code +(Anthropic, Claude Opus models), working under my direction. + +## Who did what + +The DEMOD_2400X design is mine: 12 MHz polyphase upsampling of the I/Q, a +magnitude preamble metric at every 12 MHz position, keeping I/Q in the FIFO +for later coherent processing, and the split into starch kernels. So is the +initial implementation. + +Claude Code wrote code to my specification and review, and ran tests and +measurements I asked for. I made the design decisions along the way, for +example integer arithmetic over floating point, generic kept as the plain +reference implementation, equal coefficient sums across phases, the +upsampler rounding shift, and which experiments to drop. + +Commits with a `Co-Authored-By: Claude` trailer were written with Claude +Code. Some earlier commits without the trailer also contain AI-written code, +notably the vectorised `locate_preamble`, the exact integer square root, the +NEON `upsample12` and the DC removal in the input converters +(`3a7afd3`, `5a51c13`, `d70f5d4`, `0f244bc`). The `sim/` tools, +README-demod2400x.md and this file were also written with Claude Code. + +## How it was checked + +* Every SIMD kernel is checked bit-for-bit against the generic version by + `starch-benchmark -V` (`make test`), on AArch64, ARMv7 and x86 AVX2, + including full-range worst-case input and a 64-bit check of the generic + upsampler. +* Decoded output was compared with stock dump1090-fa on real RTL-SDR + captures, message by message (contents, signal level and timestamps). +* Synthetic frames at known times (`sim/`) check sensitivity and MLAT + timestamp accuracy. +* The build has run live on my Raspberry Pi 4 receiver feeding FlightAware, + with MLAT synchronized as with the stock build. diff --git a/NOTES-demod2400x.md b/NOTES-demod2400x.md new file mode 100644 index 000000000..d02652537 --- /dev/null +++ b/NOTES-demod2400x.md @@ -0,0 +1,130 @@ +# DEMOD_2400X development notes + +What was built, why, and what was tried and dropped. Results are in +[README-demod2400x.md](README-demod2400x.md). Times are for the 60 s Pi 4 +RTL-SDR capture decoded on a Raspberry Pi 5 unless stated otherwise. + +## Signal path + +1. **Input conversion** (`iqdata_conv_uc8/sc16/sc16q11`, `convert.c`). + Converts the SDR samples to 12-bit signed I/Q and removes DC in the same + pass. The DC filter holds its correction for blocks of 16 samples + (`IQ_DC_BLOCK`) and updates a running mean with a time constant of 4096 + samples, so SIMD versions need no serial loop. The filter state lives in + `convert.c` (`converter_state`) and is passed in and out (`dc_in`, + `dc_out`), never static, which also keeps benchmark verification + repeatable. Mean level and power are computed after DC removal. The FIFO + holds I/Q (`magbuf_sample()`), so later stages, such as a coherent + demodulator, can reuse it. +2. **Upsampling** (`upsample12`). A 5-phase, 8-tap polyphase FIR takes + 2.4 MHz I/Q to 12 MHz, rounds by `UPSAMPLE_SHIFT` and takes the exact + integer magnitude, `floor(sqrt(I² + Q²))`. Output is stored as 6 rows, one + per 12 MHz tick within a 0.5 µs chip. +3. **Preamble search** (`locate_preamble`). The metric + `512·minp / (2·minp + gap + 1)` at every 12 MHz position, from the + smallest of the 4 preamble pulses and the sum of the 12 gaps. Below + `PEAMBLE_THRES` it is set to 0 without dividing, which skips most noise. +4. **Candidates** (`demod_2400x.c`). Peaks are scanned 8 metric bytes at a + time (a SWAR test against the threshold). For each peak, the first 56 bits + are decoded at the peak and its two neighbours, and the best quality wins + (`bestPhase`). The second 56 bits are always decoded too, because CRC + correction can change the DF field, including the bit that selects a + 56- or 112-bit frame. A long frame other than DF17/18 is rejected if its + second half quality is low. +5. **Bit decoding** (`decode_bits`). Bit = 1 if the first chip of the pair is + larger. Quality = 256·Σ|a−b| / Σ(a+b). +6. **Output** (`emitFrame`). Scored and decoded by the stock code. Then: + * Fine arrival time: the decoded chips are correlated with the signal at + `pos−2 … pos+2` and the best offset is used for the MLAT timestamp. + * Signal level from the I/Q, with each sample's power capped at full + scale as the stock magnitude kernels do. + * Adaptive gain gets 2.4 MHz magnitudes and each decoded message, fed the + same way as the stock demodulator (only when adaptive gain is on). + +## Decisions and why + +| Change | Why | Effect | +|---|---|---| +| DC removal in the converter | One pass over the data; FIFO keeps clean I/Q for later use | Fixed an overlap write-offset bug on the way; 3325 → 3332 messages on the SC16 test file | +| Always decode 112 bits | CRC correction can fix the DF bits, so length is only known after scoring | No lost long frames | +| Skip the metric divide below threshold | `floor(n/d) ≥ T ⇔ n ≥ T·d`; peaks below T are never used | ~1570 → ~1490 ms (SC16 file) | +| AVX2 divide/sqrt corrected both ways | `-ffast-math` lets the compiler use reciprocal estimates; results were one too small on real Intel hardware | Validation passes on Intel | +| Fine arrival time | Bit quality is flat over ~3 ticks on a clean pulse, so `pos` was biased (0 to +2 ticks by phase) and jittery (σ 0.88) | Exact on synthetic frames; MLAT synced with ~260 receivers live | +| Least-squares coefficients | The windowed sinc had uneven delay and gain across phases | Weak synthetic frames 1205 → 1405 | +| All phases sum to 128 | Less rounding error; still fits the overflow bound | Delay error 0.046 → 0.028 samples | +| `DATA_THRES` 80 → 65 | More messages, no extra CPU or junk tracks | +0.2% messages | +| `UPSAMPLE_SHIFT` 5 → 4 | One more bit for weak signals; power still < 2³¹ | Results unchanged within noise | +| Mirror-symmetric upsampler (NEON, AVX2) | Phase 4−c is phase c reversed, so pairs share sums `x[k] ± x[7−k]`: half the multiplies | `upsample12` 2.48 → 1.54 ms per call | +| `decode_bits` starch kernel | Called 5 times per candidate, ~2M candidates a minute | 82 → 26 ns per call; demod 4.8 → 4.3 s | +| Adaptive gain fed | It was not called at all by the new demodulator | Same gain decisions as stock | +| Signal power capped at full scale | Clipped frames reported up to +1.1 dBFS | RSSI matches stock (mean +0.01 dB, σ 0.2 dB) | + +### Overflow bound + +Input is 12-bit (|x| ≤ 2048). Each filter output is at most +`2048·UPSAMPLE_ABS_SUM >> UPSAMPLE_SHIFT`, and the power I² + Q² must stay +below 2³¹ for signed 32-bit arithmetic and the exact square root. With +`UPSAMPLE_ABS_SUM` 244 (the largest Σ|tap|, printed by `sim/ls_coeffs.py`) and +shift 4, the worst case is 1.95·10⁹. A `_Static_assert` in +`demod2400x_coeffs.h` enforces it, and the benchmark runs worst-case input. + +### Bit-exactness + +Generic is the reference; NEON and AVX2 must match it exactly, which +`starch-benchmark -V` checks. That means exact methods only: + +* Square root: float sqrt, then integer checks in both directions. +* Metric divide: float divide, then integer checks in both directions. +* The mirror trick is exact integer arithmetic. Generic keeps the direct form. + +## Tried and dropped + +* **2.4 MHz pre-detector gate** (skip upsampling where nothing is + happening): real frames occur at 0.75–1.25 times the mean magnitude, so a + gate cheap enough to help misses frames. +* **Sharpened upsampler** (`coeffs.py` parameter `a`): amplifies noise more + than it sharpens pulses; weak frames 1205 → 616 at `a = 0.1`. +* **Lower `PEAMBLE_THRES`**: 50 → 40 gives +1.2% messages for +64% CPU; + at 30, false aircraft appear. +* **Soft-decision bit flipping** (flip the weakest bits when the CRC fails): + `--fix` already repairs any single bit, so this only helps 2-bit errors, + at a false-decode risk. Not pursued. + +## Testing + +* `make test`: `cprtests` and `starch-benchmark -V`. Each kernel is checked + against generic, including full-range worst-case input for `upsample12` + and a 64-bit check of the generic upsampler. +* Cross-builds: ARMv7 (`ARCH=armv7l CC=arm-linux-gnueabihf-gcc + LDFLAGS=-static`, runs natively on AArch64) and x86 + (`ARCH=x86_64 CC=x86_64-linux-gnu-gcc`, run with `qemu-x86_64 -cpu max`, + correctness only; QEMU computes reciprocal estimates exactly, so it can't + reproduce `-ffast-math` problems). +* Real data: stock and DEMOD_2400X on the same capture, compared message by + message (contents, RSSI, timestamps). +* Synthetic data: `sim/gen_timing.c`. Frames need a carrier offset: at + exactly 0 Hz the DC filter removes part of the signal. +* Adaptive gain with `--ifile` needs a gain table; for testing, a fake R820T + table was patched into a scratch copy of `sdr.c` (not committed). + +## Gotchas + +* `dsp/generated/*.o` don't depend on `dsp/helpers/*.h`. After a header edit, + run `rm dsp/generated/*.o`, or generic keeps old code and verification + compares against stale results. +* After adding a starch function or changing a wisdom file, rerun + `python3 dsp/starchgen.py .`; the wisdom order is compiled into + `dispatcher.c`. +* After changing the version string, run `make clean`: objects don't depend + on the Makefile. +* starch finds `STARCH_IMPL` by regex, so implementations can't be generated + by macros. +* starch doesn't pass a kernel's return value to its verifier, so + `decode_bits` returns its quality through a pointer. + +## Possible next steps + +* Coherent demodulation as a fallback when magnitude demodulation fails, + reusing the I/Q kept in the FIFO. +* An early DF check before decoding the second half and scoring. +* Real x86 timings for `wisdom.x86` (`decode_bits` order is provisional). diff --git a/README-demod2400x.md b/README-demod2400x.md new file mode 100644 index 000000000..06facf1ae --- /dev/null +++ b/README-demod2400x.md @@ -0,0 +1,91 @@ +# DEMOD_2400X demodulator + +This branch adds a new Mode S demodulator for 2.4 MHz SDR input +(`demod_2400x.c`, enabled with `-DDEMOD_2400X -DUSE_MAGNITUDE` in the +Makefile; the build reports version `11.1x`). In outline: + +* DC removal and conversion to 12-bit I/Q in the input converter, so the + FIFO keeps I/Q samples for later processing. +* A 5-phase polyphase FIR upsamples to 12 MHz (least-squares fractional-delay + coefficients, all integer arithmetic), then takes the magnitude. +* A preamble metric at every 12 MHz position, and the 112 bits are decoded + from the best of the neighbouring positions. +* The arrival time is refined by correlating the decoded chips with the + signal, so MLAT timestamps are unbiased across sub-sample phases. +* Adaptive gain and signal level statistics behave as in the stock + demodulator. + +The heavy kernels (`iqdata_conv_*`, `upsample12`, `locate_preamble`, +`decode_bits`) are starch kernels with generic, ARM NEON (ARMv7 and AArch64) +and x86 AVX2 versions. All versions give bit-identical results, which +`starch-benchmark -V` checks, including on full-range worst-case input. + +## Results + +A 60 s capture from a Raspberry Pi 4 with a FlightAware Pro Stick Plus +(gain 49.6 dB), with an outdoor antenna on a 3 m pole and 12 m of LMR400, +decoded by stock dump1090-fa 11.1 and by this branch with the same options +(`--fix`, receiver position set): + +| | Stock | DEMOD_2400X | Change | +|---|---|---|---| +| Usable messages | 73,146 | 84,389 | +15% | +| DF17 (ADS-B) | 10,352 | 11,795 | +14% | +| Global CPR positions | 2,807 | 3,750 | +34% | +| Aircraft | 136 | 141 | +5 | + +The gain is concentrated in weak signals. Messages by signal strength: + +| RSSI (dBFS) | Stock | DEMOD_2400X | Change | +|---|---|---|---| +| −30 to −27 | 131 | 302 | +131% | +| −27 to −24 | 2,347 | 4,205 | +79% | +| −24 to −21 | 7,111 | 9,407 | +32% | +| −21 to −18 | 9,010 | 10,759 | +19% | +| −18 to −15 | 12,213 | 13,703 | +12% | +| −15 to −10 | 18,226 | 19,924 | +9% | +| −10 to −5 | 9,952 | 10,710 | +8% | +| above −5 | 14,151 | 15,375 | +9% | + +Position messages by distance from the receiver: + +| Distance | Stock | DEMOD_2400X | Change | Aircraft (stock / new) | +|---|---|---|---|---| +| 0–50 km | 643 | 792 | +23% | 12 / 13 | +| 50–100 km | 1,225 | 1,520 | +24% | 30 / 30 | +| 100–150 km | 746 | 957 | +28% | 33 / 33 | +| 150–200 km | 356 | 472 | +33% | 22 / 22 | +| 200–250 km | 93 | 130 | +40% | 8 / 13 | +| 250–300 km | 11 | 24 | +118% | 2 / 5 | + +Weak synthetic frames (2,000 DF17 frames with a random ±100 kHz carrier +offset, decoded out of 2,000): 689 (stock) against 1,404 (DEMOD_2400X). +The generator and checker are in [sim/](sim/). + +## Timing (MLAT) + +On synthetic frames at known 12 MHz ticks, covering all 5 sub-sample +phases: + +| | Stock | DEMOD_2400X | +|---|---|---| +| Bias across phases (clean) | 0 to +2 ticks | none | +| Jitter, noisy frames (σ) | 0.41–0.48 ticks | < 0.2 ticks | + +One tick is 83.3 ns. Running live on the Pi 4, mlat-client stays synchronized +with about 260 nearby receivers, as with stock. + +## CPU + +Demodulation time for the 60 s capture: + +| | Stock | DEMOD_2400X | +|---|---|---| +| Raspberry Pi 5 (Cortex-A76) | 2.0 s | 4.3 s | +| Raspberry Pi 4 (Cortex-A72) | 3.9 s | 10.6 s | + +The Pi 4 times were measured while the live receiver was also running on +the same Pi, so both are probably slightly high. Live on the Pi 4 this is +about 31% of one core. These results come from +one receiver and a one-minute capture; longer side-by-side runs are in +progress. diff --git a/README.md b/README.md index 5bf1a5ab6..c6c9afe45 100644 --- a/README.md +++ b/README.md @@ -18,94 +18,23 @@ many other Linux or Unix-like systems. ## DEMOD_2400X demodulator (this branch) -This branch adds a new Mode S demodulator for 2.4 MHz SDR input +This branch adds an alternative Mode S demodulator for 2.4 MHz SDR input (`demod_2400x.c`, enabled with `-DDEMOD_2400X -DUSE_MAGNITUDE` in the -Makefile; the build reports version `11.1x`). In outline: - -* DC removal and conversion to 12-bit I/Q in the input converter, so the - FIFO keeps I/Q samples for later processing. -* A 5-phase polyphase FIR upsamples to 12 MHz (least-squares fractional-delay - coefficients, all integer arithmetic), then takes the magnitude. -* A preamble metric at every 12 MHz position, and the 112 bits are decoded - from the best of the neighbouring positions. -* The arrival time is refined by correlating the decoded chips with the - signal, so MLAT timestamps are unbiased across sub-sample phases. -* Adaptive gain and signal level statistics behave as in the stock - demodulator. - -The heavy kernels (`iqdata_conv_*`, `upsample12`, `locate_preamble`, -`decode_bits`) are starch kernels with generic, ARM NEON (ARMv7 and AArch64) -and x86 AVX2 versions. All versions give bit-identical results, which -`starch-benchmark -V` checks, including on full-range worst-case input. - -### Results - -A 60 s capture from a Raspberry Pi 4 with a FlightAware Pro Stick Plus -(gain 49.6 dB), with an outdoor antenna on a 3 m pole and 12 m of LMR400, -decoded by stock dump1090-fa 11.1 and by this branch with the same options -(`--fix`, receiver position set): - -| | Stock | DEMOD_2400X | Change | -|---|---|---|---| -| Usable messages | 73,146 | 84,369 | +15% | -| DF17 (ADS-B) | 10,352 | 11,789 | +14% | -| Global CPR positions | 2,807 | 3,773 | +34% | -| Aircraft | 136 | 141 | +5 | - -The gain is concentrated in weak signals. Messages by signal strength: - -| RSSI (dBFS) | Stock | DEMOD_2400X | Change | -|---|---|---|---| -| −30 to −27 | 131 | 307 | +134% | -| −27 to −24 | 2,347 | 4,199 | +79% | -| −24 to −21 | 7,111 | 9,409 | +32% | -| −21 to −18 | 9,010 | 10,732 | +19% | -| −18 to −15 | 12,213 | 13,707 | +12% | -| −15 to −10 | 18,226 | 19,924 | +9% | -| −10 to −5 | 9,952 | 10,707 | +8% | -| above −5 | 14,151 | 15,378 | +9% | - -Position messages by distance from the receiver: - -| Distance | Stock | DEMOD_2400X | Change | Aircraft (stock / new) | -|---|---|---|---|---| -| 0–50 km | 643 | 799 | +24% | 12 / 13 | -| 50–100 km | 1,225 | 1,523 | +24% | 30 / 30 | -| 100–150 km | 746 | 948 | +27% | 33 / 33 | -| 150–200 km | 356 | 473 | +33% | 22 / 22 | -| 200–250 km | 93 | 133 | +43% | 8 / 14 | -| 250–300 km | 11 | 23 | +109% | 2 / 5 | - -Weak synthetic frames (2,000 DF17 frames with a random ±100 kHz carrier -offset, decoded out of 2,000): 689 (stock) against 1,406 (DEMOD_2400X). - -### Timing (MLAT) - -On synthetic frames at known 12 MHz ticks, covering all 5 sub-sample -phases: - -| | Stock | DEMOD_2400X | -|---|---|---| -| Bias across phases (clean) | 0 to +2 ticks | none | -| Jitter, noisy frames (σ) | 0.41–0.48 ticks | < 0.2 ticks | - -One tick is 83.3 ns. Running live on the Pi 4, mlat-client stays synchronized -with about 260 nearby receivers, as with stock. - -### CPU - -Demodulation time for the 60 s capture: - -| | Stock | DEMOD_2400X | -|---|---|---| -| Raspberry Pi 5 (Cortex-A76) | 2.0 s | 4.3 s | -| Raspberry Pi 4 (Cortex-A72) | 3.9 s | 10.6 s | - -The Pi 4 times were measured while the live receiver was also running on -the same Pi, so both are probably slightly high. Live on the Pi 4 this is -about 31% of one core. These results come from -one receiver and a one-minute capture; longer side-by-side runs are in -progress. +Makefile; the build reports version `11.1x`). It upsamples the I/Q to +12 MHz with a polyphase filter, finds preambles at every 12 MHz position and +refines the arrival time for MLAT. The heavy kernels are starch functions +with generic, NEON and AVX2 versions, checked bit-identical by +`starch-benchmark -V`. + +On a 60 s Raspberry Pi 4 RTL-SDR capture it decodes about 15% more messages +than stock dump1090-fa, mostly weak signals from distant aircraft, for about +2 to 3 times the CPU. + +* [README-demod2400x.md](README-demod2400x.md): design and results +* [NOTES-demod2400x.md](NOTES-demod2400x.md): development notes, decisions and + what was tried +* [AI-DISCLOSURE.md](AI-DISCLOSURE.md): use of generative AI on this branch +* [sim/](sim/): coefficient generator and test-signal tools ## Building under bullseye, buster, or stretch diff --git a/sim/README.md b/sim/README.md new file mode 100644 index 000000000..172f4f247 --- /dev/null +++ b/sim/README.md @@ -0,0 +1,19 @@ +# DEMOD_2400X test tools + +* `ls_coeffs.py fmax gain`: designs the 5-phase upsampler table in + `dsp/helpers/demod2400x_coeffs.h` (least-squares fractional delay, flat to + `fmax` MHz, every phase summing to `gain`) and prints `UPSAMPLE_ABS_SUM`. + It exits with an error if a tap doesn't fit in int8. The current table is + `python3 ls_coeffs.py 1.0 128`. Needs numpy. +* `gen_timing.c`: synthetic DF17 frames at known 12 MHz ticks as an SC16 file, + for sensitivity and timestamp tests. Needs liquid-dsp: + `gcc -O2 -o gen_timing gen_timing.c -lliquid -lm`. +* `tscheck.py`: checks the timestamps dump1090 reports for a `gen_timing` file. + +Example, 2000 weak frames with a random +-100 kHz carrier offset: + +``` +./gen_timing weak.sc16 2000 300 100000 +../dump1090 --ifile weak.sc16 --iformat SC16 --quiet --stats # accepted frames +../dump1090 --ifile weak.sc16 --iformat SC16 --raw --mlat | python3 tscheck.py /dev/stdin +``` diff --git a/sim/gen_timing.c b/sim/gen_timing.c new file mode 100644 index 000000000..10b6f79d8 --- /dev/null +++ b/sim/gen_timing.c @@ -0,0 +1,118 @@ +// Test signal for DEMOD_2400X: DF17 frames at known 12 MHz ticks, after gen_data.c. +// +// Frame k starts at 12 MHz tick (k + 1) * SPACING. Chips are rectangular on the +// 12 MHz grid (6 ticks per 0.5 us chip) and are resampled to 2.4 MHz with liquid-dsp's +// msresamp, so SPACING % 5 == 1 steps through the 5 sub-sample phases. Frame k +// carries k in its payload so tscheck.py can match reported timestamps to it. +// +// With max_offset_hz each frame gets a random carrier offset (+-max) and phase; +// without it frames sit exactly at 0 Hz, where the DEMOD_2400X DC filter removes +// part of the signal, so use an offset for sensitivity tests. +// +// Build: gcc -O2 -o gen_timing gen_timing.c -lliquid -lm +// Usage: gen_timing out.sc16 nframes noise [max_offset_hz] +// noise is the per-channel Gaussian sigma in 12-bit units (frame amplitude 1000) +// Output is SC16 (12-bit values << 4): dump1090 --ifile out.sc16 --iformat SC16 + +#include +#include +#include +#include +#include +#include + +#define SPACING 3601 // 12 MHz ticks between frame starts (300 us) +#define GAIN 1000.0f // frame amplitude, 12-bit units + +// Mode S CRC (generator 0xfff409) over the first bits - 24 bits, bit by bit +static uint32_t modes_crc(const uint8_t *msg, int bits) +{ + uint32_t rem = 0; + for (int i = 0; i < bits - 24; i++) { + uint32_t in = (msg[i >> 3] >> (7 - (i & 7))) & 1; + uint32_t top = ((rem >> 23) & 1) ^ in; + rem = (rem << 1) & 0xffffff; + if (top) + rem ^= 0xfff409; + } + return rem; +} + +static float gauss(void) +{ + float u1 = drand48() + 1e-12f, u2 = drand48(); + return sqrtf(-2.0f * logf(u1)) * cosf(2.0f * M_PI * u2); +} + +static void make_df17(uint8_t msg[14], unsigned k) +{ + memset(msg, 0, 14); + msg[0] = (17 << 3) | 5; + msg[1] = 0x40; msg[2] = 0x68; msg[3] = 0x21; + msg[4] = (11 << 3); + msg[5] = 0x12; msg[6] = 0x34; msg[7] = 0x56; msg[8] = 0x78; + msg[9] = k >> 8; msg[10] = k & 0xff; // frame number + uint32_t crc = modes_crc(msg, 112); + msg[11] = crc >> 16; msg[12] = crc >> 8; msg[13] = crc; +} + +int main(int argc, char **argv) +{ + if (argc < 4) { + fprintf(stderr, "usage: %s out.sc16 nframes noise [max_offset_hz]\n", argv[0]); + return 1; + } + unsigned nframes = atoi(argv[2]); + float noise = atof(argv[3]); + float max_fo = argc > 4 ? atof(argv[4]) : 0; + size_t n12 = (size_t) (nframes + 2) * SPACING; + float *u = calloc(n12, sizeof(float)); + float *o = calloc(n12 / 5 + 1024, sizeof(float)); + uint8_t msg[14]; + + for (unsigned k = 0; k < nframes; k++) { + float *p = u + (size_t) (k + 1) * SPACING; + int chips[16 + 224] = {0}; + chips[0] = chips[2] = chips[7] = chips[9] = 1; // preamble + make_df17(msg, k); + for (int bit = 0; bit < 112; bit++) + chips[16 + 2 * bit + !((msg[bit >> 3] >> (7 - (bit & 7))) & 1)] = 1; + for (int c = 0; c < 240; c++) + for (int j = 0; j < 6; j++) + p[6 * c + j] = chips[c]; + } + + msresamp_rrrf r = msresamp_rrrf_create(.2f, 40.0f); + unsigned ny = 0; + msresamp_rrrf_execute(r, u, n12, o, &ny); + + FILE *f = fopen(argv[1], "wb"); + if (!f) { + perror(argv[1]); + return 1; + } + double ph = 0, w = 0; + for (unsigned i = 0; i < ny; i++) { + if (max_fo > 0 && i % (SPACING / 5) == 0) { // new carrier per frame slot + ph = 2 * M_PI * drand48(); + w = 2 * M_PI * max_fo * (2 * drand48() - 1) / 2.4e6; + } + float ci = 1, cq = -1; // as gen_data.c when max_fo == 0 + if (max_fo > 0) { + ci = (float) (M_SQRT2 * cos(ph)); cq = (float) (M_SQRT2 * sin(ph)); + ph += w; + } + float v[2] = { o[i] * GAIN * ci + noise * gauss(), o[i] * GAIN * cq + noise * gauss() }; + int16_t s[2]; + for (int j = 0; j < 2; j++) { + float x = v[j] > 2047.0f ? 2047.0f : v[j] < -2048.0f ? -2048.0f : v[j]; + s[j] = (int16_t) lrintf(x * 16); + } + fwrite(s, sizeof s, 1, f); + } + fclose(f); + msresamp_rrrf_destroy(r); + free(u); + free(o); + return 0; +} diff --git a/sim/ls_coeffs.py b/sim/ls_coeffs.py new file mode 100644 index 000000000..414332b74 --- /dev/null +++ b/sim/ls_coeffs.py @@ -0,0 +1,34 @@ +# Least-squares fractional-delay upsampler phases, every phase summing to exactly 'gain'. +# Usage: ls_coeffs.py fmax_mhz gain (exits with an error if any |tap| > 127) +import sys, numpy as np +fmax, G = float(sys.argv[1]), int(sys.argv[2]) +k = np.arange(8) +w = 2 * np.pi * np.linspace(0, fmax, 400) / 2.4 +rows = [] +for p in range(5): + t = 3.1 + 0.2 * p # delay in input samples + A = np.exp(-1j * np.outer(w, k)); d = np.exp(-1j * w * t) + A2 = np.vstack([A.real, A.imag, 1000 * np.ones((1, 8))]) # heavy row: DC gain 1 + b2 = np.concatenate([d.real, d.imag, [1000]]) + h = np.linalg.lstsq(A2, b2, rcond=None)[0] * G + lo, hi = -2.0, 2.0 # common offset so the rounded taps sum to G + for _ in range(60): + m = 0.5 * (lo + hi); q = np.round(h + m).astype(int) + if q.sum() == G: break + if q.sum() > G: hi = m + else: lo = m + if q.sum() != G: + sys.exit(f"phase {p}: cannot round to sum {G}") + if np.abs(q).max() > 127: + sys.exit(f"phase {p}: tap {np.abs(q).max()} exceeds 127 at gain {G}") + rows.append(q) +def resp(h, f): return np.sum(h * np.exp(-1j * 2 * np.pi * f / 2.4 * k)) +def gd(h, f, df=0.01): + a = np.angle(resp(h, f + df)) - np.angle(resp(h, f - df)); a = (a + np.pi) % (2 * np.pi) - np.pi + return -a / (2 * np.pi * 2 * df / 2.4) +err = max(abs(gd(r, f) - (3.1 + 0.2 * p)) for p, r in enumerate(rows) for f in (0.0001, 0.4, 0.8)) +ng = sum((r.astype(float) ** 2).sum() for r in rows) / 5 / G ** 2 +print(f"// LS fmax={fmax} gain={G}: max group delay error {err:.3f} samples (0..0.8 MHz), noise gain {ng:.3f}", file=sys.stderr) +print(f"#define UPSAMPLE_ABS_SUM {max(int(abs(r).sum()) for r in rows)} // largest sum of |taps| over the phases") +for p, r in enumerate(rows): + print(" {" + ",".join("%4d" % v for v in r) + f" }}, // phase={p} sum={r.sum()}") diff --git a/sim/tscheck.py b/sim/tscheck.py new file mode 100644 index 000000000..d7fdba5d4 --- /dev/null +++ b/sim/tscheck.py @@ -0,0 +1,20 @@ +# Timestamp check for gen_timing.c output: reported 12 MHz timestamp minus the known +# frame start, by sub-sample phase (start tick mod 5). The mean includes a constant +# offset (resampler and filter delays), which doesn't matter for MLAT; it should be +# the same for every phase, with a small spread. +# Usage: dump1090 --ifile t.sc16 --iformat SC16 --raw --mlat | python3 tscheck.py /dev/stdin +import sys, numpy as np +SPACING = 3601 +err = {} +for l in open(sys.argv[1]): + l = l.strip().rstrip(';') + if not l.startswith('@'): continue + ts, h = int(l[1:13], 16), l[13:] + k = int(h[18:22], 16) # frame number in msg[9..10] + err[k] = ts - (k + 1) * SPACING +k = np.array(sorted(err)); e = np.array([err[i] for i in k]) +print(f" decoded {len(k)} frames; err - 768 (end of bit 56): mean {e.mean() - 768:.3f} std {e.std():.3f}") +for p in range(5): + m = (k + 1) * SPACING % 5 == p + v, c = np.unique(e[m] - 768, return_counts=True) + print(f" start phase {p}: mean {e[m].mean() - 768:+7.3f} " + " ".join(f"{a:+d}:{b}" for a, b in zip(v, c))) From f26af71b970cc669c0ae79d7a15d520e4ce7b992 Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 19:11:14 +0100 Subject: [PATCH 23/25] README-demod2400x: compiler results (gcc 14, clang 19) Same build as the GitHub workflow: gcc 14 and clang 19 on AArch64 build without warnings and pass make test, with byte-identical decoder output; ARMv7 and x86 cross-builds (gcc, clang) pass starch-benchmark -V. Notes that clang turns every AVX2 divide/sqrt into a reciprocal estimate, which QEMU can't reproduce, so the newest AVX2 kernels still need an Intel run. Co-Authored-By: Claude Opus 5.5 --- README-demod2400x.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README-demod2400x.md b/README-demod2400x.md index 06facf1ae..09f4634cc 100644 --- a/README-demod2400x.md +++ b/README-demod2400x.md @@ -81,7 +81,7 @@ Demodulation time for the 60 s capture: | | Stock | DEMOD_2400X | |---|---|---| -| Raspberry Pi 5 (Cortex-A76) | 2.0 s | 4.3 s | +| Raspberry Pi 5 (Cortex-A76) | 2.0 s | 4.3 s (gcc 14), 4.1 s (clang 19) | | Raspberry Pi 4 (Cortex-A72) | 3.9 s | 10.6 s | The Pi 4 times were measured while the live receiver was also running on @@ -89,3 +89,25 @@ the same Pi, so both are probably slightly high. Live on the Pi 4 this is about 31% of one core. These results come from one receiver and a one-minute capture; longer side-by-side runs are in progress. + +## Compilers + +Built with the same options as the GitHub workflow +(`make RTLSDR=yes BLADERF=yes HACKRF=yes LIMESDR=yes SOAPYSDR=yes all`, +then `make test`): + +| Compiler | Target | Warnings | `make test` | +|---|---|---|---| +| gcc 14.2 | AArch64 (Pi 5) | 0 | pass (65 validations) | +| clang 19.1 | AArch64 (Pi 5) | 0 | pass (65 validations) | +| gcc 14.2 (cross) | ARMv7 | 0 | `starch-benchmark -V` passes, run natively on AArch64 | +| gcc 14.2 (cross) | x86-64 AVX2 | 0 | pass under qemu-x86_64 | +| clang 19.1 (cross) | x86-64 AVX2 | 0 | pass under qemu-x86_64 (59 validations) | + +The gcc and clang AArch64 builds give byte-identical decoder output, +timestamps included. On x86, clang with `-ffast-math` compiles every AVX2 +divide and square root to a reciprocal estimate (`vrcpps`/`vrsqrtps`), so the +kernels correct the estimate in both directions to stay exact. QEMU computes +these estimates exactly, so x86 correctness also needs a run on real +hardware. The AVX2 kernels passed on an Intel CPU before the mirrored +upsampler and `decode_bits` were added; those still need an Intel run. From 1298bf7632fd2db768c8aa33cf3ba4f6ed2b607a Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 19:13:28 +0100 Subject: [PATCH 24/25] Docs: describe the test data (SDRplay, RTL-SDR, synthetic, live) NOTES-demod2400x.md gets a Test data section: the SDRplay SC16 recording used for early development, the 60 s RTL-SDR recording from the Pi 4 / FlightAware Pro Stick Plus, synthetic frames from sim/, and the live receiver. Results and AI disclosure say which data each comparison used. Co-Authored-By: Claude Opus 5.5 --- AI-DISCLOSURE.md | 6 ++++-- NOTES-demod2400x.md | 17 +++++++++++++++-- README-demod2400x.md | 4 +++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/AI-DISCLOSURE.md b/AI-DISCLOSURE.md index 9b4685f74..bc2d74514 100644 --- a/AI-DISCLOSURE.md +++ b/AI-DISCLOSURE.md @@ -29,8 +29,10 @@ README-demod2400x.md and this file were also written with Claude Code. `starch-benchmark -V` (`make test`), on AArch64, ARMv7 and x86 AVX2, including full-range worst-case input and a 64-bit check of the generic upsampler. -* Decoded output was compared with stock dump1090-fa on real RTL-SDR - captures, message by message (contents, signal level and timestamps). +* Decoded output was compared with stock dump1090-fa on an RTL-SDR + recording (FlightAware Pro Stick Plus), message by message (contents, + signal level and timestamps). An SDRplay recording was used to check that + each change gave the same or more messages than the version before. * Synthetic frames at known times (`sim/`) check sensitivity and MLAT timestamp accuracy. * The build has run live on my Raspberry Pi 4 receiver feeding FlightAware, diff --git a/NOTES-demod2400x.md b/NOTES-demod2400x.md index d02652537..48424edb1 100644 --- a/NOTES-demod2400x.md +++ b/NOTES-demod2400x.md @@ -4,6 +4,19 @@ What was built, why, and what was tried and dropped. Results are in [README-demod2400x.md](README-demod2400x.md). Times are for the 60 s Pi 4 RTL-SDR capture decoded on a Raspberry Pi 5 unless stated otherwise. +## Test data + +* **SDRplay recording**: 2.4 MHz SC16 I/Q, 17.8 s of signal. Used for the + early development and speed work. +* **RTL-SDR recording**: 60 s at 2.4 MHz, UC8, from the live receiver: a + Raspberry Pi 4 with the FlightAware Pro Stick Plus (blue, internal LNA and + SAW filter) at gain 49.6 dB, outdoor antenna on a 3 m pole, 12 m of + LMR400. Used for the stock comparison, the later tuning and CPU figures. +* **Synthetic frames** from `sim/gen_timing.c` (based on `gen_data.c`): + DF17 frames at known 12 MHz ticks with noise and a random carrier offset, + for sensitivity and MLAT timestamp accuracy. +* **Live**: the same Pi 4 receiver feeding FlightAware, including MLAT. + ## Signal path 1. **Input conversion** (`iqdata_conv_uc8/sc16/sc16q11`, `convert.c`). @@ -45,9 +58,9 @@ RTL-SDR capture decoded on a Raspberry Pi 5 unless stated otherwise. | Change | Why | Effect | |---|---|---| -| DC removal in the converter | One pass over the data; FIFO keeps clean I/Q for later use | Fixed an overlap write-offset bug on the way; 3325 → 3332 messages on the SC16 test file | +| DC removal in the converter | One pass over the data; FIFO keeps clean I/Q for later use | Fixed an overlap write-offset bug on the way; 3325 → 3332 messages on the SDRplay recording | | Always decode 112 bits | CRC correction can fix the DF bits, so length is only known after scoring | No lost long frames | -| Skip the metric divide below threshold | `floor(n/d) ≥ T ⇔ n ≥ T·d`; peaks below T are never used | ~1570 → ~1490 ms (SC16 file) | +| Skip the metric divide below threshold | `floor(n/d) ≥ T ⇔ n ≥ T·d`; peaks below T are never used | ~1570 → ~1490 ms (SDRplay recording) | | AVX2 divide/sqrt corrected both ways | `-ffast-math` lets the compiler use reciprocal estimates; results were one too small on real Intel hardware | Validation passes on Intel | | Fine arrival time | Bit quality is flat over ~3 ticks on a clean pulse, so `pos` was biased (0 to +2 ticks by phase) and jittery (σ 0.88) | Exact on synthetic frames; MLAT synced with ~260 receivers live | | Least-squares coefficients | The windowed sinc had uneven delay and gain across phases | Weak synthetic frames 1205 → 1405 | diff --git a/README-demod2400x.md b/README-demod2400x.md index 09f4634cc..64319ac5e 100644 --- a/README-demod2400x.md +++ b/README-demod2400x.md @@ -25,7 +25,9 @@ and x86 AVX2 versions. All versions give bit-identical results, which A 60 s capture from a Raspberry Pi 4 with a FlightAware Pro Stick Plus (gain 49.6 dB), with an outdoor antenna on a 3 m pole and 12 m of LMR400, decoded by stock dump1090-fa 11.1 and by this branch with the same options -(`--fix`, receiver position set): +(`--fix`, receiver position set). Development also used an SDRplay +recording and synthetic frames; see +[NOTES-demod2400x.md](NOTES-demod2400x.md#test-data). | | Stock | DEMOD_2400X | Change | |---|---|---|---| From b8bfca1b944eb2b9a16c7429228dc8da5ef9e77d Mon Sep 17 00:00:00 2001 From: Charles Baud Date: Mon, 19 Oct 2026 19:14:28 +0100 Subject: [PATCH 25/25] NOTES-demod2400x: SDRplay model (RSP1B); note stock demod crash on its input Co-Authored-By: Claude Opus 5.5 --- NOTES-demod2400x.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/NOTES-demod2400x.md b/NOTES-demod2400x.md index 48424edb1..8b596f087 100644 --- a/NOTES-demod2400x.md +++ b/NOTES-demod2400x.md @@ -6,8 +6,9 @@ RTL-SDR capture decoded on a Raspberry Pi 5 unless stated otherwise. ## Test data -* **SDRplay recording**: 2.4 MHz SC16 I/Q, 17.8 s of signal. Used for the - early development and speed work. +* **SDRplay RSP1B recording**: 2.4 MHz SC16 I/Q, 17.8 s of signal. Used for + the early development and speed work. The stock demodulator crashes on + RSP1B input; that is a separate issue, not fixed on this branch. * **RTL-SDR recording**: 60 s at 2.4 MHz, UC8, from the live receiver: a Raspberry Pi 4 with the FlightAware Pro Stick Plus (blue, internal LNA and SAW filter) at gain 49.6 dB, outdoor antenna on a 3 m pole, 12 m of