diff --git a/csv_reader.c b/csv_reader.c index 316d1d8..6e2386e 100644 --- a/csv_reader.c +++ b/csv_reader.c @@ -5,6 +5,48 @@ #include "csv_parser.h" #include "arena.h" +static const unsigned char UTF8_BOM[] = {0xEF, 0xBB, 0xBF}; +static const unsigned char UTF16LE_BOM[] = {0xFF, 0xFE}; +static const unsigned char UTF16BE_BOM[] = {0xFE, 0xFF}; +static const unsigned char UTF32LE_BOM[] = {0xFF, 0xFE, 0x00, 0x00}; +static const unsigned char UTF32BE_BOM[] = {0x00, 0x00, 0xFE, 0xFF}; + +static void skip_bom(FILE *file, CSVEncoding encoding) { + unsigned char header[4]; + size_t n = 0; + int c; + + while (n < 4 && (c = fgetc(file)) != EOF) { + header[n++] = (unsigned char)c; + } + + size_t bom_len = 0; + + switch (encoding) { + case CSV_ENCODING_UTF8: + if (n >= 3 && memcmp(header, UTF8_BOM, 3) == 0) bom_len = 3; + break; + case CSV_ENCODING_UTF16BE: + if (n >= 2 && memcmp(header, UTF16BE_BOM, 2) == 0) bom_len = 2; + break; + case CSV_ENCODING_UTF16LE: + if (n >= 2 && memcmp(header, UTF16LE_BOM, 2) == 0) bom_len = 2; + break; + case CSV_ENCODING_UTF32BE: + if (n >= 4 && memcmp(header, UTF32BE_BOM, 4) == 0) bom_len = 4; + break; + case CSV_ENCODING_UTF32LE: + if (n >= 4 && memcmp(header, UTF32LE_BOM, 4) == 0) bom_len = 4; + break; + case CSV_ENCODING_ASCII: + case CSV_ENCODING_LATIN1: + default: + break; + } + + fseek(file, bom_len, SEEK_SET); +} + CSVReader* csv_reader_init_with_config(Arena *persistent_arena, Arena *temp_arena, CSVConfig *config) { void *ptr; ArenaResult result = arena_alloc(persistent_arena, sizeof(CSVReader), &ptr); @@ -18,6 +60,8 @@ CSVReader* csv_reader_init_with_config(Arena *persistent_arena, Arena *temp_aren return NULL; } + skip_bom(reader->file, config->encoding); + reader->persistent_arena = persistent_arena; reader->temp_arena = temp_arena; reader->config = config; @@ -88,6 +132,8 @@ CSVReader* csv_reader_init_standalone(CSVConfig *config) { return NULL; } + skip_bom(reader->file, config->encoding); + reader->persistent_arena = persistent_arena; reader->temp_arena = temp_arena; reader->config = config; @@ -194,6 +240,7 @@ char** csv_reader_get_headers(CSVReader *reader, int *header_count) { void csv_reader_rewind(CSVReader *reader) { if (reader && reader->file) { rewind(reader->file); + skip_bom(reader->file, reader->config->encoding); reader->line_number = 0; if (reader->config->hasHeader && reader->headers_loaded) { @@ -227,6 +274,7 @@ long csv_reader_get_record_count(CSVReader *reader) { } rewind(reader->file); + skip_bom(reader->file, reader->config->encoding); long record_count = 0; diff --git a/tests/test_csv_reader.c b/tests/test_csv_reader.c index aa5bccd..e6b9e8e 100644 --- a/tests/test_csv_reader.c +++ b/tests/test_csv_reader.c @@ -260,6 +260,57 @@ void test_csv_reader_set_config() { printf("āœ“ csv_reader_set_config test passed\n"); } +void test_csv_reader_bom() { + printf("Testing CSV reader with BOM...\n"); + + // UTF-8 BOM (EF BB BF) followed by CSV content + const unsigned char bom_content[] = { + 0xEF, 0xBB, 0xBF, + 'N','a','m','e',',','A','g','e','\n', + 'J','o','h','n',',','2','5','\n', + 'J','a','n','e',',','3','0','\n' + }; + FILE *file = fopen("test_bom.csv", "wb"); + assert(file != NULL); + assert(fwrite(bom_content, 1, sizeof(bom_content), file) == sizeof(bom_content)); + fclose(file); + + Arena arena; + assert(arena_create(&arena, 4096) == ARENA_OK); + CSVConfig *config = csv_config_create(&arena); + csv_config_set_path(config, "test_bom.csv"); + csv_config_set_has_header(config, true); + + CSVReader *reader = csv_reader_init_standalone(config); + assert(reader != NULL); + assert(reader->headers_loaded); + assert(reader->cached_header_count == 2); + assert(strcmp(reader->cached_headers[0], "Name") == 0); + assert(strcmp(reader->cached_headers[1], "Age") == 0); + + CSVRecord *record1 = csv_reader_next_record(reader); + assert(record1 != NULL); + assert(record1->field_count == 2); + assert(strcmp(record1->fields[0], "John") == 0); + assert(strcmp(record1->fields[1], "25") == 0); + + CSVRecord *record2 = csv_reader_next_record(reader); + assert(record2 != NULL); + assert(strcmp(record2->fields[0], "Jane") == 0); + assert(strcmp(record2->fields[1], "30") == 0); + + // Test rewind still works correctly with BOM + csv_reader_rewind(reader); + CSVRecord *after_rewind = csv_reader_next_record(reader); + assert(after_rewind != NULL); + assert(strcmp(after_rewind->fields[0], "John") == 0); + + csv_reader_free(reader); + arena_destroy(&arena); + remove("test_bom.csv"); + printf("āœ“ CSV reader BOM test passed\n"); +} + void test_csv_reader_null_safety() { printf("Testing csv_reader null safety...\n"); @@ -373,6 +424,7 @@ int main() { test_csv_reader_position(); test_csv_reader_set_config(); test_csv_reader_get_record_count(); + test_csv_reader_bom(); test_csv_reader_null_safety(); printf("\nāœ… All CSV Reader tests passed!\n"); return 0;