added BOM stripping functionality

csv_reader.c

@@ -5,6 +5,48 @@
 #include "csv_parser.h"
 #include "arena.h"
 
+static const unsigned char UTF8_BOM[]    = {0xEF, 0xBB, 0xBF};
+static const unsigned char UTF16LE_BOM[] = {0xFF, 0xFE};
+static const unsigned char UTF16BE_BOM[] = {0xFE, 0xFF};
+static const unsigned char UTF32LE_BOM[] = {0xFF, 0xFE, 0x00, 0x00};
+static const unsigned char UTF32BE_BOM[] = {0x00, 0x00, 0xFE, 0xFF};
+
+static void skip_bom(FILE *file, CSVEncoding encoding) {
+    unsigned char header[4];
+    size_t n = 0;
+    int c;
+
+    while (n < 4 && (c = fgetc(file)) != EOF) {
+        header[n++] = (unsigned char)c;
+    }
+
+    size_t bom_len = 0;
+
+    switch (encoding) {
+        case CSV_ENCODING_UTF8:
+            if (n >= 3 && memcmp(header, UTF8_BOM, 3) == 0)   bom_len = 3;
+            break;
+        case CSV_ENCODING_UTF16BE:
+            if (n >= 2 && memcmp(header, UTF16BE_BOM, 2) == 0) bom_len = 2;
+            break;
+        case CSV_ENCODING_UTF16LE:
+            if (n >= 2 && memcmp(header, UTF16LE_BOM, 2) == 0) bom_len = 2;
+            break;
+        case CSV_ENCODING_UTF32BE:
+            if (n >= 4 && memcmp(header, UTF32BE_BOM, 4) == 0) bom_len = 4;
+            break;
+        case CSV_ENCODING_UTF32LE:
+            if (n >= 4 && memcmp(header, UTF32LE_BOM, 4) == 0) bom_len = 4;
+            break;
+        case CSV_ENCODING_ASCII:
+        case CSV_ENCODING_LATIN1:
+        default:
+            break;
+    }
+
+    fseek(file, bom_len, SEEK_SET);
+}
+
 CSVReader* csv_reader_init_with_config(Arena *persistent_arena, Arena *temp_arena, CSVConfig *config) {
     void *ptr;
     ArenaResult result = arena_alloc(persistent_arena, sizeof(CSVReader), &ptr);
@@ -18,6 +60,8 @@ CSVReader* csv_reader_init_with_config(Arena *persistent_arena, Arena *temp_aren
         return NULL;
     }
 
+    skip_bom(reader->file, config->encoding);
+
     reader->persistent_arena = persistent_arena;
     reader->temp_arena = temp_arena;
     reader->config = config;
@@ -88,6 +132,8 @@ CSVReader* csv_reader_init_standalone(CSVConfig *config) {
         return NULL;
     }
 
+    skip_bom(reader->file, config->encoding);
+
     reader->persistent_arena = persistent_arena;
     reader->temp_arena = temp_arena;
     reader->config = config;
@@ -194,6 +240,7 @@ char** csv_reader_get_headers(CSVReader *reader, int *header_count) {
 void csv_reader_rewind(CSVReader *reader) {
     if (reader && reader->file) {
         rewind(reader->file);
+        skip_bom(reader->file, reader->config->encoding);
         reader->line_number = 0;
 
         if (reader->config->hasHeader && reader->headers_loaded) {
@@ -227,6 +274,7 @@ long csv_reader_get_record_count(CSVReader *reader) {
     }
 
     rewind(reader->file);
+    skip_bom(reader->file, reader->config->encoding);
 
     long record_count = 0;
 

tests/test_csv_reader.c

@@ -260,6 +260,57 @@ void test_csv_reader_set_config() {
     printf("✓ csv_reader_set_config test passed\n");
 }
 
+void test_csv_reader_bom() {
+    printf("Testing CSV reader with BOM...\n");
+
+    // UTF-8 BOM (EF BB BF) followed by CSV content
+    const unsigned char bom_content[] = {
+        0xEF, 0xBB, 0xBF,
+        'N','a','m','e',',','A','g','e','\n',
+        'J','o','h','n',',','2','5','\n',
+        'J','a','n','e',',','3','0','\n'
+    };
+    FILE *file = fopen("test_bom.csv", "wb");
+    assert(file != NULL);
+    assert(fwrite(bom_content, 1, sizeof(bom_content), file) == sizeof(bom_content));
+    fclose(file);
+
+    Arena arena;
+    assert(arena_create(&arena, 4096) == ARENA_OK);
+    CSVConfig *config = csv_config_create(&arena);
+    csv_config_set_path(config, "test_bom.csv");
+    csv_config_set_has_header(config, true);
+
+    CSVReader *reader = csv_reader_init_standalone(config);
+    assert(reader != NULL);
+    assert(reader->headers_loaded);
+    assert(reader->cached_header_count == 2);
+    assert(strcmp(reader->cached_headers[0], "Name") == 0);
+    assert(strcmp(reader->cached_headers[1], "Age") == 0);
+
+    CSVRecord *record1 = csv_reader_next_record(reader);
+    assert(record1 != NULL);
+    assert(record1->field_count == 2);
+    assert(strcmp(record1->fields[0], "John") == 0);
+    assert(strcmp(record1->fields[1], "25") == 0);
+
+    CSVRecord *record2 = csv_reader_next_record(reader);
+    assert(record2 != NULL);
+    assert(strcmp(record2->fields[0], "Jane") == 0);
+    assert(strcmp(record2->fields[1], "30") == 0);
+
+    // Test rewind still works correctly with BOM
+    csv_reader_rewind(reader);
+    CSVRecord *after_rewind = csv_reader_next_record(reader);
+    assert(after_rewind != NULL);
+    assert(strcmp(after_rewind->fields[0], "John") == 0);
+
+    csv_reader_free(reader);
+    arena_destroy(&arena);
+    remove("test_bom.csv");
+    printf("✓ CSV reader BOM test passed\n");
+}
+
 void test_csv_reader_null_safety() {
     printf("Testing csv_reader null safety...\n");
 
@@ -373,6 +424,7 @@ int main() {
     test_csv_reader_position();
     test_csv_reader_set_config();
     test_csv_reader_get_record_count();
+    test_csv_reader_bom();
     test_csv_reader_null_safety();
     printf("\n✅ All CSV Reader tests passed!\n");
     return 0;