Fix key derivation for escaped encrypt strings; fail on undecryptable PDFs

CHANGELOG.md

@@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Encrypted PDFs whose `/O`, `/U` or `/ID` strings are written as literal
+  strings with escapes (`\n`, `\r`, `\ddd` octal, line continuations) now
+  decrypt. Each escape was read as the escaped letter, so ffpdf derived the
+  wrong file key and `fields` reported `count: 0` with exit 0 (#14). Literal
+  strings inside encrypted objects are now decoded the same spec-compliant way.
+- An encrypted PDF that cannot be decrypted is now an error instead of a silent
+  empty or garbled result. This covers files that need a password, use an
+  unsupported security handler, or derive a key that fails `/U` authentication.
+  `fields`, `fdf-extract`, `xfdf-extract` and `fill` exit 1 with the reason on
+  stderr and write nothing to stdout.
 - Encrypted forms: field names stored outside object streams no longer read as
   `????`. Decrypted strings are re-emitted as hex, and hex field names were
   always decoded as UTF-16BE even without the FE FF byte-order mark. `fields`

crypto.c

@@ -391,8 +391,8 @@ static const unsigned char PDF_PAD[32] = {
     0x28,0xBF,0x4E,0x5E,0x4E,0x75,0x8A,0x41,0x64,0x00,0x4E,0x56,0xFF,0xFA,0x01,0x08,
     0x2E,0x2E,0x00,0xB6,0xD0,0x68,0x3E,0x80,0x2F,0x0C,0xA9,0xFE,0x64,0x53,0x69,0x7A };
 
-// Parse a hex string "<...>" starting at *p (which points just past '<') into
-// buf; returns byte count. Also handles literal "(...)" if p points at '('.
+// Parse the hex "<...>" or literal "(...)" string at `v` into buf; returns the
+// byte count (at most `cap`).
 static int parse_pdf_bytes(const char *v, unsigned char *buf, int cap) {
     if (*v == '<') {
         v++;
@@ -410,13 +410,9 @@ static int parse_pdf_bytes(const char *v, unsigned char *buf, int cap) {
         return n;
     }
     if (*v == '(') {                                  // literal string with escapes
-        v++;
-        int n = 0;
-        while (*v && *v != ')' && n < cap) {
-            if (*v == '\\' && v[1]) { v++; buf[n++] = (unsigned char)*v++; continue; }
-            buf[n++] = (unsigned char)*v++;
-        }
-        return n;
+        size_t n;
+        pdf_literal_bytes(v, buf, (size_t)cap, &n);
+        return (int)n;
     }
     return 0;
 }
@@ -486,7 +482,6 @@ int pdf_crypt_init(PdfCrypt *c, const char *enc, const unsigned char *id0, size_
     if (!po || !pu) return 0;
     int olen = parse_pdf_bytes(po, O, sizeof(O));
     int ulen = parse_pdf_bytes(pu, U, sizeof(U));
-    (void)ulen;
 
     if (c->R >= 5) {
         // R6 (AES-256): validate empty user password and unwrap the file key.
@@ -534,7 +529,28 @@ int pdf_crypt_init(PdfCrypt *c, const char *enc, const unsigned char *id0, size_
     md5(buf, n, h);
     if (c->R >= 3) for (int i = 0; i < 50; i++) md5(h, c->key_len, h);
     memcpy(c->key, h, c->key_len);
-    return 1;
+
+    // Authenticate the empty user password by recomputing /U from the derived
+    // key (Algorithm 4 for R2, Algorithm 5 for R3/R4). A mismatch means the key
+    // is wrong: the document needs a password, or /O, /P or /ID was misread.
+    // Accepting it anyway would decrypt every string and stream to garbage.
+    unsigned char ucheck[32];
+    if (c->R < 3) {
+        rc4(c->key, c->key_len, PDF_PAD, 32, ucheck);
+        return ulen >= 32 && memcmp(ucheck, U, 32) == 0;
+    }
+    unsigned char uin[32 + 64];
+    size_t il = id0_len < 64 ? id0_len : 64;
+    memcpy(uin, PDF_PAD, 32);
+    memcpy(uin + 32, id0, il);
+    md5(uin, 32 + il, h);
+    rc4(c->key, c->key_len, h, 16, ucheck);
+    for (int i = 1; i <= 19; i++) {
+        unsigned char k2[16];
+        for (int j = 0; j < c->key_len; j++) k2[j] = c->key[j] ^ (unsigned char)i;
+        rc4(k2, c->key_len, ucheck, 16, ucheck);
+    }
+    return ulen >= 16 && memcmp(ucheck, U, 16) == 0;
 }
 
 // Derive the per-object key (Algorithm 1) for RC4/AESV2 into `ok`, returns len.
@@ -602,13 +618,7 @@ char *pdf_decrypt_dict_strings(const PdfCrypt *c, int num, int gen, const char *
                 if (hi >= 0) ct[cl++] = (unsigned char)(hi*16);
                 if (*q == '>') q++;
             } else {
-                int sdepth = 1;
-                while (*q && sdepth > 0) {
-                    if (*q == '\\' && q[1]) { q++; ct[cl++] = (unsigned char)*q++; continue; }
-                    if (*q == '(') { sdepth++; ct[cl++] = '('; q++; continue; }
-                    if (*q == ')') { sdepth--; if (sdepth == 0) { q++; break; } ct[cl++] = ')'; q++; continue; }
-                    ct[cl++] = (unsigned char)*q++;
-                }
+                q = pdf_literal_bytes(p, ct, len + 1, &cl);
             }
             unsigned char *pt = malloc(cl + 1);
             int pl = pdf_decrypt(c, num, gen, ct, cl, pt);
@@ -667,13 +677,7 @@ char *pdf_encrypt_dict_strings(const PdfCrypt *c, int num, int gen, const char *
                 if (hi >= 0) pt[pl++] = (unsigned char)(hi*16);
                 if (*q == '>') q++;
             } else {
-                int sdepth = 1;
-                while (*q && sdepth > 0) {
-                    if (*q == '\\' && q[1]) { q++; pt[pl++] = (unsigned char)*q++; continue; }
-                    if (*q == '(') { sdepth++; pt[pl++] = '('; q++; continue; }
-                    if (*q == ')') { sdepth--; if (sdepth == 0) { q++; break; } pt[pl++] = ')'; q++; continue; }
-                    pt[pl++] = (unsigned char)*q++;
-                }
+                q = pdf_literal_bytes(p, pt, len + 1, &pl);
             }
             unsigned char *ct = malloc(pl + 40);
             int cl = pdf_encrypt(c, num, gen, pt, pl, ct, pl + 40);

ffpdf.1

@@ -250,8 +250,10 @@ Maximum total decompressed bytes over one run (default 1 GB).
 Success.
 .TP
 .B 1
-An error occurred: invalid usage, a file that could not be opened, or a PDF
-whose cross\-reference table could not be parsed.
+An error occurred: invalid usage, a file that could not be opened, a PDF
+whose cross\-reference table could not be parsed, or an encrypted PDF that
+cannot be decrypted (it needs a password, or uses an unsupported security
+handler).
 .TP
 .B 2
 .B fill

field_map.c

@@ -528,23 +528,25 @@ int pdf_doc_id0(unsigned char *out, int cap) {
 
 // Detect /Encrypt in the trailer / xref-stream dict and, for the empty user
 // password, initialize the document crypt handler so subsequent object reads
-// decrypt. No-op (and clears any previous handler) when the document is not
-// encrypted or the handler is unsupported.
-static void pdf_crypt_setup(FILE *f, XRefTable *xref) {
+// decrypt. Returns 0 when the document is not encrypted or the handler is
+// ready. Returns -1 (after printing why) when /Encrypt is present but unusable:
+// every read would yield ciphertext, so callers must stop rather than report an
+// empty or garbled form as if it were real.
+static int pdf_crypt_setup(FILE *f, XRefTable *xref) {
     g_doc.encrypted = 0;
     long sx = find_startxref(f);
-    if (sx < 0) return;
+    if (sx < 0) return 0;
     fseek(f, 0, SEEK_END);
     long end = ftell(f), span = end - sx;
-    if (span <= 0) return;
+    if (span <= 0) return 0;
     char *buf = malloc((size_t)span + 1);
-    if (!buf) return;
+    if (!buf) return 0;
     fseek(f, sx, SEEK_SET);
     size_t got = fread(buf, 1, (size_t)span, f);
     buf[got] = '\0';
 
     const char *ep = strstr(buf, "/Encrypt");
-    if (!ep) { free(buf); return; }
+    if (!ep) { free(buf); return 0; }
     int enc_obj = parse_ref_num(ep + 8);
 
     // First element of the /ID array (part of the R2-4 key derivation).
@@ -568,39 +570,41 @@ static void pdf_crypt_setup(FILE *f, XRefTable *xref) {
             }
             if (hi >= 0 && id0_len < (int)sizeof(id0)) id0[id0_len++] = (unsigned char)(hi*16);
         } else if (*b == '(') {
-            b++;
-            while (*b && *b != ')' && id0_len < (int)sizeof(id0)) {
-                if (*b == '\\' && b[1]) { b++; id0[id0_len++] = (unsigned char)*b++; continue; }
-                id0[id0_len++] = (unsigned char)*b++;
-            }
+            size_t n;
+            pdf_literal_bytes(b, id0, sizeof(id0), &n);
+            id0_len = (int)n;
         }
     }
     free(buf);
-    if (enc_obj <= 0) return;
 
     // Read the /Encrypt dictionary raw -- the handler is not active yet, so its
     // /O /U /Perms strings (which are never encrypted) are read verbatim.
-    char *ed = get_object_raw(f, xref, enc_obj, NULL);
-    if (ed) {
-        if (pdf_crypt_init(&g_doc.crypt, ed, id0, id0_len)) {
-            g_doc.encrypted = 1;
-            g_doc.enc_obj = enc_obj;
-            g_doc.enc_id0_len = id0_len < (int)sizeof(g_doc.enc_id0) ? id0_len : (int)sizeof(g_doc.enc_id0);
-            memcpy(g_doc.enc_id0, id0, g_doc.enc_id0_len);
-            fprintf(stderr, "Encrypted document: standard handler V%d R%d, %s (empty user password)\n",
-                    g_doc.crypt.V, g_doc.crypt.R,
-                    g_doc.crypt.cfm == CFM_RC4 ? "RC4" : g_doc.crypt.cfm == CFM_AESV2 ? "AES-128" : "AES-256");
-        } else {
-            fprintf(stderr, "WARNING: /Encrypt present but unsupported or non-empty password; "
-                            "reads will fail\n");
-        }
-        free(ed);
+    char *ed = enc_obj > 0 ? get_object_raw(f, xref, enc_obj, NULL) : NULL;
+    if (!ed) {
+        fprintf(stderr, "ERROR: encrypted document, but its /Encrypt dictionary could not be read\n");
+        return -1;
     }
+    int ok = pdf_crypt_init(&g_doc.crypt, ed, id0, id0_len);
+    free(ed);
+    if (!ok) {
+        fprintf(stderr, "ERROR: encrypted document (V%d R%d) cannot be decrypted: it needs a "
+                        "password or uses an unsupported security handler (only the empty "
+                        "user password is supported)\n", g_doc.crypt.V, g_doc.crypt.R);
+        return -1;
+    }
+    g_doc.encrypted = 1;
+    g_doc.enc_obj = enc_obj;
+    g_doc.enc_id0_len = id0_len < (int)sizeof(g_doc.enc_id0) ? id0_len : (int)sizeof(g_doc.enc_id0);
+    memcpy(g_doc.enc_id0, id0, g_doc.enc_id0_len);
+    fprintf(stderr, "Encrypted document: standard handler V%d R%d, %s (empty user password)\n",
+            g_doc.crypt.V, g_doc.crypt.R,
+            g_doc.crypt.cfm == CFM_RC4 ? "RC4" : g_doc.crypt.cfm == CFM_AESV2 ? "AES-128" : "AES-256");
+    return 0;
 }
 
 int build_field_map(FILE *f, XRefTable *xref, int root_obj, FieldMap *map) {
     decompress_reset();       // fresh per-run decompression budget
-    pdf_crypt_setup(f, xref);
+    if (pdf_crypt_setup(f, xref) < 0) return -1;   // encrypted but undecryptable
     if (root_obj <= 0) root_obj = find_root_obj(f);
     if (root_obj <= 0) return 0;
 

field_map.h

@@ -38,8 +38,10 @@ typedef struct {
 // Walk the AcroForm field tree, recording every named node. EXTENDS support must
 // already be initialised by the caller (init_form_extractor_extends_support).
 // `root_obj` is the /Root (catalog) object number, or <=0 to have it resolved
-// from the trailer. Returns the AcroForm object number (>0) on success, or 0 if
-// no AcroForm/Fields were found. Populates `map` (free with field_map_free).
+// from the trailer. Returns the AcroForm object number (>0) on success, 0 if no
+// AcroForm/Fields were found, or -1 if the document is encrypted but cannot be
+// decrypted (the reason is printed to stderr and nothing is walked). Populates
+// `map` (free with field_map_free).
 int build_field_map(FILE *f, XRefTable *xref, int root_obj, FieldMap *map);
 void field_map_free(FieldMap *map);
 

form_extractor.c

@@ -57,9 +57,10 @@ static void print_xml_attr(const char *name) {
     }
 }
 
-void extract_form_fields_xfdf(FILE *f, const XRefTable *xref_table) {
+int extract_form_fields_xfdf(FILE *f, const XRefTable *xref_table) {
     FieldMap map = {0};
     int acroform = build_field_map(f, (XRefTable *)xref_table, 0, &map);
+    if (acroform < 0) { objstm_cache_reset(); return 1; }   // reason on stderr
 
     printf("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
     printf("<xfdf xmlns=\"http://ns.adobe.com/xfdf/\" xml:space=\"preserve\">\n");
@@ -96,11 +97,15 @@ void extract_form_fields_xfdf(FILE *f, const XRefTable *xref_table) {
 
     field_map_free(&map);
     objstm_cache_reset();
+    return 0;
 }
 
-void extract_form_fields_fdf(FILE *f, const XRefTable *xref_table) {
+int extract_form_fields_fdf(FILE *f, const XRefTable *xref_table) {
     FieldMap map = {0};
-    build_field_map(f, (XRefTable *)xref_table, 0, &map);
+    if (build_field_map(f, (XRefTable *)xref_table, 0, &map) < 0) {
+        objstm_cache_reset();
+        return 1;                                   // reason on stderr
+    }
 
     printf("%%FDF-1.2\n1 0 obj\n<<\n/FDF\n<<\n/Fields [\n");
     for (int i = 0; i < map.count; i++) {
@@ -127,6 +132,7 @@ void extract_form_fields_fdf(FILE *f, const XRefTable *xref_table) {
 
     field_map_free(&map);
     objstm_cache_reset();
+    return 0;
 }
 
 /* ==========================================================================
@@ -317,9 +323,12 @@ static void map_annots_to_pages(FILE *f, XRefTable *xref, int node, int depth,
     free(d);
 }
 
-void extract_form_fields_json(FILE *f, const XRefTable *xref_table) {
+int extract_form_fields_json(FILE *f, const XRefTable *xref_table) {
     FieldMap map = {0};
-    build_field_map(f, (XRefTable *)xref_table, 0, &map);
+    if (build_field_map(f, (XRefTable *)xref_table, 0, &map) < 0) {
+        objstm_cache_reset();
+        return 1;                                   // reason on stderr
+    }
 
     // Document-level facts an agent needs to interpret the field list: whether
     // the form is XFA (AcroForm /XFA) and, if so, whether it is dynamic
@@ -449,4 +458,5 @@ void extract_form_fields_json(FILE *f, const XRefTable *xref_table) {
     free(pmap);
     field_map_free(&map);
     objstm_cache_reset();
+    return 0;
 }

form_extractor.h

@@ -18,12 +18,16 @@
 
 // Extract the form's fillable fields to stdout, as FDF or XFDF. Both are thin
 // presentation layers over build_field_map() (see field_map.h).
-void extract_form_fields_fdf(FILE *f, const XRefTable *xref_table);
-void extract_form_fields_xfdf(FILE *f, const XRefTable *xref_table);
+//
+// All three extractors return 0 on success, or 1 when the document cannot be
+// read (an encrypted PDF that cannot be decrypted); the reason goes to stderr
+// and nothing is written to stdout.
+int extract_form_fields_fdf(FILE *f, const XRefTable *xref_table);
+int extract_form_fields_xfdf(FILE *f, const XRefTable *xref_table);
 
 // The `fields` command: list every fillable field as JSON (name, type, current
 // value, and per-type extras: a choice field's options and combo/multi-select
 // flags, a checkbox's on-state). Designed for programmatic callers.
-void extract_form_fields_json(FILE *f, const XRefTable *xref_table);
+int extract_form_fields_json(FILE *f, const XRefTable *xref_table);
 
 #endif // FORM_EXTRACTOR_H

main.c

@@ -175,16 +175,17 @@ static int cmd_extract(int mode, const char *path) {
         return 1;
     }
 
+    int rc;
     if (mode == 2)
-        extract_form_fields_json(f, &xref_table);
+        rc = extract_form_fields_json(f, &xref_table);
     else if (mode == 1)
-        extract_form_fields_xfdf(f, &xref_table);
+        rc = extract_form_fields_xfdf(f, &xref_table);
     else
-        extract_form_fields_fdf(f, &xref_table);
+        rc = extract_form_fields_fdf(f, &xref_table);
 
     xref_free(&xref_table);
     fclose(f);
-    return 0;
+    return rc;
 }
 
 // Run `fill`: resolve the stdin-FDF and -o conventions, then delegate to

pdf_filler.c

@@ -1784,7 +1784,8 @@ static int fill_core(const char *pdf_filename, FdfData *fdf,
     // are resolved lazily by get_object_raw (only the needed streams are ever
     // decompressed); objstm_cache_reset() frees that cache at cleanup.
     int acroform = build_field_map(f, &xref, root, &map);
-    if (acroform <= 0) {
+    if (acroform < 0) goto cleanup;                   // undecryptable; reason on stderr
+    if (acroform == 0) {
         fprintf(stderr, "ERROR: no AcroForm found in document catalog\n");
         goto cleanup;
     }

pdf_lex.c

@@ -209,6 +209,46 @@ const char *read_literal_alloc(const char *p, char **out) {
     return p;
 }
 
+const char *pdf_literal_bytes(const char *p, unsigned char *out, size_t cap, size_t *len) {
+    size_t n = 0;
+    int depth = 1;
+#define PUT(b) do { if (n < cap) out[n++] = (unsigned char)(b); } while (0)
+    p++;   // skip '('
+    while (*p) {
+        unsigned char c = (unsigned char)*p++;
+        if (c == '\\') {
+            if (!*p) break;
+            c = (unsigned char)*p++;
+            switch (c) {
+                case 'n': PUT('\n'); break;
+                case 'r': PUT('\r'); break;
+                case 't': PUT('\t'); break;
+                case 'b': PUT('\b'); break;
+                case 'f': PUT('\f'); break;
+                case '\r': if (*p == '\n') p++; break;   // line continuation: no byte
+                case '\n': break;
+                default:
+                    if (c >= '0' && c <= '7') {           // \d, \dd or \ddd octal
+                        int v = c - '0';
+                        for (int i = 0; i < 2 && *p >= '0' && *p <= '7'; i++)
+                            v = v * 8 + (*p++ - '0');
+                        PUT(v & 0xff);                    // high-order overflow ignored
+                    } else {
+                        PUT(c);                           // \( \) \\ and unknown escapes
+                    }
+            }
+            continue;
+        }
+        if (c == '(') depth++;
+        else if (c == ')' && --depth == 0) break;
+        else if (c == '\r') { if (*p == '\n') p++; c = '\n'; }   // bare EOL reads as LF
+        PUT(c);
+    }
+#undef PUT
+    *len = n;
+    return p;
+}
+
 char *extract_dict_inner_alloc(const char *obj_dict) {
     const char *open = strstr(obj_dict, "<<");
     if (!open) return NULL;