Fix 10x support

cmd/count.go

@@ -63,6 +63,17 @@ var countFqCmd = &cobra.Command{
 				return err
 			}
 		}
+		mapfile, err := cmd.Flags().GetString("bc-list")
+		if err != nil {
+			return err
+		}
+		switch mapfile {
+		case "", "10x", "10X": // skip check
+		default:
+			if err := filecheck(mapfile); err != nil {
+				return err
+			}
+		}
 		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -83,7 +94,7 @@ func init() {
 	countXamCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes")
 	countXamCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 
-	convertFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
+	countFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
 	countFqCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes")
 	countFqCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 }

cmd/extract.go

@@ -96,7 +96,7 @@ func init() {
 	extractXamCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes")
 	extractXamCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 
-	convertFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
+	extractFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
 	extractFqCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes")
 	extractFqCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 }

cmd/ncbi.go

@@ -69,8 +69,12 @@ var ncbiFq2XamCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		if mapfile != "" && filecheck(mapfile) != nil {
-			return err
+		switch mapfile {
+		case "", "10x", "10X": // skip check
+		default:
+			if filecheck(mapfile) != nil {
+				return err
+			}
 		}
 		return nil
 	},

cmd/rm_unlinked.go

@@ -101,17 +101,6 @@ var unlinkedFqCmd = &cobra.Command{
 				return err
 			}
 		}
-		bcCount, err := cmd.Flags().GetString("bc-count")
-		if err != nil {
-			return err
-		}
-		if bcCount != "" {
-			err = ensureWritableDir(bcCount)
-			if err != nil {
-				return err
-			}
-		}
-
 		mapfile, err := cmd.Flags().GetString("bc-list")
 		if err != nil {
 			return err
@@ -149,7 +138,7 @@ func init() {
 
 	//TODO SINGLETONS SHOULD BE A FLAG, OUTPUT SHOULD BE PREFIX.SINGLE....
 	unlinkedFqCmd.Flags().BoolP("unlinked", "u", false, "Write valid singleton records to prefix.unlinked (optional)")
-	unlinkedFqCmd.Flags().BoolP("bc-count", "b", false, "Write valid barcode counts to prefix.counts (optional)")
+	unlinkedFqCmd.Flags().Bool("bc-count", false, "Write valid barcode counts to prefix.counts (optional)")
 	unlinkedFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
 	unlinkedFqCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 }

cmd/root.go

@@ -26,7 +26,7 @@ var samCmd = &cobra.Command{
 var fqCmd = &cobra.Command{
 	Use:   "fq",
 	Short: "FASTQ operations",
-	Long:  "Input can be one FASTQ file or file pair. If input is 10X data, provide a barcode 'whitelist' to --bc-map or use the embedded offical 10X whitelist with '--bc-map 10x'.",
+	Long:  "Input can be one FASTQ file or file pair. If input is 10X data, provide a barcode whitelist to --bc-list, or use the embedded official 10X whitelist with '--bc-list 10x'.",
 }
 
 func init() {

cmd/sort.go

@@ -81,7 +81,9 @@ var sortFqCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		if bcMap != "" {
+		switch bcMap {
+		case "", "10x", "10X": // skip check
+		default:
 			if err := filecheck(bcMap); err != nil {
 				return err
 			}

cmd/stats.go

@@ -89,9 +89,21 @@ var statsFqCmd = &cobra.Command{
 		}
 		switch strings.ToLower(platform) {
 		case "", stats.Haplotagging, stats.Stlfr, stats.Tellseq:
-			return nil
+		default:
+			return fmt.Errorf("unknown platform %q, expected haplotagging, stlfr or tellseq", platform)
 		}
-		return fmt.Errorf("unknown platform %q, expected haplotagging, stlfr or tellseq", platform)
+		mapfile, err := cmd.Flags().GetString("bc-list")
+		if err != nil {
+			return err
+		}
+		switch mapfile {
+		case "", "10x", "10X": // skip check
+		default:
+			if err := filecheck(mapfile); err != nil {
+				return err
+			}
+		}
+		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		platform, _ := cmd.Flags().GetString("platform")
@@ -111,7 +123,7 @@ func init() {
 	statsXamCmd.Flags().IntP("distance-threshold", "d", 50000, "Distance (bp) that splits a barcode's reads into separate molecules, where 0 skips deconvolution")
 	statsXamCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 
-	statsFqCmd.Flags().StringP("platform", "p", "", "Linked-read technology: haplotagging, stlfr or tellseq (default: detect from the first 100 reads)")
-	unlinkedFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
-	unlinkedFqCmd.Flags().IntP("threads", "@", 2, "Threads to use")
+	statsFqCmd.Flags().StringP("platform", "p", "", "Linked-read technology: haplotagging, stlfr or tellseq (default: detect from the first 100 reads). For 10X data, use --bc-list instead.")
+	statsFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]")
+	statsFqCmd.Flags().IntP("threads", "@", 2, "Threads to use")
 }

count/count_fq_test.go

@@ -1,7 +1,9 @@
 package count
 
 import (
+	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -22,7 +24,7 @@ func TestCountFQ_R1Only(t *testing.T) {
 	writeHapFastq(t, r1, reads)
 
 	out, err := captureStdout(t, func() error {
-		return CountFQ([]string{r1}, false)
+		return CountFQ([]string{r1}, false, "")
 	})
 	if err != nil {
 		t.Fatalf("CountFQ returned error: %v", err)
@@ -52,7 +54,7 @@ func TestCountFQ_IncludeInvalid(t *testing.T) {
 	writeHapFastq(t, r1, reads)
 
 	out, err := captureStdout(t, func() error {
-		return CountFQ([]string{r1}, true)
+		return CountFQ([]string{r1}, true, "")
 	})
 	if err != nil {
 		t.Fatalf("CountFQ returned error: %v", err)
@@ -84,7 +86,7 @@ func TestCountFQ_R1R2Union(t *testing.T) {
 	writeHapFastq(t, r2, r2Reads)
 
 	out, err := captureStdout(t, func() error {
-		return CountFQ([]string{r1, r2}, false)
+		return CountFQ([]string{r1, r2}, false, "")
 	})
 	if err != nil {
 		t.Fatalf("CountFQ returned error: %v", err)
@@ -109,7 +111,7 @@ func TestCountFQ_FewerThan100Records(t *testing.T) {
 	writeHapFastq(t, r1, []hapRead{{bc: "AAAA", vx: 1}})
 
 	_, err := captureStdout(t, func() error {
-		return CountFQ([]string{r1}, false)
+		return CountFQ([]string{r1}, false, "")
 	})
 	if err != nil {
 		t.Fatalf("expected no error for a fastq file with fewer than 100 records, got %v", err)
@@ -118,9 +120,84 @@ func TestCountFQ_FewerThan100Records(t *testing.T) {
 
 func TestCountFQ_MissingFile(t *testing.T) {
 	_, err := captureStdout(t, func() error {
-		return CountFQ([]string{"/nonexistent/path/does-not-exist.fq"}, false)
+		return CountFQ([]string{"/nonexistent/path/does-not-exist.fq"}, false, "")
 	})
 	if err == nil {
 		t.Fatal("expected an error opening a nonexistent file, got nil")
 	}
 }
+
+// writeTenXFastq writes a plain-text FASTQ file of 10X-format records: a
+// 16bp barcode inline as the sequence prefix, no separate tag.
+func writeTenXFastq(t *testing.T, path string, ids []string, barcode string) {
+	t.Helper()
+	var sb strings.Builder
+	for _, id := range ids {
+		seq := barcode + "ACGTACGTAC"
+		sb.WriteString("@" + id + "\n" + seq + "\n+\n" + strings.Repeat("I", len(seq)) + "\n")
+	}
+	if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func writeTenXWhitelist(t *testing.T, path string, barcodes ...string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(strings.Join(barcodes, "\n")+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCount10XFQ(t *testing.T) {
+	dir := t.TempDir()
+	r1 := filepath.Join(dir, "r1.fq")
+	wl := filepath.Join(dir, "whitelist.txt")
+
+	writeTenXWhitelist(t, wl, "AAAACCCCGGGGTTTT")
+	// two reads with the known barcode, one with an unrecognized prefix
+	writeTenXFastq(t, r1, []string{"read1", "read2"}, "AAAACCCCGGGGTTTT")
+	writeTenXFastq(t, filepath.Join(dir, "r1b.fq"), []string{"read3"}, "ZZZZZZZZZZZZZZZZ")
+	// append the unrecognized-barcode read to r1 so both are in one file
+	data, err := os.ReadFile(r1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	extra, err := os.ReadFile(filepath.Join(dir, "r1b.fq"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(r1, append(data, extra...), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out, err := captureStdout(t, func() error {
+		return CountFQ([]string{r1}, false, wl)
+	})
+	if err != nil {
+		t.Fatalf("CountFQ (10X): %v", err)
+	}
+	counts := parseCounts(t, out)
+	if counts["AAAACCCCGGGGTTTT"] != 2 {
+		t.Errorf("AAAACCCCGGGGTTTT count = %d, want 2", counts["AAAACCCCGGGGTTTT"])
+	}
+	if len(counts) != 1 {
+		t.Errorf("expected only the whitelisted barcode to be counted, got %v", counts)
+	}
+}
+
+func TestCount10XFQ_EmbeddedWhitelistSentinel(t *testing.T) {
+	dir := t.TempDir()
+	r1 := filepath.Join(dir, "r1.fq")
+	writeTenXFastq(t, r1, []string{"read1"}, "AAAACCCCGGGGTTTT")
+
+	// "10x"/"10X" means "use the embedded official whitelist" (fastq.Load10X),
+	// not a literal file named "10x" -- this used to be unreachable because
+	// the CLI's own Args validator ran filecheck("10x") before this point,
+	// but CountFQ/Count10XFQ themselves have always honored the sentinel.
+	_, err := captureStdout(t, func() error {
+		return CountFQ([]string{r1}, false, "10x")
+	})
+	if err != nil {
+		t.Fatalf("CountFQ with embedded whitelist sentinel: %v", err)
+	}
+}

docs/src/content/docs/fastq/convert.mdx

@@ -9,10 +9,13 @@ import { Badge } from '@astrojs/starlight/components';
 
 
 If you need to convert between the linked-read technologies' distinct FASTQ formats and their
-barcoding systems, use `convert`. It auto-detects the input format as one of 10X, haplotagging, TELLseq, or stLFR,
-and converts it to the format provided as the `TARGET` positional argument. If the
-input data is in 10X format, where the barcode is usually the first 16 bases in the R1 sequence,
-you will need to provide a `--bc-map` file that lets Djinn know what barcodes to look for.
+barcoding systems, use `convert`. It auto-detects the input format as one of haplotagging, TELLseq, or stLFR
+by inspecting the first 100 records. 10X cannot be auto-detected this way (its barcode is just the
+first 16 bases of the R1 sequence, with nothing marking it as a barcode), so converting **from** 10X
+requires `--bc-list` to tell Djinn which 16-base sequences are real barcodes: either a path to your
+own whitelist file, or the literal value `10x` to use the official 10X Genomics barcode whitelist
+embedded in Djinn. Converting **to** 10X draws new barcodes from that same embedded whitelist and
+needs no `--bc-list` at all.
 In all cases, a file will be created with the barcode conversion map.
 
 ## Usage
@@ -36,11 +39,11 @@ convert 10x gcurratum_as10X gcurratum.R1.fq.gz gcurratum.R2.fq.gz
 | `PREFIX`         |  output filename prefix   <Badge text="required" variant="tip" />    |
 | `TARGET`         | output barcode and FASTQ style: `10x`, `haplotagging`, `tellseq`, `stlfr`, `standard` |
 | `file.R*.fq.gz`  | FASTQ file or file pair <Badge text="required" variant="tip" />                                       |
-| `-b` `--bc-map`  | Valid barcode list <Badge text="required: for 10X Genomics input" variant="tip" />   |
+| `-b` `--bc-list`  | Valid barcode list (or literal `10x` for the embedded whitelist) <Badge text="required: for 10X Genomics input" variant="tip" />   |
 
 
 <details>
-    <summary> example `--bc-map` file </summary>
+    <summary> example `--bc-list` file </summary>
 ```
 ATGGAAGCCGTAGTTA
 ACGGAAGCCGTAGTTC

docs/src/content/docs/fastq/count.mdx

@@ -13,6 +13,12 @@ You can use one file of a pair (e.g. R1 or R2), but including both files will pr
 and only count occurences of _new_ barcodes in the second file that aren't present in the first file. This is done
 to avoid double-counting barcodes.
 
+Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data
+cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking
+it as a barcode), so counting 10X barcodes requires `--bc-list`: either a path to your own
+whitelist, or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn.
+For 10X input, only the first file (R1) is read, since that's the only place the barcode lives.
+
 
 ## Usage
 ```bash
@@ -29,3 +35,4 @@ djinn fastq count sample1.F.fq.gz sample1.R.fq.gz > sample1.bxcount
 |:------------------|:------------------------------------------------------------------------------|
 | `file.R*.fq.gz`   | FASTQ file or file pair <Badge text="required" variant="tip" />                          |
 | `-i` `--invalid`  | Include invalid barcodes                                                      |
+| `-b` `--bc-list`  | Valid barcode list, or literal `10x` for the embedded official whitelist <Badge text="required: for 10X Genomics input" variant="tip" /> |

docs/src/content/docs/fastq/extract.mdx

@@ -8,6 +8,12 @@ import { Badge } from '@astrojs/starlight/components';
 
 To get a list of all the unique barcodes in your data, use `extract`. Writes to `stdout`.
 
+Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data
+cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking
+it as a barcode), so extracting 10X barcodes requires `--bc-list`: either a path to your own
+whitelist, or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn.
+For 10X input, only the first file (R1) is read, since that's the only place the barcode lives.
+
 ## Usage
 ```bash
 djinn fastq extract file.R1.fq.gz [file.R2.fq.gz]
@@ -23,3 +29,4 @@ djinn fastq extract -i sample1.F.fq.gz sample1.R.fq.gz > sample1.bc
 |:------------------|:------------------------------------------------------------------------------|
 | `file.R*fq.gz`    | FASTQ file or file pair <Badge text="required" variant="tip" />                          |
 | `-i` `--invalid`  | Include invalid barcodes                                                      |
+| `-b` `--bc-list`  | Valid barcode list, or literal `10x` for the embedded official whitelist <Badge text="required: for 10X Genomics input" variant="tip" /> |

docs/src/content/docs/fastq/ncbi.mdx

@@ -23,6 +23,12 @@ files into a single unaligned BAM file without any data loss (and also free up d
     you risk losing the barcode forever!
 </Aside>
 
+Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X cannot be
+auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking it as a
+barcode), so converting 10X FASTQ to BAM requires `--bc-list` to tell Djinn which 16-base sequences
+are real barcodes -- either a path to your own whitelist, or the literal value `10x` to use the
+official 10X Genomics whitelist embedded in Djinn.
+
 
 ## Usage
 ```bash
@@ -43,5 +49,5 @@ samtools import -O BAM -T "*" -1 file.R1.fq file.R2.fq > out.bam
 | argument         | description                                                                |
 |:-----------------|:---------------------------------------------------------------------------|
 | `file.R*.fq.gz`  | FASTQ file or file pair <Badge text="required" variant="tip" />            |
-| `-b` `--bc-map`  | Valid barcode list <Badge text="required: for 10X Genomics input" variant="tip" />                   |
+| `-b` `--bc-list`  | Valid barcode list, or literal `10x` for the embedded official whitelist <Badge text="required: for 10X Genomics input" variant="tip" />                   |
 | `-t` `--threads` | Threads to use (default: 2)                                      |

docs/src/content/docs/fastq/rm-invalid.mdx

@@ -10,6 +10,13 @@ You may be interested in retaining only "useful" linked reads, that is, reads wi
 Using `rm-invalid` will remove alignment records invalid barcodes. If you want to keep a separate
 file (or files) of records with invalid barcodes use the `--invalid` flag.
 
+Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data
+cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking
+it as a barcode), so filtering 10X data requires `--bc-list`: either a path to your own whitelist,
+or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn. A 10X R1
+read whose first 16 bases don't match any whitelist entry has no barcode at all and is treated as
+invalid.
+
 ## Usage
 ```bash
 djinn sam rm-invalid [--invalid] PREFIX file.R1.fq.gz [file.R2.fq.gz]
@@ -26,3 +33,4 @@ djinn sam rm-invalid --invalid echinda.valid echidna.R*.fq.gz
 | `PREFIX`          |  output filename prefix   <Badge text="required" variant="tip" />    |
 | `file.R*.fq.gz`   | FASTQ file or file pair <Badge text="required" variant="tip" />                                       |
 | `-i` `--invalid` | output records with invalid barcodes to `PREFIX.invalid.*`                                 |
+| `-b` `--bc-list`  | Valid barcode list, or literal `10x` for the embedded official whitelist <Badge text="required: for 10X Genomics input" variant="tip" /> |

docs/src/content/docs/fastq/rm-unlinked.mdx

@@ -12,6 +12,11 @@ to isolate records with meaningful linked-read information. If you want to keep
 file (or files) of unlinked/singleton records, use the `--unlinked` flag. Barcodes are counted
 per read-pair or unpair read.
 
+Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data
+cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking
+it as a barcode), so filtering 10X data requires `--bc-list`: either a path to your own whitelist,
+or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn.
+
 ## Usage
 
 ```bash
@@ -28,5 +33,6 @@ djinn fastq rm-unlinked --unlinked hyena.clean hyena.R*.fq.gz
 | :------------------ | :--------------------------------------------------------------------------------- |
 | `PREFIX`          |  output filename prefix   <Badge text="required" variant="tip" />    |
 | `file.R*.fq.gz`             | FASTQ file or file pair <Badge text="required" variant="tip" />                               |
-| `-b` `--bc-count`   | write valid barcode counts to this file <Badge text="optional" variant="note" />    |
+| `--bc-count`   | write valid barcode counts to `PREFIX.count` <Badge text="optional" variant="note" />    |
 | `-u` `--unlinked`   | write valid singleton records to `PREFIX.unlinked`                                |
+| `-b` `--bc-list`  | Valid barcode list, or literal `10x` for the embedded official whitelist <Badge text="required: for 10X Genomics input" variant="tip" /> |