diff --git a/cmd/count.go b/cmd/count.go index 0d0d6e5..6a7901d 100644 --- a/cmd/count.go +++ b/cmd/count.go @@ -63,6 +63,17 @@ var countFqCmd = &cobra.Command{ return err } } + mapfile, err := cmd.Flags().GetString("bc-list") + if err != nil { + return err + } + switch mapfile { + case "", "10x", "10X": // skip check + default: + if err := filecheck(mapfile); err != nil { + return err + } + } return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -83,7 +94,7 @@ func init() { countXamCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes") countXamCmd.Flags().IntP("threads", "@", 2, "Threads to use") - convertFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") + countFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") countFqCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes") countFqCmd.Flags().IntP("threads", "@", 2, "Threads to use") } diff --git a/cmd/extract.go b/cmd/extract.go index 9b25bff..cddf353 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -96,7 +96,7 @@ func init() { extractXamCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes") extractXamCmd.Flags().IntP("threads", "@", 2, "Threads to use") - convertFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") + extractFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") extractFqCmd.Flags().BoolP("invalid", "i", false, "Include invalid barcodes") extractFqCmd.Flags().IntP("threads", "@", 2, "Threads to use") } diff --git a/cmd/ncbi.go b/cmd/ncbi.go index 82ec4a6..f6c24ca 100644 --- a/cmd/ncbi.go +++ b/cmd/ncbi.go @@ -69,8 +69,12 @@ var ncbiFq2XamCmd = &cobra.Command{ if err != nil { return err } - if mapfile != "" && filecheck(mapfile) != nil { - return err + switch mapfile { + case "", "10x", "10X": // skip check + default: + if filecheck(mapfile) != nil { + return err + } } return nil }, diff --git a/cmd/rm_unlinked.go b/cmd/rm_unlinked.go index 38a82f3..faccc71 100644 --- a/cmd/rm_unlinked.go +++ b/cmd/rm_unlinked.go @@ -101,17 +101,6 @@ var unlinkedFqCmd = &cobra.Command{ return err } } - bcCount, err := cmd.Flags().GetString("bc-count") - if err != nil { - return err - } - if bcCount != "" { - err = ensureWritableDir(bcCount) - if err != nil { - return err - } - } - mapfile, err := cmd.Flags().GetString("bc-list") if err != nil { return err @@ -149,7 +138,7 @@ func init() { //TODO SINGLETONS SHOULD BE A FLAG, OUTPUT SHOULD BE PREFIX.SINGLE.... unlinkedFqCmd.Flags().BoolP("unlinked", "u", false, "Write valid singleton records to prefix.unlinked (optional)") - unlinkedFqCmd.Flags().BoolP("bc-count", "b", false, "Write valid barcode counts to prefix.counts (optional)") + unlinkedFqCmd.Flags().Bool("bc-count", false, "Write valid barcode counts to prefix.counts (optional)") unlinkedFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") unlinkedFqCmd.Flags().IntP("threads", "@", 2, "Threads to use") } diff --git a/cmd/root.go b/cmd/root.go index 7407045..ac2bfdd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,7 +26,7 @@ var samCmd = &cobra.Command{ var fqCmd = &cobra.Command{ Use: "fq", Short: "FASTQ operations", - Long: "Input can be one FASTQ file or file pair. If input is 10X data, provide a barcode 'whitelist' to --bc-map or use the embedded offical 10X whitelist with '--bc-map 10x'.", + Long: "Input can be one FASTQ file or file pair. If input is 10X data, provide a barcode whitelist to --bc-list, or use the embedded official 10X whitelist with '--bc-list 10x'.", } func init() { diff --git a/cmd/sort.go b/cmd/sort.go index 23c90c4..d211030 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -81,7 +81,9 @@ var sortFqCmd = &cobra.Command{ if err != nil { return err } - if bcMap != "" { + switch bcMap { + case "", "10x", "10X": // skip check + default: if err := filecheck(bcMap); err != nil { return err } diff --git a/cmd/stats.go b/cmd/stats.go index d55869d..9a61e62 100644 --- a/cmd/stats.go +++ b/cmd/stats.go @@ -89,9 +89,21 @@ var statsFqCmd = &cobra.Command{ } switch strings.ToLower(platform) { case "", stats.Haplotagging, stats.Stlfr, stats.Tellseq: - return nil + default: + return fmt.Errorf("unknown platform %q, expected haplotagging, stlfr or tellseq", platform) } - return fmt.Errorf("unknown platform %q, expected haplotagging, stlfr or tellseq", platform) + mapfile, err := cmd.Flags().GetString("bc-list") + if err != nil { + return err + } + switch mapfile { + case "", "10x", "10X": // skip check + default: + if err := filecheck(mapfile); err != nil { + return err + } + } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { platform, _ := cmd.Flags().GetString("platform") @@ -111,7 +123,7 @@ func init() { statsXamCmd.Flags().IntP("distance-threshold", "d", 50000, "Distance (bp) that splits a barcode's reads into separate molecules, where 0 skips deconvolution") statsXamCmd.Flags().IntP("threads", "@", 2, "Threads to use") - statsFqCmd.Flags().StringP("platform", "p", "", "Linked-read technology: haplotagging, stlfr or tellseq (default: detect from the first 100 reads)") - unlinkedFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") - unlinkedFqCmd.Flags().IntP("threads", "@", 2, "Threads to use") + statsFqCmd.Flags().StringP("platform", "p", "", "Linked-read technology: haplotagging, stlfr or tellseq (default: detect from the first 100 reads). For 10X data, use --bc-list instead.") + statsFqCmd.Flags().StringP("bc-list", "b", "", "Valid barcode list [if 10X input]") + statsFqCmd.Flags().IntP("threads", "@", 2, "Threads to use") } diff --git a/count/count_fq_test.go b/count/count_fq_test.go index e761175..4fbf110 100644 --- a/count/count_fq_test.go +++ b/count/count_fq_test.go @@ -1,7 +1,9 @@ package count import ( + "os" "path/filepath" + "strings" "testing" ) @@ -22,7 +24,7 @@ func TestCountFQ_R1Only(t *testing.T) { writeHapFastq(t, r1, reads) out, err := captureStdout(t, func() error { - return CountFQ([]string{r1}, false) + return CountFQ([]string{r1}, false, "") }) if err != nil { t.Fatalf("CountFQ returned error: %v", err) @@ -52,7 +54,7 @@ func TestCountFQ_IncludeInvalid(t *testing.T) { writeHapFastq(t, r1, reads) out, err := captureStdout(t, func() error { - return CountFQ([]string{r1}, true) + return CountFQ([]string{r1}, true, "") }) if err != nil { t.Fatalf("CountFQ returned error: %v", err) @@ -84,7 +86,7 @@ func TestCountFQ_R1R2Union(t *testing.T) { writeHapFastq(t, r2, r2Reads) out, err := captureStdout(t, func() error { - return CountFQ([]string{r1, r2}, false) + return CountFQ([]string{r1, r2}, false, "") }) if err != nil { t.Fatalf("CountFQ returned error: %v", err) @@ -109,7 +111,7 @@ func TestCountFQ_FewerThan100Records(t *testing.T) { writeHapFastq(t, r1, []hapRead{{bc: "AAAA", vx: 1}}) _, err := captureStdout(t, func() error { - return CountFQ([]string{r1}, false) + return CountFQ([]string{r1}, false, "") }) if err != nil { t.Fatalf("expected no error for a fastq file with fewer than 100 records, got %v", err) @@ -118,9 +120,84 @@ func TestCountFQ_FewerThan100Records(t *testing.T) { func TestCountFQ_MissingFile(t *testing.T) { _, err := captureStdout(t, func() error { - return CountFQ([]string{"/nonexistent/path/does-not-exist.fq"}, false) + return CountFQ([]string{"/nonexistent/path/does-not-exist.fq"}, false, "") }) if err == nil { t.Fatal("expected an error opening a nonexistent file, got nil") } } + +// writeTenXFastq writes a plain-text FASTQ file of 10X-format records: a +// 16bp barcode inline as the sequence prefix, no separate tag. +func writeTenXFastq(t *testing.T, path string, ids []string, barcode string) { + t.Helper() + var sb strings.Builder + for _, id := range ids { + seq := barcode + "ACGTACGTAC" + sb.WriteString("@" + id + "\n" + seq + "\n+\n" + strings.Repeat("I", len(seq)) + "\n") + } + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } +} + +func writeTenXWhitelist(t *testing.T, path string, barcodes ...string) { + t.Helper() + if err := os.WriteFile(path, []byte(strings.Join(barcodes, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestCount10XFQ(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + wl := filepath.Join(dir, "whitelist.txt") + + writeTenXWhitelist(t, wl, "AAAACCCCGGGGTTTT") + // two reads with the known barcode, one with an unrecognized prefix + writeTenXFastq(t, r1, []string{"read1", "read2"}, "AAAACCCCGGGGTTTT") + writeTenXFastq(t, filepath.Join(dir, "r1b.fq"), []string{"read3"}, "ZZZZZZZZZZZZZZZZ") + // append the unrecognized-barcode read to r1 so both are in one file + data, err := os.ReadFile(r1) + if err != nil { + t.Fatal(err) + } + extra, err := os.ReadFile(filepath.Join(dir, "r1b.fq")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(r1, append(data, extra...), 0o644); err != nil { + t.Fatal(err) + } + + out, err := captureStdout(t, func() error { + return CountFQ([]string{r1}, false, wl) + }) + if err != nil { + t.Fatalf("CountFQ (10X): %v", err) + } + counts := parseCounts(t, out) + if counts["AAAACCCCGGGGTTTT"] != 2 { + t.Errorf("AAAACCCCGGGGTTTT count = %d, want 2", counts["AAAACCCCGGGGTTTT"]) + } + if len(counts) != 1 { + t.Errorf("expected only the whitelisted barcode to be counted, got %v", counts) + } +} + +func TestCount10XFQ_EmbeddedWhitelistSentinel(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + writeTenXFastq(t, r1, []string{"read1"}, "AAAACCCCGGGGTTTT") + + // "10x"/"10X" means "use the embedded official whitelist" (fastq.Load10X), + // not a literal file named "10x" -- this used to be unreachable because + // the CLI's own Args validator ran filecheck("10x") before this point, + // but CountFQ/Count10XFQ themselves have always honored the sentinel. + _, err := captureStdout(t, func() error { + return CountFQ([]string{r1}, false, "10x") + }) + if err != nil { + t.Fatalf("CountFQ with embedded whitelist sentinel: %v", err) + } +} diff --git a/docs/src/content/docs/fastq/convert.mdx b/docs/src/content/docs/fastq/convert.mdx index 56b8753..d6f5334 100644 --- a/docs/src/content/docs/fastq/convert.mdx +++ b/docs/src/content/docs/fastq/convert.mdx @@ -9,10 +9,13 @@ import { Badge } from '@astrojs/starlight/components'; If you need to convert between the linked-read technologies' distinct FASTQ formats and their -barcoding systems, use `convert`. It auto-detects the input format as one of 10X, haplotagging, TELLseq, or stLFR, -and converts it to the format provided as the `TARGET` positional argument. If the -input data is in 10X format, where the barcode is usually the first 16 bases in the R1 sequence, -you will need to provide a `--bc-map` file that lets Djinn know what barcodes to look for. +barcoding systems, use `convert`. It auto-detects the input format as one of haplotagging, TELLseq, or stLFR +by inspecting the first 100 records. 10X cannot be auto-detected this way (its barcode is just the +first 16 bases of the R1 sequence, with nothing marking it as a barcode), so converting **from** 10X +requires `--bc-list` to tell Djinn which 16-base sequences are real barcodes: either a path to your +own whitelist file, or the literal value `10x` to use the official 10X Genomics barcode whitelist +embedded in Djinn. Converting **to** 10X draws new barcodes from that same embedded whitelist and +needs no `--bc-list` at all. In all cases, a file will be created with the barcode conversion map. ## Usage @@ -36,11 +39,11 @@ convert 10x gcurratum_as10X gcurratum.R1.fq.gz gcurratum.R2.fq.gz | `PREFIX` | output filename prefix | | `TARGET` | output barcode and FASTQ style: `10x`, `haplotagging`, `tellseq`, `stlfr`, `standard` | | `file.R*.fq.gz` | FASTQ file or file pair | -| `-b` `--bc-map` | Valid barcode list | +| `-b` `--bc-list` | Valid barcode list (or literal `10x` for the embedded whitelist) |
- example `--bc-map` file + example `--bc-list` file ``` ATGGAAGCCGTAGTTA ACGGAAGCCGTAGTTC diff --git a/docs/src/content/docs/fastq/count.mdx b/docs/src/content/docs/fastq/count.mdx index 9796e53..93b3db6 100644 --- a/docs/src/content/docs/fastq/count.mdx +++ b/docs/src/content/docs/fastq/count.mdx @@ -13,6 +13,12 @@ You can use one file of a pair (e.g. R1 or R2), but including both files will pr and only count occurences of _new_ barcodes in the second file that aren't present in the first file. This is done to avoid double-counting barcodes. +Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data +cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking +it as a barcode), so counting 10X barcodes requires `--bc-list`: either a path to your own +whitelist, or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn. +For 10X input, only the first file (R1) is read, since that's the only place the barcode lives. + ## Usage ```bash @@ -29,3 +35,4 @@ djinn fastq count sample1.F.fq.gz sample1.R.fq.gz > sample1.bxcount |:------------------|:------------------------------------------------------------------------------| | `file.R*.fq.gz` | FASTQ file or file pair | | `-i` `--invalid` | Include invalid barcodes | +| `-b` `--bc-list` | Valid barcode list, or literal `10x` for the embedded official whitelist | diff --git a/docs/src/content/docs/fastq/extract.mdx b/docs/src/content/docs/fastq/extract.mdx index 01b40d4..782a91e 100644 --- a/docs/src/content/docs/fastq/extract.mdx +++ b/docs/src/content/docs/fastq/extract.mdx @@ -8,6 +8,12 @@ import { Badge } from '@astrojs/starlight/components'; To get a list of all the unique barcodes in your data, use `extract`. Writes to `stdout`. +Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data +cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking +it as a barcode), so extracting 10X barcodes requires `--bc-list`: either a path to your own +whitelist, or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn. +For 10X input, only the first file (R1) is read, since that's the only place the barcode lives. + ## Usage ```bash djinn fastq extract file.R1.fq.gz [file.R2.fq.gz] @@ -23,3 +29,4 @@ djinn fastq extract -i sample1.F.fq.gz sample1.R.fq.gz > sample1.bc |:------------------|:------------------------------------------------------------------------------| | `file.R*fq.gz` | FASTQ file or file pair | | `-i` `--invalid` | Include invalid barcodes | +| `-b` `--bc-list` | Valid barcode list, or literal `10x` for the embedded official whitelist | diff --git a/docs/src/content/docs/fastq/ncbi.mdx b/docs/src/content/docs/fastq/ncbi.mdx index 2b69309..2a2c4f2 100644 --- a/docs/src/content/docs/fastq/ncbi.mdx +++ b/docs/src/content/docs/fastq/ncbi.mdx @@ -23,6 +23,12 @@ files into a single unaligned BAM file without any data loss (and also free up d you risk losing the barcode forever! +Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X cannot be +auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking it as a +barcode), so converting 10X FASTQ to BAM requires `--bc-list` to tell Djinn which 16-base sequences +are real barcodes -- either a path to your own whitelist, or the literal value `10x` to use the +official 10X Genomics whitelist embedded in Djinn. + ## Usage ```bash @@ -43,5 +49,5 @@ samtools import -O BAM -T "*" -1 file.R1.fq file.R2.fq > out.bam | argument | description | |:-----------------|:---------------------------------------------------------------------------| | `file.R*.fq.gz` | FASTQ file or file pair | -| `-b` `--bc-map` | Valid barcode list | +| `-b` `--bc-list` | Valid barcode list, or literal `10x` for the embedded official whitelist | | `-t` `--threads` | Threads to use (default: 2) | diff --git a/docs/src/content/docs/fastq/rm-invalid.mdx b/docs/src/content/docs/fastq/rm-invalid.mdx index 832dc81..f57b03c 100644 --- a/docs/src/content/docs/fastq/rm-invalid.mdx +++ b/docs/src/content/docs/fastq/rm-invalid.mdx @@ -10,6 +10,13 @@ You may be interested in retaining only "useful" linked reads, that is, reads wi Using `rm-invalid` will remove alignment records invalid barcodes. If you want to keep a separate file (or files) of records with invalid barcodes use the `--invalid` flag. +Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data +cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking +it as a barcode), so filtering 10X data requires `--bc-list`: either a path to your own whitelist, +or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn. A 10X R1 +read whose first 16 bases don't match any whitelist entry has no barcode at all and is treated as +invalid. + ## Usage ```bash djinn sam rm-invalid [--invalid] PREFIX file.R1.fq.gz [file.R2.fq.gz] @@ -26,3 +33,4 @@ djinn sam rm-invalid --invalid echinda.valid echidna.R*.fq.gz | `PREFIX` | output filename prefix | | `file.R*.fq.gz` | FASTQ file or file pair | | `-i` `--invalid` | output records with invalid barcodes to `PREFIX.invalid.*` | +| `-b` `--bc-list` | Valid barcode list, or literal `10x` for the embedded official whitelist | diff --git a/docs/src/content/docs/fastq/rm-unlinked.mdx b/docs/src/content/docs/fastq/rm-unlinked.mdx index e5d770a..2de7e5e 100644 --- a/docs/src/content/docs/fastq/rm-unlinked.mdx +++ b/docs/src/content/docs/fastq/rm-unlinked.mdx @@ -12,6 +12,11 @@ to isolate records with meaningful linked-read information. If you want to keep file (or files) of unlinked/singleton records, use the `--unlinked` flag. Barcodes are counted per read-pair or unpair read. +Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR. 10X data +cannot be auto-detected this way (its barcode is just the first 16 bases of R1, with nothing marking +it as a barcode), so filtering 10X data requires `--bc-list`: either a path to your own whitelist, +or the literal value `10x` to use the official 10X Genomics whitelist embedded in Djinn. + ## Usage ```bash @@ -28,5 +33,6 @@ djinn fastq rm-unlinked --unlinked hyena.clean hyena.R*.fq.gz | :------------------ | :--------------------------------------------------------------------------------- | | `PREFIX` | output filename prefix | | `file.R*.fq.gz` | FASTQ file or file pair | -| `-b` `--bc-count` | write valid barcode counts to this file | +| `--bc-count` | write valid barcode counts to `PREFIX.count` | | `-u` `--unlinked` | write valid singleton records to `PREFIX.unlinked` | +| `-b` `--bc-list` | Valid barcode list, or literal `10x` for the embedded official whitelist | diff --git a/docs/src/content/docs/fastq/sort.mdx b/docs/src/content/docs/fastq/sort.mdx index 37c6dac..fff11f0 100644 --- a/docs/src/content/docs/fastq/sort.mdx +++ b/docs/src/content/docs/fastq/sort.mdx @@ -14,7 +14,10 @@ in the process. Sorting is done using a `samtools` style k-way merge, where reads (or read pairs) are inlined with an intermediate format and cast across several temporary files located at `--tmp-dir`, where -they are then merged back together. Using the `--standardize` flag will result in sorted +they are then merged back together. Format is auto-detected from the first 100 records for +haplotagging, TELLseq, and stLFR; 10X cannot be auto-detected this way (its barcode is just the +first 16 bases of R1, with nothing marking it as a barcode), so sorting 10X data requires +`--bc-list` to tell Djinn which 16-base sequences are real barcodes. Using the `--standardize` flag will result in sorted [standard format](https://blinkseq.github.io/lastq/) data that, among other things, is ready for use in [arachne](https://github.com/blinkseq/arachne). @@ -40,7 +43,7 @@ When input is a FASTQ file pair, the reads are expected to be synced between R1 |:------------------|:------------------------------------------------------------------------------| | `PREFIX` | output filename prefix | | `file.R*.fq.gz` | FASTQ file or file pair | -| `-b` `--bc-map` | valid barcode list | +| `-b` `--bc-list` | valid barcode list, or literal `10x` for the embedded official whitelist | | `-s` `--standardize` | output using standardized BX:Z:/VX:i: tags instead of the detected input format | | `-@` `--threads` | treads to use (default: 2) | | `-t` `--tmp-prefix` | folder for temporary files | diff --git a/docs/src/content/docs/fastq/stats.mdx b/docs/src/content/docs/fastq/stats.mdx new file mode 100644 index 0000000..95ea170 --- /dev/null +++ b/docs/src/content/docs/fastq/stats.mdx @@ -0,0 +1,49 @@ +--- +title: Barcode statistics for a FASTQ file +sidebar: + order: 4 + label: stats +--- +import { Badge } from '@astrojs/starlight/components'; + +If you want a quick read on how clean your linked-read barcodes are, use `stats`. It reports how many +reads carry a barcode, how many of those barcodes are valid, and, for the barcodes that aren't, which +position tends to go wrong. Writes to `stdout`. + +Format is auto-detected from the first 100 records for haplotagging, TELLseq, and stLFR, or you can +name it with `--platform` to skip detection. 10X data cannot be auto-detected this way (its barcode is +just the first 16 bases of R1, with nothing marking it as a barcode), so 10X input requires `--bc-list` +instead: either a path to your own whitelist, or the literal value `10x` to use the official 10X +Genomics whitelist embedded in Djinn. + +## Usage +```bash +djinn fastq stats [options] file.fq.gz +``` + +Example +```bash +djinn fastq stats curratum.R1.fq.gz > curratum.R1.stats +``` + +## Options +| argument | description | +|:--------------------|:----------------------------------------------------------------------------------------------| +| `file.fq.gz` | FASTQ file | +| `-p` `--platform` | Linked-read technology: `haplotagging`, `stlfr` or `tellseq` (default: detect from the first 100 reads) | +| `-b` `--bc-list` | Valid barcode list, or literal `10x` for the embedded official whitelist | +| `-@` `--threads` | Threads to use (default: 2) | + +## Output + +| column | description | +|:-------------------|:-------------------------------------------------------------------------| +| `Reads` | total sequences in the file | +| `Barcodes` | reads that carry a linked-read barcode | +| `Barcodes_Valid` | barcodes that are valid | +| `Barcodes_Invalid` | barcodes that are not valid | + +followed by one line per barcode position that can go wrong, with the number of barcodes that were bad +at that position: haplotagging segments `A`, `C`, `B`, `D` (e.g. a segment reading `00`), stLFR parts +`1`-`3` (a part reading `0`), TELLseq bases `1`-`18` (an `N`), or 10X bases `1`-`16` (an `N`). A barcode +with several bad positions adds to each of them. diff --git a/docs/src/content/docs/sam/stats.mdx b/docs/src/content/docs/sam/stats.mdx new file mode 100644 index 0000000..239371a --- /dev/null +++ b/docs/src/content/docs/sam/stats.mdx @@ -0,0 +1,54 @@ +--- +title: Linked-read metrics per molecule +sidebar: + order: 4 + label: stats +--- +import { Badge, Aside } from '@astrojs/starlight/components'; + +Once your reads are aligned, `stats` reports per-molecule metrics: how many reads a barcode has on a +contig, where they start and end, how many bases aligned, and how well the molecule is covered. A +"molecule" is one barcode's reads on one contig; use `--distance-threshold` to split a barcode's reads +into separate molecules (`BARCODE-1`, `BARCODE-2`, ...) when the gap between consecutive reads is too +large, the same splitting `deconvolve` does, without writing a new BAM. Writes a table to `stdout`. + +Barcodes come from the `BX:Z` tag (or a TELLseq/stLFR-style read name), and a barcode is valid if the +`VX:i` tag says so, or, when there is no `VX:i` tag, if the barcode itself is valid. Reads without a +valid barcode are pooled into one `invalid` line per contig. + + + +## Usage +```bash +djinn sam stats [options] file.bam +``` + +Example +```bash +djinn sam stats -d 50000 -@ 4 curratum.bam > curratum.molecules.tsv +``` + +## Options +| argument | description | +|:------------------------------|:------------------------------------------------------------------------------------------------------| +| `file.bam` | coordinate-sorted SAM/BAM file, or `-` for stdin | +| `-d` `--distance-threshold` | distance (bp) that splits a barcode's reads into separate molecules, where `0` skips splitting (default: 50000) | +| `-@` `--threads` | Threads to use (default: 2) | + +## Output + +| column | description | +|:-------------------|:------------------------------------------------------------------------| +| `contig` | chromosome/contig the molecule is on | +| `molecule` | the barcode, suffixed with `-N` if split by `--distance-threshold` | +| `reads` | number of reads sharing the barcode on this molecule | +| `start` | leftmost aligned position | +| `end` | rightmost aligned position | +| `length_inferred` | `end - start` | +| `aligned_bp` | total aligned bases across the molecule's reads | +| `insert_len` | inferred insert length | +| `coverage_bp` | `aligned_bp` / `length_inferred` | +| `coverage_inserts` | inferred-insert coverage of the molecule | diff --git a/extract/extract.fq.go b/extract/extract.fq.go index c25639d..35a9647 100644 --- a/extract/extract.fq.go +++ b/extract/extract.fq.go @@ -6,7 +6,6 @@ import ( "io" "os" - "github.com/blinkseq/djinn/barcodes" "github.com/blinkseq/djinn/fastq" "github.com/shenwei356/bio/seq" @@ -77,23 +76,11 @@ func ExtractFQ(fqs []string, invalid bool, bcmap string) error { func Extract10XFQ(fqs []string, invalid bool, bcmap string) error { seq.ValidateSeq = false - var bclist map[string]struct{} - var err error - - switch bcmap { - case "10X", "10x": - bclist, err = barcodes.LoadTenXSet() - if err != nil { - return err - } - default: - bclist, err = barcodes.LoadTenXFromFile(bcmap) - if err != nil { - return err - } + bclist, err := fastq.Load10X(bcmap) + if err != nil { + return err } - seq.ValidateSeq = false // ── open barcode writer ─────────────────────────────────────────────────────────── writer := bufio.NewWriter(os.Stdout) defer writer.Flush() diff --git a/extract/extract_fq_test.go b/extract/extract_fq_test.go index 8fc1e0a..0d7ed36 100644 --- a/extract/extract_fq_test.go +++ b/extract/extract_fq_test.go @@ -34,7 +34,7 @@ func TestExtractFQHaplotagging(t *testing.T) { var err error out := captureStdout(t, func() { - err = ExtractFQ([]string{fq}, false) + err = ExtractFQ([]string{fq}, false, "") }) if err != nil { t.Fatalf("ExtractFQ: %v", err) @@ -59,7 +59,7 @@ func TestExtractFQIncludeInvalid(t *testing.T) { var err error out := captureStdout(t, func() { - err = ExtractFQ([]string{fq}, true) + err = ExtractFQ([]string{fq}, true, "") }) if err != nil { t.Fatalf("ExtractFQ: %v", err) @@ -84,7 +84,7 @@ func TestExtractFQMultipleFiles(t *testing.T) { var err error out := captureStdout(t, func() { - err = ExtractFQ([]string{fq1, fq2}, false) + err = ExtractFQ([]string{fq1, fq2}, false, "") }) if err != nil { t.Fatalf("ExtractFQ: %v", err) @@ -96,8 +96,79 @@ func TestExtractFQMultipleFiles(t *testing.T) { } func TestExtractFQBadFirstFile(t *testing.T) { - err := ExtractFQ([]string{"/nonexistent/path/reads.fq"}, false) + err := ExtractFQ([]string{"/nonexistent/path/reads.fq"}, false, "") if err == nil { t.Fatal("expected error for nonexistent input file, got nil") } } + +func writeTenXFastqExtract(t *testing.T, path string, ids []string, barcode string) { + t.Helper() + var sb strings.Builder + for _, id := range ids { + seq := barcode + "ACGTACGTAC" + sb.WriteString("@" + id + "\n" + seq + "\n+\n" + strings.Repeat("I", len(seq)) + "\n") + } + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestExtract10XFQ(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\nTTTTGGGGCCCCAAAA\n"), 0o644); err != nil { + t.Fatal(err) + } + + var sb strings.Builder + sb.WriteString("@read1\nAAAACCCCGGGGTTTTACGTACGTAC\n+\nIIIIIIIIIIIIIIIIIIIIIIIIII\n") + sb.WriteString("@read2\nAAAACCCCGGGGTTTTACGTACGTAC\n+\nIIIIIIIIIIIIIIIIIIIIIIIIII\n") // duplicate barcode + sb.WriteString("@read3\nTTTTGGGGCCCCAAAAACGTACGTAC\n+\nIIIIIIIIIIIIIIIIIIIIIIIIII\n") // second barcode + sb.WriteString("@read4\nZZZZZZZZZZZZZZZZACGTACGTAC\n+\nIIIIIIIIIIIIIIIIIIIIIIIIII\n") // unrecognized -> dropped + if err := os.WriteFile(r1, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } + + var err error + out := captureStdout(t, func() { + err = ExtractFQ([]string{r1}, false, wl) + }) + if err != nil { + t.Fatalf("ExtractFQ (10X): %v", err) + } + lines := splitNonEmpty(string(out)) + if len(lines) != 2 { + t.Fatalf("got %d barcodes, want 2 (deduped, unrecognized dropped): %v", len(lines), lines) + } +} + +// ExtractFQ only ever reads fqs[0] for 10X input (the barcode lives solely +// in R1); passing a second file must not error, and its contents must be +// ignored rather than misread as a second R1. +func TestExtract10XFQ_IgnoresR2(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + r2 := filepath.Join(dir, "r2.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\n"), 0o644); err != nil { + t.Fatal(err) + } + writeTenXFastqExtract(t, r1, []string{"read1"}, "AAAACCCCGGGGTTTT") + if err := os.WriteFile(r2, []byte("@read1\nGGGGCCCCAAAATTTT\n+\nIIIIIIIIIIIIIIII\n"), 0o644); err != nil { + t.Fatal(err) + } + + var err error + out := captureStdout(t, func() { + err = ExtractFQ([]string{r1, r2}, false, wl) + }) + if err != nil { + t.Fatalf("ExtractFQ (10X, paired): %v", err) + } + lines := splitNonEmpty(string(out)) + if len(lines) != 1 || lines[0] != "AAAACCCCGGGGTTTT" { + t.Errorf("got %v, want exactly [AAAACCCCGGGGTTTT]", lines) + } +} diff --git a/fastq/parser_10x.go b/fastq/parser_10x.go index f3201f7..617678a 100644 --- a/fastq/parser_10x.go +++ b/fastq/parser_10x.go @@ -60,7 +60,7 @@ func TenXR22Corefq(rec *fastx.Record) (core CoreFq) { } // Load a 10X barcode whitelist. If bcmap is '10x' or '10X', loads the official list -// embedded in djinn, otherwise rease the bcmap file +// embedded in djinn, otherwise reads the bcmap file func Load10X(bcmap string) (barcodeMap map[string]struct{}, err error) { switch bcmap { case "10X", "10x": diff --git a/invalid/invalid.fq.go b/invalid/invalid.fq.go index ae43024..db56ab5 100644 --- a/invalid/invalid.fq.go +++ b/invalid/invalid.fq.go @@ -112,6 +112,17 @@ func FilterInvalid10XFQ(fqs []string, prefix string, keepInvalid bool, bcmap str } defer R1Writer.Close() + // Invalid records go to their own prefix.invalid.R*.fq.gz files, same + // as the non-10X path -- not mixed back into the main output. + var R1WriterInv *xopen.Writer + if keepInvalid { + R1WriterInv, err = xopen.Wopen(prefix + ".invalid.R1.fq.gz") + if err != nil { + return err + } + defer R1WriterInv.Close() + } + if len(fqs) == 1 { // Single-end 10X input: no R2 to propagate the barcode onto. for { @@ -125,7 +136,7 @@ func FilterInvalid10XFQ(fqs []string, prefix string, keepInvalid bool, bcmap str r1core, ok := fastq.TenXR12Corefq(_r1, bclist) if !ok || !r1core.Valid { if keepInvalid { - r1core.ToTenX(R1Writer.Writer) + r1core.ToTenX(R1WriterInv.Writer) } continue } @@ -144,6 +155,15 @@ func FilterInvalid10XFQ(fqs []string, prefix string, keepInvalid bool, bcmap str } defer R2Writer.Close() + var R2WriterInv *xopen.Writer + if keepInvalid { + R2WriterInv, err = xopen.Wopen(prefix + ".invalid.R2.fq.gz") + if err != nil { + return err + } + defer R2WriterInv.Close() + } + var r1core, r2core fastq.CoreFq Loop: @@ -158,7 +178,7 @@ func FilterInvalid10XFQ(fqs []string, prefix string, keepInvalid bool, bcmap str r1core, ok = fastq.TenXR12Corefq(_r1, bclist) if !ok || !r1core.Valid { if keepInvalid { - r1core.ToTenX(R1Writer.Writer) + r1core.ToTenX(R1WriterInv.Writer) } continue } @@ -177,19 +197,25 @@ func FilterInvalid10XFQ(fqs []string, prefix string, keepInvalid bool, bcmap str } case nil: r2core = fastq.TenXR22Corefq(_r2) - // check if R1 exists and match the seq ID + // check if R1 exists and match the seq ID; a record with + // no matching R1 (should only happen if the files are + // mismatched/truncated) has no barcode info and is + // therefore treated the same as an invalid barcode. + valid := false if r1err == nil { if ok := slices.Equal(r1core.ID, r2core.ID); !ok { return fmt.Errorf("Misaligned R1 and R2 records. The 10X format only stores the barcode at the beginning of read 1, meaning Read 2 must have the same sequence ID as Read 1 to reliably assign a barcode to Read 2, but they do not match.\nRead 1:\n%s\nRead 2:\n%s", r1core.ID, r2core.ID) } r2core.Barcode = r1core.Barcode - if !r1core.Valid { - if keepInvalid { - r2core.ToTenX(R2Writer.Writer) - } - continue + valid = r1core.Valid + } + if !valid { + if keepInvalid { + r2core.ToTenX(R2WriterInv.Writer) } + continue } + r2core.ToTenX(R2Writer.Writer) default: return r2err } diff --git a/invalid/invalid_fq_test.go b/invalid/invalid_fq_test.go index cce28de..6d77fd5 100644 --- a/invalid/invalid_fq_test.go +++ b/invalid/invalid_fq_test.go @@ -1,8 +1,10 @@ package invalid import ( + "os" "path/filepath" "slices" + "strings" "testing" ) @@ -21,7 +23,7 @@ func TestFilterInvalidFQ_SplitsValidAndInvalid(t *testing.T) { prefix := filepath.Join(dir, "out") invPrefix := true - if err := FilterInvalidFQ([]string{r1}, prefix, invPrefix); err != nil { + if err := FilterInvalidFQ([]string{r1}, prefix, invPrefix, ""); err != nil { t.Fatalf("FilterInvalidFQ returned error: %v", err) } @@ -54,7 +56,7 @@ func TestFilterInvalidFQ_NoInvalidPrefix(t *testing.T) { writeHapFastq(t, r1, reads) prefix := filepath.Join(dir, "out") - if err := FilterInvalidFQ([]string{r1}, prefix, false); err != nil { + if err := FilterInvalidFQ([]string{r1}, prefix, false, ""); err != nil { t.Fatalf("FilterInvalidFQ returned error: %v", err) } @@ -78,7 +80,7 @@ func TestFilterInvalidFQ_PairedFiles(t *testing.T) { writeHapFastq(t, r2, []hapRead{{id: "p1", bc: "TTTT", vx: 1}}) prefix := filepath.Join(dir, "out") - if err := FilterInvalidFQ([]string{r1, r2}, prefix, false); err != nil { + if err := FilterInvalidFQ([]string{r1, r2}, prefix, false, ""); err != nil { t.Fatalf("FilterInvalidFQ returned error: %v", err) } if ids := readGzFastqIDs(t, prefix+".R1.fq.gz"); !slices.Contains(ids, "p1") { @@ -91,8 +93,137 @@ func TestFilterInvalidFQ_PairedFiles(t *testing.T) { func TestFilterInvalidFQ_MissingFile(t *testing.T) { dir := t.TempDir() - err := FilterInvalidFQ([]string{"/nonexistent/file.fq"}, filepath.Join(dir, "out"), false) + err := FilterInvalidFQ([]string{"/nonexistent/file.fq"}, filepath.Join(dir, "out"), false, "") if err == nil { t.Fatal("expected an error for a nonexistent input file") } } + +func writeTenXR1(t *testing.T, path string, reads [][2]string) { + t.Helper() + var sb strings.Builder + for _, r := range reads { + id, barcode := r[0], r[1] + seq := barcode + "ACGTACGTAC" + sb.WriteString("@" + id + "\n" + seq + "\n+\n" + strings.Repeat("I", len(seq)) + "\n") + } + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } +} + +func writeTenXR2(t *testing.T, path string, ids []string) { + t.Helper() + var sb strings.Builder + for _, id := range ids { + sb.WriteString("@" + id + "\nGGGGCCCCAAAATTTT\n+\nIIIIIIIIIIIIIIII\n") + } + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } +} + +// Regression test: FilterInvalid10XFQ's paired branch used to never write +// a valid R2 record to its own output file at all (only the keepInvalid +// path wrote R2, for invalid records) -- prefix.R2.fq.gz came out empty +// for every properly-paired, valid-barcode input. +func TestFilterInvalid10XFQ_PairedWritesValidR2(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + r2 := filepath.Join(dir, "r2.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\n"), 0o644); err != nil { + t.Fatal(err) + } + writeTenXR1(t, r1, [][2]string{{"good1", "AAAACCCCGGGGTTTT"}}) + writeTenXR2(t, r2, []string{"good1"}) + + prefix := filepath.Join(dir, "out") + if err := FilterInvalidFQ([]string{r1, r2}, prefix, false, wl); err != nil { + t.Fatalf("FilterInvalidFQ (10X paired): %v", err) + } + + r1IDs := readGzFastqIDs(t, prefix+".R1.fq.gz") + r2IDs := readGzFastqIDs(t, prefix+".R2.fq.gz") + if !slices.Contains(r1IDs, "good1:AAAACCCCGGGGTTTT") { + t.Errorf("expected good1 in R1 output, got %v", r1IDs) + } + if len(r2IDs) == 0 { + t.Fatal("R2 output is empty -- valid R2 records must be written, not just invalid ones") + } + if !slices.Contains(r2IDs, "good1:AAAACCCCGGGGTTTT") { + t.Errorf("expected good1 in R2 output, got %v", r2IDs) + } +} + +// Regression test: FilterInvalid10XFQ used to have no separate invalid-output +// file at all for 10X input, unlike every other format -- "kept" invalid +// records were written into the same file as valid ones instead of their +// own prefix.invalid.R*.fq.gz. +func TestFilterInvalid10XFQ_SeparatesInvalidOutput(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + r2 := filepath.Join(dir, "r2.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\n"), 0o644); err != nil { + t.Fatal(err) + } + writeTenXR1(t, r1, [][2]string{ + {"good1", "AAAACCCCGGGGTTTT"}, + {"bad1", "ZZZZZZZZZZZZZZZZ"}, + }) + writeTenXR2(t, r2, []string{"good1", "bad1"}) + + prefix := filepath.Join(dir, "out") + if err := FilterInvalidFQ([]string{r1, r2}, prefix, true, wl); err != nil { + t.Fatalf("FilterInvalidFQ (10X paired, keepInvalid): %v", err) + } + + validR1 := readGzFastqIDs(t, prefix+".R1.fq.gz") + validR2 := readGzFastqIDs(t, prefix+".R2.fq.gz") + invalidR1 := readGzFastqIDs(t, prefix+".invalid.R1.fq.gz") + invalidR2 := readGzFastqIDs(t, prefix+".invalid.R2.fq.gz") + + for _, ids := range [][]string{validR1, validR2} { + if slices.ContainsFunc(ids, func(s string) bool { return strings.HasPrefix(s, "bad1") }) { + t.Errorf("invalid record bad1 leaked into the main valid output: %v", ids) + } + } + for _, ids := range [][]string{invalidR1, invalidR2} { + if !slices.ContainsFunc(ids, func(s string) bool { return strings.HasPrefix(s, "bad1") }) { + t.Errorf("expected bad1 in the separate invalid output, got %v", ids) + } + if slices.ContainsFunc(ids, func(s string) bool { return strings.HasPrefix(s, "good1") }) { + t.Errorf("valid record good1 leaked into the invalid output: %v", ids) + } + } +} + +func TestFilterInvalid10XFQ_SingleEnd(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\n"), 0o644); err != nil { + t.Fatal(err) + } + writeTenXR1(t, r1, [][2]string{ + {"good1", "AAAACCCCGGGGTTTT"}, + {"bad1", "ZZZZZZZZZZZZZZZZ"}, + }) + + prefix := filepath.Join(dir, "out") + if err := FilterInvalidFQ([]string{r1}, prefix, true, wl); err != nil { + t.Fatalf("FilterInvalidFQ (10X single-end, keepInvalid): %v", err) + } + if _, err := os.Stat(prefix + ".R2.fq.gz"); err == nil { + t.Error("single-end 10X input must not produce an R2 output file") + } + validIDs := readGzFastqIDs(t, prefix+".R1.fq.gz") + invalidIDs := readGzFastqIDs(t, prefix+".invalid.R1.fq.gz") + if !slices.ContainsFunc(validIDs, func(s string) bool { return strings.HasPrefix(s, "good1") }) { + t.Errorf("expected good1 in valid output, got %v", validIDs) + } + if !slices.ContainsFunc(invalidIDs, func(s string) bool { return strings.HasPrefix(s, "bad1") }) { + t.Errorf("expected bad1 in invalid output, got %v", invalidIDs) + } +} diff --git a/stats/stats.fq.go b/stats/stats.fq.go index d5e6092..5bbe94b 100644 --- a/stats/stats.fq.go +++ b/stats/stats.fq.go @@ -43,18 +43,18 @@ func StatsFQ(fq, platform string, bcMap string) error { var sch scheme if bcMap != "" { - var bclist map[string]struct{} bclist, err := fastq.Load10X(bcMap) if err != nil { return err } parse = func(rec *fastx.Record) (fastq.CoreFq, bool) { return fastq.TenXR12Corefq(rec, bclist) } sch = tenxScheme - } - - parse, sch, err := chooseParser(fq, platform) - if err != nil { - return err + } else { + var err error + parse, sch, err = chooseParser(fq, platform) + if err != nil { + return err + } } seq.ValidateSeq = false // reads can hold N and other ambiguity codes; nothing here needs them checked diff --git a/stats/stats_fq_test.go b/stats/stats_fq_test.go index fade62b..60b0134 100644 --- a/stats/stats_fq_test.go +++ b/stats/stats_fq_test.go @@ -24,7 +24,7 @@ func writeFastq(t *testing.T, records ...string) string { func statsFQOf(t *testing.T, path, platform string) string { t.Helper() - out, err := captureStdout(t, func() error { return StatsFQ(path, platform) }) + out, err := captureStdout(t, func() error { return StatsFQ(path, platform, "") }) if err != nil { t.Fatalf("StatsFQ: %v", err) } @@ -156,7 +156,7 @@ func TestStatsFQ_StlfrAndTellseqByName(t *testing.T) { func TestStatsFQ_Errors(t *testing.T) { noBarcodes := writeFastq(t, fq("r1 1:N:0:ATGGAGA"), fq("r2 1:N:0:ATGGAGA")) - if _, err := captureStdout(t, func() error { return StatsFQ(noBarcodes, "") }); err == nil || + if _, err := captureStdout(t, func() error { return StatsFQ(noBarcodes, "", "") }); err == nil || !strings.Contains(err.Error(), "unable to determine") { t.Errorf("no detectable technology: err = %v", err) } @@ -166,11 +166,45 @@ func TestStatsFQ_Errors(t *testing.T) { t.Errorf("no barcodes with a named platform: got:\n%s\nwant:\n%s", got, want) } - if _, err := captureStdout(t, func() error { return StatsFQ(noBarcodes, "10x") }); err == nil || + if _, err := captureStdout(t, func() error { return StatsFQ(noBarcodes, "10x", "") }); err == nil || !strings.Contains(err.Error(), "unknown platform") { t.Errorf("unknown platform: err = %v", err) } - if err := StatsFQ(fmt.Sprintf("%s/missing.fq", t.TempDir()), Haplotagging); err == nil { + if err := StatsFQ(fmt.Sprintf("%s/missing.fq", t.TempDir()), Haplotagging, ""); err == nil { t.Error("a missing file should be an error") } } + +func tenxRecord(id, barcode string) string { + seq := barcode + "ACGTACGTAC" + return "@" + id + "\n" + seq + "\n+\n" + strings.Repeat("I", len(seq)) + "\n" +} + +// Regression test: StatsFQ's 10X branch built parse/sch from bcMap but then +// unconditionally called chooseParser(fq, platform) right after, which +// reassigned parse/sch every time regardless of bcMap -- --bc-list was a +// complete no-op and 10X input always fell through to plain +// haplotagging/stlfr/tellseq auto-detection (which would just fail, since +// 10X has no self-describing marker). +func TestStatsFQ_TenX(t *testing.T) { + dir := t.TempDir() + wl := fmt.Sprintf("%s/whitelist.txt", dir) + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\n"), 0o644); err != nil { + t.Fatal(err) + } + + path := writeFastq(t, + tenxRecord("read1", "AAAACCCCGGGGTTTT"), + tenxRecord("read2", "AAAACCCCGGGGTTTT"), + tenxRecord("read3", "ZZZZZZZZZZZZZZZZ"), // not in whitelist -> no barcode at all + ) + + got, err := captureStdout(t, func() error { return StatsFQ(path, "", wl) }) + if err != nil { + t.Fatalf("StatsFQ (10X): %v", err) + } + lines := strings.Split(strings.TrimRight(got, "\n"), "\n") + if lines[0] != "Reads\t3" || lines[1] != "Barcodes\t2" || lines[2] != "Barcodes_Valid\t2" || lines[3] != "Barcodes_Invalid\t0" { + t.Errorf("unexpected 10X stats output:\n%s", got) + } +} diff --git a/unlinked/unlinked.fq.go b/unlinked/unlinked.fq.go index 534ff90..ac6bbf9 100644 --- a/unlinked/unlinked.fq.go +++ b/unlinked/unlinked.fq.go @@ -6,6 +6,7 @@ import ( "io" "maps" "os" + "slices" "strconv" "github.com/blinkseq/djinn/fastq" @@ -86,7 +87,184 @@ func getFqCount(infiles []string) (map[string]int, error) { return set, nil } +// getFqCount10X counts valid 10X barcodes from R1 only -- unlike the other +// linked-read formats, a 10X barcode lives solely in R1's sequence prefix, +// so R2 has nothing to independently count. +func getFqCount10X(fqs []string, bclist map[string]struct{}) (map[string]int, error) { + seq.ValidateSeq = false + set := make(map[string]int, 4_000_000) + + fqReader, err := fastx.NewReader(seq.DNA, fqs[0], "") + if err != nil { + return nil, fmt.Errorf("opening %s: %w", fqs[0], err) + } + defer fqReader.Close() + + for { + rec, err := fqReader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + core, ok := fastq.TenXR12Corefq(rec, bclist) + if !ok || !core.Valid { + continue + } + set[string(core.Barcode)]++ + } + return set, nil +} + +// RemoveUnlinked10XFQ is the 10X-input counterpart of RemoveUnlinkedFQ: a +// barcode is "linked" if its R1 sequence-prefix matches the whitelist on 2 +// or more reads, same threshold as every other format. +func RemoveUnlinked10XFQ(fqs []string, prefix string, keepSingle, barcodecount bool, bcmap string) error { + bclist, err := fastq.Load10X(bcmap) + if err != nil { + return err + } + + bcCounts, err := getFqCount10X(fqs, bclist) + if err != nil { + return err + } + if barcodecount { + f, err := os.Create(prefix + ".count") + if err != nil { + return err + } + writer := bufio.NewWriter(f) + for key, val := range bcCounts { + writer.WriteString(key) + writer.WriteByte('\t') + writer.WriteString(strconv.Itoa(val)) + writer.WriteByte('\n') + } + writer.Flush() + f.Close() + } + + R1Reader, err := fastx.NewReader(seq.DNA, fqs[0], "") + if err != nil { + return fmt.Errorf("opening %s: %w", fqs[0], err) + } + defer R1Reader.Close() + + R1Writer, err := xopen.Wopen(prefix + ".R1.fq.gz") + if err != nil { + return err + } + defer R1Writer.Close() + + var R1WriterSingle *xopen.Writer + if keepSingle { + R1WriterSingle, err = xopen.Wopen(prefix + ".unlinked.R1.fq.gz") + if err != nil { + return err + } + defer R1WriterSingle.Close() + } + + route := func(core *fastq.CoreFq, w, wSingle *xopen.Writer) { + switch { + case bcCounts[string(core.Barcode)] >= 2: + core.ToTenX(w.Writer) + case keepSingle: + core.ToTenX(wSingle.Writer) + } + } + + if len(fqs) == 1 { + // Single-end 10X input: no R2 to propagate the barcode onto. + for { + _r1, err := R1Reader.Read() + if err == io.EOF { + break + } + if err != nil { + return err + } + r1core, ok := fastq.TenXR12Corefq(_r1, bclist) + if !ok || !r1core.Valid { + continue + } + route(&r1core, R1Writer, R1WriterSingle) + } + return nil + } + + R2Reader, err := fastx.NewReader(seq.DNA, fqs[1], "") + if err != nil { + return fmt.Errorf("opening %s: %w", fqs[1], err) + } + defer R2Reader.Close() + + R2Writer, err := xopen.Wopen(prefix + ".R2.fq.gz") + if err != nil { + return err + } + defer R2Writer.Close() + + var R2WriterSingle *xopen.Writer + if keepSingle { + R2WriterSingle, err = xopen.Wopen(prefix + ".unlinked.R2.fq.gz") + if err != nil { + return err + } + defer R2WriterSingle.Close() + } + + var r1core, r2core fastq.CoreFq + +Loop: + for { + // process R1, which doesn't need an R2 + _r1, r1err := R1Reader.Read() + switch r1err { + case io.EOF: + + case nil: + var ok bool + r1core, ok = fastq.TenXR12Corefq(_r1, bclist) + if !ok || !r1core.Valid { + r1core.Barcode = nil // marks this pair as having no usable barcode for R2 below + } else { + route(&r1core, R1Writer, R1WriterSingle) + } + default: + return r1err + } + + // process R2 + _r2, r2err := R2Reader.Read() + switch r2err { + case io.EOF: + // break when both are exhausted + if r1err == io.EOF { + break Loop + } + case nil: + r2core = fastq.TenXR22Corefq(_r2) + if r1err == nil && len(r1core.Barcode) > 0 { + if ok := slices.Equal(r1core.ID, r2core.ID); !ok { + return fmt.Errorf("Misaligned R1 and R2 records. The 10X format only stores the barcode at the beginning of read 1, meaning Read 2 must have the same sequence ID as Read 1 to reliably assign a barcode to Read 2, but they do not match.\nRead 1:\n%s\nRead 2:\n%s", r1core.ID, r2core.ID) + } + r2core.Barcode = r1core.Barcode + route(&r2core, R2Writer, R2WriterSingle) + } + default: + return r2err + } + } + return nil +} + func RemoveUnlinkedFQ(fqs []string, prefix string, keepSingle, barcodecount bool, bcmap string) error { + if bcmap != "" { + return RemoveUnlinked10XFQ(fqs, prefix, keepSingle, barcodecount, bcmap) + } // guard against draining stdin when getting barcode counts bcCounts, err := getFqCount(fqs) if err != nil { diff --git a/unlinked/unlinked_fq_test.go b/unlinked/unlinked_fq_test.go index e1b7dbc..076ceb8 100644 --- a/unlinked/unlinked_fq_test.go +++ b/unlinked/unlinked_fq_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" ) @@ -88,9 +89,8 @@ func TestRemoveUnlinkedFQ_SplitsSingletonsAndNot(t *testing.T) { writeHapFastq(t, r1, reads) prefix := filepath.Join(dir, "out") - bcCountFile := filepath.Join(dir, "counts.tsv") - if err := RemoveUnlinkedFQ([]string{r1}, prefix, true, bcCountFile); err != nil { + if err := RemoveUnlinkedFQ([]string{r1}, prefix, true, true, ""); err != nil { t.Fatalf("RemoveUnlinkedFQ returned error: %v", err) } @@ -110,7 +110,7 @@ func TestRemoveUnlinkedFQ_SplitsSingletonsAndNot(t *testing.T) { t.Errorf("invalid-barcode read g1 should be dropped entirely, got main=%v single=%v", mainIDs, singleIDs) } - if _, err := os.Stat(bcCountFile); err != nil { + if _, err := os.Stat(prefix + ".count"); err != nil { t.Errorf("expected barcode count file to be created: %v", err) } } @@ -126,7 +126,7 @@ func TestRemoveUnlinkedFQ_NoSingletonOutput(t *testing.T) { writeHapFastq(t, r1, reads) prefix := filepath.Join(dir, "out") - if err := RemoveUnlinkedFQ([]string{r1}, prefix, false, ""); err != nil { + if err := RemoveUnlinkedFQ([]string{r1}, prefix, false, false, ""); err != nil { t.Fatalf("RemoveUnlinkedFQ returned error: %v", err) } mainIDs := readGzFastqIDs(t, prefix+".R1.fq.gz") @@ -137,8 +137,109 @@ func TestRemoveUnlinkedFQ_NoSingletonOutput(t *testing.T) { func TestRemoveUnlinkedFQ_MissingFile(t *testing.T) { dir := t.TempDir() - err := RemoveUnlinkedFQ([]string{"/nonexistent/file.fq"}, filepath.Join(dir, "out"), false, "") + err := RemoveUnlinkedFQ([]string{"/nonexistent/file.fq"}, filepath.Join(dir, "out"), false, false, "") if err == nil { t.Fatal("expected an error for a nonexistent input file") } } + +func writeTenXR1Unlinked(t *testing.T, path string, reads [][2]string) { + t.Helper() + var sb strings.Builder + for _, r := range reads { + id, barcode := r[0], r[1] + seq := barcode + "ACGTACGTAC" + sb.WriteString("@" + id + "\n" + seq + "\n+\n" + strings.Repeat("I", len(seq)) + "\n") + } + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } +} + +func writeTenXR2Unlinked(t *testing.T, path string, ids []string) { + t.Helper() + var sb strings.Builder + for _, id := range ids { + sb.WriteString("@" + id + "\nGGGGCCCCAAAATTTT\n+\nIIIIIIIIIIIIIIII\n") + } + if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { + t.Fatal(err) + } +} + +// Regression test: RemoveUnlinkedFQ accepted a bcmap parameter but never +// used it anywhere in the function body, so --bc-list/-b was silently a +// no-op for rm-unlinked -- a genuine 10X input would always fall through +// to auto-detection and fail with "unable to determine linked-read +// technology", since 10X has no self-describing marker to detect. +func TestRemoveUnlinked10XFQ_SplitsSingletonsAndNot(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + r2 := filepath.Join(dir, "r2.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\nTTTTGGGGCCCCAAAA\n"), 0o644); err != nil { + t.Fatal(err) + } + + writeTenXR1Unlinked(t, r1, [][2]string{ + {"a1", "AAAACCCCGGGGTTTT"}, + {"a2", "AAAACCCCGGGGTTTT"}, // same barcode as a1 -> linked + {"c1", "TTTTGGGGCCCCAAAA"}, // singleton + }) + writeTenXR2Unlinked(t, r2, []string{"a1", "a2", "c1"}) + + prefix := filepath.Join(dir, "out") + if err := RemoveUnlinkedFQ([]string{r1, r2}, prefix, true, true, wl); err != nil { + t.Fatalf("RemoveUnlinkedFQ (10X): %v", err) + } + + mainR1 := readGzFastqIDs(t, prefix+".R1.fq.gz") + mainR2 := readGzFastqIDs(t, prefix+".R2.fq.gz") + singleR1 := readGzFastqIDs(t, prefix+".unlinked.R1.fq.gz") + singleR2 := readGzFastqIDs(t, prefix+".unlinked.R2.fq.gz") + + hasPrefix := func(ids []string, id string) bool { + return slices.ContainsFunc(ids, func(s string) bool { return strings.HasPrefix(s, id) }) + } + + if !hasPrefix(mainR1, "a1") || !hasPrefix(mainR1, "a2") { + t.Errorf("expected a1,a2 in main R1 output, got %v", mainR1) + } + if !hasPrefix(mainR2, "a1") || !hasPrefix(mainR2, "a2") { + t.Errorf("expected a1,a2 in main R2 output, got %v", mainR2) + } + if hasPrefix(mainR1, "c1") || hasPrefix(mainR2, "c1") { + t.Errorf("singleton c1 should not be in main output, got R1=%v R2=%v", mainR1, mainR2) + } + if !hasPrefix(singleR1, "c1") || !hasPrefix(singleR2, "c1") { + t.Errorf("expected c1 in singleton output, got R1=%v R2=%v", singleR1, singleR2) + } + if _, err := os.Stat(prefix + ".count"); err != nil { + t.Errorf("expected barcode count file to be created: %v", err) + } +} + +func TestRemoveUnlinked10XFQ_SingleEnd(t *testing.T) { + dir := t.TempDir() + r1 := filepath.Join(dir, "r1.fq") + wl := filepath.Join(dir, "whitelist.txt") + if err := os.WriteFile(wl, []byte("AAAACCCCGGGGTTTT\n"), 0o644); err != nil { + t.Fatal(err) + } + writeTenXR1Unlinked(t, r1, [][2]string{ + {"a1", "AAAACCCCGGGGTTTT"}, + {"a2", "AAAACCCCGGGGTTTT"}, + }) + + prefix := filepath.Join(dir, "out") + if err := RemoveUnlinkedFQ([]string{r1}, prefix, false, false, wl); err != nil { + t.Fatalf("RemoveUnlinkedFQ (10X single-end): %v", err) + } + if _, err := os.Stat(prefix + ".R2.fq.gz"); err == nil { + t.Error("single-end 10X input must not produce an R2 output file") + } + mainR1 := readGzFastqIDs(t, prefix+".R1.fq.gz") + if len(mainR1) != 2 { + t.Errorf("got %d records in main output, want 2: %v", len(mainR1), mainR1) + } +}