Merge pull request #1178: Optimize rule combine_samples

Author: victorlin

docs/src/reference/change_log.md

@@ -5,6 +5,8 @@ We also use this change log to document new features that maintain backward comp
 
 ## New features since last version update
 
+- 29 July 2025: Improved performance of calls to `augur filter`. This requires a minimum Augur version of 31.3.0. [PR 1178](https://github.com/nextstrain/ncov/pull/1178)
+
 ## v17 (17 July 2025)
 
 - 17 July 2025: Snakemake version 8 (or above) is now required. Various aspects of the workflow were incompatible with v8 including our support for remote files and have now been updated. The nextstrain runtimes have been correspondingly updated; see [the nextstrain-cli docs for how to upgrade these](https://docs.nextstrain.org/projects/cli/en/stable/commands/update/). [PR 1180](https://github.com/nextstrain/ncov/pull/1180)

nextstrain_profiles/100k/config-gisaid.yaml

@@ -12,6 +12,7 @@ inputs:
     metadata: "s3://nextstrain-ncov-private/metadata.tsv.zst"
     aligned: "s3://nextstrain-ncov-private/sequences.fasta.zst"
     skip_sanitize_metadata: true
+    deduplicated: true
 
 builds:
   100k:

nextstrain_profiles/100k/config-open.yaml

@@ -10,6 +10,7 @@ inputs:
     metadata: "s3://nextstrain-data/files/ncov/open/metadata.tsv.zst"
     aligned: "s3://nextstrain-data/files/ncov/open/sequences.fasta.zst"
     skip_sanitize_metadata: true
+    deduplicated: true
 builds:
   100k:
     subsampling_scheme: 100k_scheme

nextstrain_profiles/nextstrain-gisaid-21L/builds.yaml

@@ -34,6 +34,7 @@ inputs:
     metadata: "results/gisaid_21L_metadata.tsv.zst"
     aligned: "results/gisaid_21L_aligned.fasta.zst"
     skip_sanitize_metadata: true
+    deduplicated: true
 
 # Define locations for which builds should be created.
 # For each build we specify a subsampling scheme via an explicit key.

nextstrain_profiles/nextstrain-gisaid/builds.yaml

@@ -27,6 +27,7 @@ inputs:
     metadata: "s3://nextstrain-ncov-private/metadata.tsv.zst"
     aligned: "s3://nextstrain-ncov-private/aligned.fasta.zst"
     skip_sanitize_metadata: true
+    deduplicated: true
 
 # Define locations for which builds should be created.
 # For each build we specify a subsampling scheme via an explicit key.

nextstrain_profiles/nextstrain-open/builds.yaml

@@ -27,6 +27,7 @@ inputs:
     metadata: "s3://nextstrain-data/files/ncov/open/metadata.tsv.zst"
     aligned: "s3://nextstrain-data/files/ncov/open/aligned.fasta.zst"
     skip_sanitize_metadata: true
+    deduplicated: true
 
 # Define locations for which builds should be created.
 # For each build we specify a subsampling scheme via an explicit key.

workflow/schemas/config.schema.yaml

@@ -29,6 +29,8 @@ properties:
           minLength: 1
         skip_sanitize_metadata:
           type: boolean
+        deduplicated:
+          type: boolean
       additionalProperties: false
 
   builds:

workflow/snakemake_rules/main_workflow.smk

@@ -424,27 +424,26 @@ def _get_subsampled_files(wildcards):
     ]
 
 rule combine_samples:
-    message:
-        """
-        Combine and deduplicate FASTAs
-        """
     input:
         sequences=_get_unified_alignment,
         metadata=_get_unified_metadata,
         include=_get_subsampled_files,
     output:
         sequences = "results/{build_name}/{build_name}_subsampled_sequences.fasta.xz",
         metadata = "results/{build_name}/{build_name}_subsampled_metadata.tsv.xz"
+    params:
+        skip_checks = lambda w: "--skip-checks" if all(input_record.get("deduplicated", False) for input_record in config["inputs"].values()) else ""
     log:
-        "logs/subsample_regions_{build_name}.txt"
+        "logs/combine_samples_{build_name}.txt"
     benchmark:
-        "benchmarks/subsample_regions_{build_name}.txt"
+        "benchmarks/combine_samples_{build_name}.txt"
     conda: config["conda_environment"]
     shell:
         r"""
         augur filter \
             --sequences {input.sequences} \
             --metadata {input.metadata} \
+            {params.skip_checks} \
             --exclude-all \
             --include {input.include} \
             --output-sequences {output.sequences} \