Merge pull request #373 from JosuaCarl/provenance-file

Refactor: Move to provenance file and update related tests

Author: JosuaCarl

docs/assets/co2footprint_report_sample.html

@@ -24,7 +24,7 @@ nf-co2footprint options:
   powerdrawMem: 0.3725
   pue: 1.3
   customCpuTdpFile: null
-  dataFile: out/pipeline_info/co2footprint_data_2026-03-06_13-25-02.yaml
+  provenanceFile: out/pipeline_info/co2footprint_data_2026-03-06_13-25-02.yaml
   reportFile: out/pipeline_info/co2footprint_report_2026-03-06_13-25-02.html
   summaryFile: out/pipeline_info/co2footprint_summary_2026-03-06_13-25-02.txt
   traceFile: out/pipeline_info/co2footprint_trace_2026-03-06_13-25-02.txt

docs/assets/co2footprint_summary_sample.txt

@@ -16,8 +16,8 @@ The nf-co2footprint plugin creates three output files:
 - **`report`** ([sample](../assets/co2footprint_report_sample.html))  
   The HTML report contains information about the carbon footprint of the whole pipeline run as well as plots showing the distributions of the CO₂ emissions for the different processes. The CO₂ emissions are separated into newly generated (i.e. from non-cached tasks) and total (including cached tasks). Additionally, it contains a table with the metrics for all individual tasks. The table is limited to 10000 entries by default. It finishes up with an overview plot of the carbon intensities during the workflow execution.
 
-- **`dataFile`**
-  The data file contains all trace information in a tree structure with the levels in descending order `session -> workflow -> process -> task`. Example: A workflow consists of multiple processes / has multiple `process` level children.
+- **`provenance`**
+  The provenance file contains all trace information in a tree structure with the levels in descending order `session -> workflow -> process -> task`. Example: A workflow consists of multiple processes / has multiple `process` level children.
   The total session emission estimation includes everything from the point of plugin start to stop in a similar manner to how Nextflow defines a `TraceRecord`, but through the Java-native OSHI library.
 
     !!! note "Comparison session vs. workflow"

docs/usage/output.md

@@ -32,14 +32,14 @@ The following parameters are currently available:
 
     **Default**: `[enabled: true, file: co2footprint_report_<timestamp>.html, overwrite: true, maxTasks: 10000]`
 
-- **`dataFile`**  
+- **`provenance`**  
   Map containing:
     - `enabled`: Whether to produce this file
     - `file`: Name of the data/machine-actionable file containing all metrics that were used during footprint calculation in a structured way.
     - `overwrite`: Whether to overwrite the file, if it already exists.
     - `emissionMetricsOnly`: Whether to only include emission metrics, such as CO₂ equivalents and electricity consumption.
 
-    **Default**: `[enabled: false, file: co2footprint_report_<timestamp>.html, overwrite: true, emissionMetricsOnly: false]`
+    **Default**: `[enabled: false, file: co2footprint_report_<timestamp>.html, overwrite: true, emissionMetricsOnly: true]`
 
     !!! warning "Preliminary feature"
         The data file is currently not in its final version. Changes in the near future are very likely.

docs/usage/parameters.md

@@ -1,7 +1,7 @@
 package nextflow.co2footprint
 
 import groovy.util.logging.Slf4j
-import nextflow.co2footprint.Config.DataFileConfig
+import nextflow.co2footprint.Config.ProvenanceFileConfig
 import nextflow.co2footprint.Config.ReportFileConfig
 import nextflow.co2footprint.Config.SummaryFileConfig
 import nextflow.co2footprint.Config.TraceFileConfig
@@ -70,8 +70,8 @@ class CO2FootprintConfig implements ConfigScope {
     final ReportFileConfig report
 
     @ConfigOption(types=[Map])
-    @Description('Configuration for the data/machine-readable file.')
-    final DataFileConfig dataFile
+    @Description('Configuration for the provenance data/machine-readable file.')
+    final ProvenanceFileConfig provenance
 
     @ConfigOption(types=[GString])
     @Description('Location GeoCode from Electricity maps.')
@@ -136,7 +136,7 @@ class CO2FootprintConfig implements ConfigScope {
         trace = new TraceFileConfig(getCollect('trace', configMap, usedKeys) as Map ?: [:], timestamp)
         summary = new SummaryFileConfig(getCollect('summary', configMap, usedKeys) as Map ?: [:], timestamp)
         report = new ReportFileConfig(getCollect('report', configMap, usedKeys) as Map ?: [:], timestamp)
-        dataFile = new DataFileConfig(getCollect('dataFile', configMap, usedKeys) as Map ?: [:], timestamp)
+        provenance = new ProvenanceFileConfig(getCollect('provenance', configMap, usedKeys) as Map ?: [:], timestamp)
 
         // Location
         location = getCollect('location', configMap, usedKeys) as String ?: getLocationFromAWSRegion()
@@ -317,7 +317,7 @@ class CO2FootprintConfig implements ConfigScope {
             reportFile: report.file,
             summaryFile: summary.file,
             traceFile: trace.file,
-            dataFile: dataFile.file
+            provenanceFile: provenance.file
         ].sort() as SortedMap
     }
 

src/main/nextflow/co2footprint/CO2FootprintConfig.groovy

@@ -3,7 +3,7 @@ package nextflow.co2footprint
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.Session
-import nextflow.co2footprint.FileCreation.DataFileCreator
+import nextflow.co2footprint.FileCreation.ProvenanceFileCreator
 import nextflow.co2footprint.FileCreation.ReportFileCreator
 import nextflow.co2footprint.FileCreation.SummaryFileCreator
 import nextflow.co2footprint.FileCreation.TraceFileCreator
@@ -38,7 +38,7 @@ class CO2FootprintObserver implements TraceObserver {
     final TraceFileCreator traceFile
     final SummaryFileCreator summaryFile
     final ReportFileCreator reportFile
-    final DataFileCreator dataFile
+    final ProvenanceFileCreator provenanceFile
 
     // Plugin configuration
     CO2FootprintConfig config
@@ -77,9 +77,9 @@ class CO2FootprintObserver implements TraceObserver {
         this.traceFile = new TraceFileCreator(config.trace)
         this.summaryFile = new SummaryFileCreator(config.summary)
         this.reportFile = new ReportFileCreator(config.report)
-        this.dataFile = new DataFileCreator(config.dataFile)
+        this.provenanceFile = new ProvenanceFileCreator(config.provenance)
 
-        if (!config.trace.enabled && !config.summary.enabled && !config.report.enabled && !config.dataFile.enabled) {
+        if (!config.trace.enabled && !config.summary.enabled && !config.report.enabled && !config.provenance.enabled) {
             log.warn('No output files are enabled - to enable, set `enabled: true` in the sections `trace`, `summary` or `report`.')
         }
 
@@ -164,15 +164,15 @@ class CO2FootprintObserver implements TraceObserver {
             reportFile.write()
         }
         if (co2RecordTree) {
-            dataFile.create()
-            dataFile.write(co2RecordTree)
+            provenanceFile.create()
+            provenanceFile.write(co2RecordTree)
         }
 
         // Close all files (writes remaining tasks in the trace file)
         traceFile.close(runningTasks)
         summaryFile.close()
         reportFile.close()
-        dataFile.close()
+        provenanceFile.close()
     }
 
     // ------ OBSERVER METHODS ------
@@ -221,8 +221,8 @@ class CO2FootprintObserver implements TraceObserver {
         workflowStats.summarize()
 
         log.info(
-            "🌱 The workflow run used ${workflowStats.co2Record.toReadable('energy')} of electricity, " +
-            "resulting in the release of ${workflowStats.co2Record.toReadable('co2e')} of CO₂ equivalents into the atmosphere."
+            "🌱 The workflow run used ${workflowStats.co2Record.toReadable('energy_consumption')} of electricity, " +
+            "resulting in the release of ${workflowStats.co2Record.toReadable('CO2e')} of CO₂ equivalents into the atmosphere."
         )
     }
 

src/main/nextflow/co2footprint/CO2FootprintObserver.groovy

@@ -0,0 +1,26 @@
+package nextflow.co2footprint.Config
+
+import nextflow.co2footprint.CO2FootprintConfig
+import nextflow.config.spec.ConfigOption
+import nextflow.config.spec.ConfigScope
+import nextflow.config.spec.ScopeName
+import nextflow.script.dsl.Description
+
+@ScopeName('co2footprint.provenance')
+@Description('The `co2footprint.provenance` scope allows you to configure the data/machine-actionable file of the `nf-co2footprint` plugin.')
+class ProvenanceFileConfig extends BaseFileConfig implements ConfigScope {
+
+    @ConfigOption
+    @Description('Whether only emission metrics should be reported in the data file.')
+    final boolean emissionMetricsOnly
+
+    ProvenanceFileConfig(Map provenanceFileConfig, String timestamp=null) {
+        super(provenanceFileConfig, timestamp, 'data', 'yaml', false)
+
+        emissionMetricsOnly = provenanceFileConfig.containsKey('emissionMetricsOnly') ?
+                CO2FootprintConfig.getCollect('emissionMetricsOnly', provenanceFileConfig, usedKeys) as boolean :
+                true
+
+        CO2FootprintConfig.checkKeyUsage(provenanceFileConfig, usedKeys)
+    }
+}

src/main/nextflow/co2footprint/Config/DataFileConfig.groovy

@@ -1,12 +1,12 @@
 package nextflow.co2footprint.FileCreation
 
 import groovyx.gpars.agent.Agent
-import nextflow.co2footprint.Config.DataFileConfig
+import nextflow.co2footprint.Config.ProvenanceFileConfig
 import nextflow.co2footprint.Records.CO2RecordTree
 import nextflow.trace.TraceHelper
 import org.yaml.snakeyaml.Yaml
 
-class DataFileCreator extends BaseFileCreator {
+class ProvenanceFileCreator extends BaseFileCreator {
     // Yaml string creator
     Yaml yaml = new Yaml()
 
@@ -19,9 +19,9 @@ class DataFileCreator extends BaseFileCreator {
     /**
      * Constructor for the data/machine-readable file.
      *
-     * @param config A {@link DataFileConfig} that defines the created file.
+     * @param config A {@link ProvenanceFileConfig} that defines the created file.
      */
-    DataFileCreator(DataFileConfig config) {
+    ProvenanceFileCreator(ProvenanceFileConfig config) {
         super(config)
 
         emissionMetricsOnly = config.emissionMetricsOnly

src/main/nextflow/co2footprint/Config/ProvenanceFileConfig.groovy

@@ -185,14 +185,14 @@ class ReportFileCreator extends BaseFileCreator{
         ).sum() as CO2Record
 
         // Retrieve total CO₂ emissions and energy consumption for the given suffix
-        Double co2e = workflowRecord.get("co2e${suffix}") as Double
-        Double energy = workflowRecord.get("energy${suffix}") as Double
+        Double co2e = workflowRecord.get("CO2e${suffix}") as Double
+        Double energy = workflowRecord.get("energy_consumption${suffix}") as Double
 
         if (co2e != null) {
             CO2EquivalencesRecord equivalences = co2FootprintComputer.computeCO2footprintEquivalences(co2e)
             return [
-                ("co2e${suffix}" as String): new Quantity(co2e,'', 'g').toReadable(),
-                ("energy${suffix}" as String): new Quantity(energy,'k','Wh').toReadable(),
+                ("CO2e${suffix}" as String): new Quantity(co2e,'', 'g').toReadable(),
+                ("energy_consumption${suffix}" as String): new Quantity(energy,'k','Wh').toReadable(),
                 ("car${suffix}" as String): equivalences.getCarKilometersReadable(),
                 ("tree${suffix}" as String): equivalences.getTreeMonthsReadable(),
                 ("plane_percent${suffix}" as String): equivalences.getPlanePercent() < 100.0 ? equivalences.getPlanePercentReadable() : null,
@@ -237,7 +237,7 @@ class ReportFileCreator extends BaseFileCreator{
      */
     protected Map<String, Object> collectSummary(CO2RecordTree stats=this.stats) {
         // Add an empty map if the process is not already present
-        return stats.collectByLevel('process', ['co2e', 'energy', 'co2e_non_cached', 'energy_non_cached'])
+        return stats.collectByLevel('process', ['CO2e', 'energy_consumption', 'CO2e_non_cached', 'energy_consumption_non_cached'])
     }