Extension of ESA CCI OZONE CMORizer (#4125)

Co-authored-by: Birgit Hassler <[email protected]>
Co-authored-by: Valeriu Predoi <[email protected]>
Co-authored-by: Bettina Gier <[email protected]>

Author: 4 people

esmvaltool/cmorizers/data/cmor_config/ESACCI-OZONE.yml

@@ -5,16 +5,40 @@ attributes:
   tier: 2
   type: sat
   project_id: OBS6
-  source: "https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1"
+  source: "https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1 and https://webdav.aeronomie.be/guest/o3_cci/webdata/Nadir_Profiles/L3/IASI_MG_FORLI/"
   reference: "esacci-ozone"
   comment: ""
 
 variables:
-  toz:
+  # Note: Do not change variable names, these are used to handle the different
+  # datasets in the formatting script esacci_ozone.py.
+  toz_gto_ecv:
+    version: L3-GTO-ECV
     mip: AERmon
+    output: toz
     raw: total_ozone_column
-    filename: C3S-L3_OZONE-O3_PRODUCTS-MERGED_UV-MERGED-ALG-MONTHLY-v2000.nc #The filename needs also "YYYYMM-" as prefix added during the cmorization
-  o3:
+    filename: "{year}{month}-C3S-L3_OZONE-O3_PRODUCTS-MERGED_UV-MERGED-ALG-MONTHLY-v2000.nc"
+  o3_sage_omps:
+    version: L3-SAGE-OMPS
     mip: AERmon
+    output: o3
     raw: merged_ozone_concentration
-    filename: C3S-L3_OZONE-O3_PRODUCTS-CONC_MZM-MERGED-ALG-MONTHLY-v0008.nc #The filename needs also "YYYYMM-" as prefix added during the cmorization
+    filename: "{year}{month}-C3S-L3_OZONE-O3_PRODUCTS-CONC_MZM-MERGED-ALG-MONTHLY-v0008.nc"
+  o3_megridop:
+    version: L3-MEGRIDOP
+    mip: AERmon
+    output: o3
+    raw: merged_ozone_concentration
+    filename: "{year}{month}-C3S-L3_OZONE-O3_PRODUCTS-CONC_LLG-MERGED-ALG-MONTHLY-v0004.nc"
+  o3_iasi:
+    version: L3-IASI
+    mip: AERmon
+    output: o3
+    raw: O3_partial_column_profile
+    filename: "IASI_FORLI_O3_MERGED_{year}{month}*_V1.0.nc"
+  toz_iasi:
+    version: L3-IASI
+    mip: AERmon
+    output: toz
+    raw: O3_total_column
+    filename: "IASI_FORLI_O3_MERGED_{year}{month}*_V1.0.nc"

esmvaltool/cmorizers/data/datasets.yml

@@ -573,34 +573,54 @@ datasets:
   ESACCI-OZONE:
     tier: 2
     source: https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1
-    last_access: 2025-02-11
+    last_access: 2026-02-12
     info: |
       Download the data from:
             GTO-ECV total column (variable toz)
             Select the following from the CDS:
             https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1?tab=download
+            Put all files under a single directory (no subdirectories with years).
 
             Processing Level = "Level 3"
             Variable = "Atm. mole content of ozone"
-            Vertical aggregation = "Total column"
-            Sensor = "MERGED-UV"
+            Vertical aggregation = "Total columns UV"
+            Sensor = "GTO-ECV"
             Year = select all (1995-2023)
             Month = select all (1-12)
             Version = "v2000"
 
             SAGE-CCI-OMPS (variable o3)
             Select the following options from the same link:
             https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1?tab=download
+            Put all files under a single directory (no subdirectories with years).
 
             Processing Level = "Level 3"
             Variable = "Mole concentration of ozone in air"
-            Vertical aggregation = " Vertical profiles from limb sensors"
-            Sensor = " CMZM (Monthly zonal mean merged concentration product from limb
-            sensors ACE, GOMOS, MIPAS, OMPS, OSIRIS, SAGE-2 and SCIAMACHY)"
+            Vertical aggregation = "Vertical profiles from limb sensors"
+            Sensor = "SAGE-CCI-OMPS_CONC"
             Year = select all (1984-2022)
             Month = select all (1-12)
             Version = "v0008"
-      Put all files under a single directory (no subdirectories with years).
+
+            MEGRIDOP (variable o3)
+            Select the following options from the same link:
+            https://cds.climate.copernicus.eu/datasets/satellite-ozone-v1?tab=download
+            Put all files under a single directory (no subdirectories with years).
+
+            Processing Level = "Level 3"
+            Variable = "Mole concentration of ozone in air"
+            Vertical aggregation = "Vertical profiles from limb sensors"
+            Sensor = "MEGRIDOP_CONC"
+            Year = select all (2001-2022)
+            Month = select all (1-12)
+            Version = "v0004"
+
+            IASI (variables o3, toz)
+            Download from BIRA WebDAV server: https://webdav.aeronomie.be
+            Path: /guest/o3_cci/webdata/Nadir_Profiles/L3/IASI_MG_FORLI/<yyyy>
+            Username: o3_cci_public
+            No password (leave empty)
+            Download each year <yyyy> into the same folder
 
   ESACCI-SEAICE:
     tier: 2

esmvaltool/cmorizers/data/downloaders/datasets/esacci_ozone.py

@@ -1,11 +1,15 @@
-"""Script to download ESACCI-OZONE from the CDS."""
+"""Script to download ESACCI-OZONE from the CDS and BIRA WebDAV."""
 
 import gzip
 import logging
 import shutil
 import zipfile
+from datetime import datetime
 
 import cdsapi
+from dateutil import relativedelta
+
+from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
 
 logger = logging.getLogger(__name__)
 
@@ -26,28 +30,61 @@ def download_dataset(
       the ECMWF account needs to be saved in user's ${HOME} directory.
     - All the files will be saved in Tier2/ESACCI-OZONE.
     """
-    cds_url = "https://cds.climate.copernicus.eu/api"
-
     if dataset == "ESACCI-OZONE":
+        output_folder = (
+            original_data_dir / f"Tier{dataset_info['tier']}" / dataset
+        )
+        output_folder.mkdir(parents=True, exist_ok=True)
+
+        cds_url = "https://cds.climate.copernicus.eu/api"
+
+        if start_date is None:
+            gto_year1 = 1995
+            omps_year1 = 1984
+            megridop_year1 = 2001
+        else:
+            gto_year1 = start_date.year
+            omps_year1 = start_date.year
+            megridop_year1 = start_date.year
+        if end_date is None:
+            gto_year2 = 2024
+            omps_year2 = 2023
+            megridop_year2 = 2023
+        else:
+            gto_year2 = end_date.year
+            omps_year2 = end_date.year
+            megridop_year2 = end_date.year
+
         requests = {
-            "toz": {
+            "toz_gto_ecv": {
                 "processing_level": "level_3",
                 "variable": "atmosphere_mole_content_of_ozone",
-                "vertical_aggregation": "total_column",
-                "sensor": ["merged_uv"],
-                "year": [str(y) for y in range(1995, 2024)],
+                "vertical_aggregation": "total_columns_uv",
+                "sensor": ["gto_ecv"],
+                "year": [str(y) for y in range(gto_year1, gto_year2)],
                 "month": [f"{m:02d}" for m in range(1, 13)],
                 "version": ["v2000"],
             },
-            "o3": {
+            "o3_sage_omps": {
                 "processing_level": "level_3",
                 "variable": "mole_concentration_of_ozone_in_air",
                 "vertical_aggregation": "vertical_profiles_from_limb_sensors",
-                "sensor": ["cmzm"],
-                "year": [str(y) for y in range(1984, 2023)],
+                "sensor": ["sage_cci_omps_conc"],
+                "year": [str(y) for y in range(omps_year1, omps_year2)],
                 "month": [f"{m:02d}" for m in range(1, 13)],
                 "version": ["v0008"],
             },
+            "o3_sage_megridop": {
+                "processing_level": "level_3",
+                "variable": "mole_concentration_of_ozone_in_air",
+                "vertical_aggregation": "vertical_profiles_from_limb_sensors",
+                "sensor": ["megridop_conc"],
+                "year": [
+                    str(y) for y in range(megridop_year1, megridop_year2)
+                ],
+                "month": [f"{m:02d}" for m in range(1, 13)],
+                "version": ["v0004"],
+            },
         }
 
         client = cdsapi.Client(cds_url)
@@ -88,5 +125,41 @@ def download_dataset(
                     with open(output_folder / file_path.stem, "wb") as f_out:
                         shutil.copyfileobj(f_in, f_out)
 
+        # download IASI data from BIRA WebDAV (IASI data not available on CDS)
+        # all the files will be saved by in ${RAWOBS}/Tier2/ESACCI-OZONE
+
+        if start_date is None:
+            start_date = datetime(2008, 1, 1)
+        if end_date is None:
+            end_date = datetime(2023, 12, 31)
+
+        downloader = WGetDownloader(
+            original_data_dir=original_data_dir,
+            dataset=dataset,
+            dataset_info=dataset_info,
+            overwrite=overwrite,
+        )
+
+        basepath = "https://webdav.aeronomie.be/guest/o3_cci/webdata/Nadir_Profiles/L3/IASI_MG_FORLI"
+
+        wget_options = [
+            "-e robots=off",  # ignore robots.txt
+            "--no-parent",  # don't ascend to the parent directory
+            "--accept=nc",  # download only *.nc files
+            "--user=o3_cci_public",  # user name
+            "--password=",  # empty password (no password needed for public access)
+        ]
+
+        loop_date = start_date
+        while loop_date <= end_date:
+            year = loop_date.year
+
+            # directory on server to download
+            remotepath = f"{basepath}/{year}"
+            downloader.download_folder(remotepath, wget_options)
+
+            loop_date += relativedelta.relativedelta(years=1)
+
     else:
-        raise ValueError(f"Unknown dataset: {dataset}")
+        errmsg = f"Unknown dataset: {dataset}"
+        raise ValueError(errmsg)