extract_dt_plugin/extract_dt_plugin_pcpvars.toml at main · svianaj/extract_dt_plugin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
[general]
  case = "EXTRACT_DT_iekm_NRTpcp"

[general.plugin_registry.plugins]
  extract_dt_plugin = "/ec/res4/hpcperm/snh02/DE_Verification/plugins/"

[general.times]
  end = "2026-06-30T00:00:00Z"
  forecast_range = "PT54H"
  start = "2026-01-01T00:00:00Z"

[extract_dt]
  class_mars = "RD"
  class_polytope = "D1"
  dataset = "extremes_dt"
  delay = "PT1D14H"
  dt_grib_path = "/ec/res4/scratch/snh02/DE_Verification/GRIBS_pcp/GDT_iekm/@YYYY@/@MM@/@DD@"
  expver_mars = "iekm"
  expver_polytope = "0001"
  # area = "72.0/-26.0/26.0/30.0"
  # sfc is grid point, so we can retrieve at high resolution
  # but transforms at this resolution are too slow, so UA data at lower reso
  grid_sfc = "0.05/0.05"
  grid_ua = "0.1/0.1"
  levelist_ua = "50/100/150/200/250/300/400/500/700/850/925/1000"
  levtype_sfc = "SFC"
  levtype_ua = "PL"
  method = "mars"
  param_sfc = "228.128/228051"
  param_ua = ""
  stream = "OPER"
  type = "FC"
  paramtypes= ["sfc"]

[extractsqlite]
  selection = "PT1H"
  sqlite_model_name = "GDT_iekm"
  sqlite_path = "/ec/res4/scratch/snh02/DE_Verification/REF_FCTABLES/@SQLITE_MODEL_NAME@/@YYYY@/@MM@"
  sqlite_template = "FCTABLE_{PP}_{YYYY}{MM}_{HH}.sqlite"
  station_list_sfc = "@DEODE_HOME@/data/sqlite/station_list_default.csv"
  station_list_ua = "@DEODE_HOME@/data/sqlite/temp_list_default.csv"
  log_file = "extracted_files.log"

[[extractsqlite.parameter_list_default_det]]
  location_file = "@DEODE_HOME@/data/sqlite/station_list_default.csv"
  param_file = "/ec/res4/hpcperm/snh02/DE_Verification/plugins/extract_dt_plugin/param_list_IFSGDT_sfc.json"

[[extractsqlite.parameter_list_default_det]]
  location_file = "@DEODE_HOME@/data/sqlite/temp_list_default.csv"
  param_file = "/ec/res4/hpcperm/snh02/DE_Verification/plugins/extract_dt_plugin/param_list_IFSGDT_ua.json"

[[extractsqlite.parameter_list_default_eps]]
  location_file = "@DEODE_HOME@/data/sqlite/station_list_default.csv"
  param_file = "/ec/res4/hpcperm/snh02/DE_Verification/plugins/extract_dt_plugin/param_list_IFSGDT_sfc.json"

[[extractsqlite.parameter_list_default_eps]]
  location_file = "@DEODE_HOME@/data/sqlite/temp_list_default.csv"
  param_file = "/ec/res4/hpcperm/snh02/DE_Verification/plugins/extract_dt_plugin/param_list_IFSGDT_ua.json"

[scheduler.ecfvars]
  ecf_tries = 2

[submission.task_exceptions.RetrieveDT]
  # to find mars executable
  bindir = "/usr/local/bin/"

[submission.task_exceptions.RetrieveDT.BATCH]
  MEM = "#SBATCH --mem=0GB"
  NODES = "#SBATCH --nodes=1"
  NTASKS = "#SBATCH --ntasks=1"
  QOS = "#SBATCH --qos=np"
  WALLTIME = "#SBATCH --time=18:00:00"

[submission.task_exceptions.RetrieveDT.ENV]
  MARS_READANY_BUFFER_SIZE = 17893020000

[submission.task_exceptions.ExtractDT.BATCH]
  WALLTIME = "#SBATCH --time=03:00:00"

[suite_control]
  suite_definition = "DtExtractSuiteDefinition"