From d01720ea6cefbd008aff72ac1da12529a8573347 Mon Sep 17 00:00:00 2001
From: "Daniel.Frisinghelli" <daniel.frisinghelli@eurac.edu>
Date: Tue, 22 Jun 2021 13:01:41 +0200
Subject: [PATCH] Group preprocessing by dataset.
---
Scripts/{preprocess.sh => preprocess_CORDEX.sh} | 6 +++---
climax/main/download_ERA5.py | 2 ++
climax/main/{preprocess.py => preprocess_CORDEX.py} | 4 ++--
climax/main/preprocess_ERA5.py | 6 ++++++
4 files changed, 13 insertions(+), 5 deletions(-)
rename Scripts/{preprocess.sh => preprocess_CORDEX.sh} (63%)
rename climax/main/{preprocess.py => preprocess_CORDEX.py} (98%)
create mode 100644 climax/main/preprocess_ERA5.py
diff --git a/Scripts/preprocess.sh b/Scripts/preprocess_CORDEX.sh
similarity index 63%
rename from Scripts/preprocess.sh
rename to Scripts/preprocess_CORDEX.sh
index edd3001..f39aa96 100644
--- a/Scripts/preprocess.sh
+++ b/Scripts/preprocess_CORDEX.sh
@@ -17,10 +17,10 @@ PR_INVENTORY='/mnt/CEPH_PROJECTS/FACT_CLIMAX/CORDEX/Inventory/pr/pr_inventory_se
# preprocess EuroCordex simulations
# tasmin
-time python climax/main/preprocess.py $TARGET_GRID $SOURCE_PATH $TARGET_PATH -f $TASMIN_INVENTORY -m 'bilinear' -rm -a -o
+time python climax/main/preprocess_CORDEX.py $TARGET_GRID $SOURCE_PATH $TARGET_PATH -f $TASMIN_INVENTORY -m 'bilinear' -rm -a -o
# tasmax
-time python climax/main/preprocess.py $TARGET_GRID $SOURCE_PATH $TARGET_PATH -f $TASMAX_INVENTORY -m 'bilinear' -rm -a -o
+time python climax/main/preprocess_CORDEX.py $TARGET_GRID $SOURCE_PATH $TARGET_PATH -f $TASMAX_INVENTORY -m 'bilinear' -rm -a -o
# pr
-time python climax/main/preprocess.py $TARGET_GRID $SOURCE_PATH $TARGET_PATH -f $PR_INVENTORY -m 'bilinear' -rm -a -o
\ No newline at end of file
+time python climax/main/preprocess_CORDEX.py $TARGET_GRID $SOURCE_PATH $TARGET_PATH -f $PR_INVENTORY -m 'bilinear' -rm -a -o
\ No newline at end of file
diff --git a/climax/main/download_ERA5.py b/climax/main/download_ERA5.py
index 4835c97..82a2bfd 100644
--- a/climax/main/download_ERA5.py
+++ b/climax/main/download_ERA5.py
@@ -82,6 +82,8 @@ if __name__ == '__main__':
product, {**CONFIG, **{'variable': var, 'year': year}}, file)
for file, year in zip(files, years) if not file.exists())
+ # TODO: Move to preprocess_ERA5.py
+
# aggregate files for different years into a single file using xarray
# and dask
ds = xr.open_mfdataset(files, parallel=True).compute()
diff --git a/climax/main/preprocess.py b/climax/main/preprocess_CORDEX.py
similarity index 98%
rename from climax/main/preprocess.py
rename to climax/main/preprocess_CORDEX.py
index 6f24d00..446fb68 100644
--- a/climax/main/preprocess.py
+++ b/climax/main/preprocess_CORDEX.py
@@ -20,7 +20,7 @@ from pysegcnn.core.logging import log_conf
from pysegcnn.core.trainer import LogConfig
from climax.core.utils import (get_inventory, reproject_cdo,
_parse_cordex_time_span)
-from climax.core.cli import preprocess_parser
+from climax.core.cli import preprocess_cordex_parser
from climax.core.constants import EUROCORDEX_DOMAIN, CORDEX_PARAMETERS
# module level logger
@@ -47,7 +47,7 @@ if __name__ == '__main__':
dictConfig(log_conf())
# define command line argument parser
- parser = preprocess_parser()
+ parser = preprocess_cordex_parser()
# parse command line arguments
args = sys.argv[1:]
diff --git a/climax/main/preprocess_ERA5.py b/climax/main/preprocess_ERA5.py
new file mode 100644
index 0000000..303f1d4
--- /dev/null
+++ b/climax/main/preprocess_ERA5.py
@@ -0,0 +1,6 @@
+"""Preprocess ERA-5 data: aggregate to daily and resample to target grid."""
+
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
--
GitLab