From e542c0e40dbd905cb7d47646cec8af5779383f50 Mon Sep 17 00:00:00 2001
From: "Daniel.Frisinghelli" <daniel.frisinghelli@eurac.edu>
Date: Tue, 22 Jun 2021 12:00:37 +0200
Subject: [PATCH] Check parallel implementation.
---
climax/main/download_ERA5.py | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/climax/main/download_ERA5.py b/climax/main/download_ERA5.py
index d19de35..2dc62ac 100644
--- a/climax/main/download_ERA5.py
+++ b/climax/main/download_ERA5.py
@@ -55,12 +55,15 @@ if __name__ == '__main__':
c = cdsapi.Client()
# download data for the different variables
- for var in variables:
- c.retrieve(product, {**CONFIG, **{'variable': var}}, str(
- target.joinpath('_'.join(['ERA5', var, years[0], years[-1]])
- + '.nc')))
- # Parallel(n_jobs=min(len(variables), os.cpu_count()), verbose=51)(
- # delayed(c.retrieve)(product, {**CONFIG, **{'variable': var}}, str(
+ # sequential implementation
+ # for var in variables:
+ # c.retrieve(product, {**CONFIG, **{'variable': var}}, str(
# target.joinpath('_'.join(['ERA5', var, years[0], years[-1]])
- # + '.nc'))) for var in variables)
+ # + '.nc')))
+
+ # parallel implementation
+ Parallel(n_jobs=min(len(variables), os.cpu_count()), verbose=51)(
+ delayed(c.retrieve)(product, {**CONFIG, **{'variable': var}}, str(
+ target.joinpath('_'.join(['ERA5', var, years[0], years[-1]])
+ + '.nc'))) for var in variables)
--
GitLab