From f655da60a47c95e1a9cb9470ae95b78be3df99a9 Mon Sep 17 00:00:00 2001
From: "Daniel.Frisinghelli" <daniel.frisinghelli@eurac.edu>
Date: Thu, 21 Oct 2021 17:25:10 +0200
Subject: [PATCH] Setup for hyperparameter grid search.
---
Scripts/downscale.sh | 63 -------------------------------
Scripts/grid_search.sh | 86 ++++++++++++++++++++++++++++++++++++++++++
climax/main/config.py | 50 ++++++++++++++++--------
3 files changed, 121 insertions(+), 78 deletions(-)
delete mode 100644 Scripts/downscale.sh
create mode 100644 Scripts/grid_search.sh
diff --git a/Scripts/downscale.sh b/Scripts/downscale.sh
deleted file mode 100644
index d607826..0000000
--- a/Scripts/downscale.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-
-# activate conda environment
-conda activate climax
-
-# move to project repository
-cd ~/git/climax
-
-# loss functions
-# LOSS=(L1Loss BernoulliGammaLoss BernoulliWeibullLoss MSELoss)
-LOSS=(L1Loss BernoulliGammaLoss MSELoss)
-
-# wet day thresholds to test
-# WET_DAY_THRESHOLDS=(0 0.5 1 2 3 5)
-
-# weight decay values to test
-LAMBDA=(0 0.000001 0.00001 0.0001 0.001 0.01 1)
-
-# learning rate values to test
-LR=(0.1 0.01 0.005 0.001 0.0005)
-
-# iterate over loss functions
-for loss in ${LOSS[@]}; do
-
- # change loss function in configuration
- if [ "$loss" = "L1Loss" ] || [ "$loss" = "MSELoss" ]; then
- sed -i "s/LOSS\s*=.*/LOSS=$loss()/" ./climax/main/config.py
- else
- sed -i "s/LOSS\s*=.*/LOSS=$loss(min_amount=1)/" ./climax/main/config.py
- fi
-
- # iterate over weight decay values
- for lambda in ${LAMBDA[@]}; do
- # change weight regularization in configuration
- sed -i "s/'weight_decay':.*/'weight_decay': $lambda/" ./climax/main/config.py
-
- # run downscaling
- # python climax/main/downscale.py
- python climax/main/downscale_train.py
- python climax/main/downscale_infer.py
- done
-
- # iterate over learning rate values
- for lr in ${LR[@]}; do
- # change weight regularization in configuration
- sed -i "s/'lr':.*/'lr': $lr/" ./climax/main/config.py
-
- # run downscaling
- # python climax/main/downscale.py
- python climax/main/downscale_train.py
- python climax/main/downscale_infer.py
- done
-
- # for w in ${WET_DAY_THRESHOLDS[@]}; do
- # change wet day threshold in configuration
- # sed -i "s/min_amount\s*=.*/min_amount=$w)/" ./climax/main/config.py
-
- # run downscaling
- # python climax/main/downscale.py
- # python climax/main/downscale_train.py
- # python climax/main/downscale_infer.py
- # done
-done
diff --git a/Scripts/grid_search.sh b/Scripts/grid_search.sh
new file mode 100644
index 0000000..b65f43e
--- /dev/null
+++ b/Scripts/grid_search.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+# activate conda environment
+conda activate climax
+
+# move to project repository
+cd ~/git/climax
+
+# predictands
+# PREDICTAND=(pr tasmin tasmax)
+PREDICTAND=(pr)
+
+# optimizers
+# OPTIM=(torch.optim.Adam torch.optim.SGD)
+OPTIM=(torch.optim.Adam)
+
+# learning rate scheduler
+LRSCHEDULER=(None torch.optim.lr_scheduler.CyclicLR)
+
+# wet day thresholds to test
+# WET_DAY_THRESHOLDS=(0 0.5 1 2 3 5)
+
+# weight decay values to test
+LAMBDA=(0 0.000001 0.00001 0.0001 0.001 0.01 1)
+
+# iterate over predictands
+for predictand in ${PREDICTAND[@]}; do
+
+ # change predictand in configuration
+ sed -i "s/PREDICTAND\s*=.*/PREDICTAND='$predictand'/" ./climax/main/config.py
+
+ # define available loss functions for current predictand
+ if [ "$predictand" = "pr" ]; then
+ LOSS=(L1Loss BernoulliGammaLoss MSELoss)
+ else
+ LOSS=(L1Loss MSELoss)
+ fi
+
+ # iterate over loss functions
+ for loss in ${LOSS[@]}; do
+
+ # change loss function in configuration
+ if [ "$loss" = "L1Loss" ] || [ "$loss" = "MSELoss" ]; then
+ sed -i "s/LOSS\s*=.*/LOSS=$loss()/" ./climax/main/config.py
+ else
+ sed -i "s/LOSS\s*=.*/LOSS=$loss(min_amount=1)/" ./climax/main/config.py
+ fi
+
+ # iterate over the optimizer
+ for optim in ${OPTIM[@]}; do
+
+ # change optimizer in configuration
+ sed -i "s/OPTIM\s*=.*/OPTIM=$optim/" ./climax/main/config.py
+
+ # SGD with fixed and cyclic learning rate policy
+ if [ "$optim" = "torch.optim.SGD" ]; then
+ for scheduler in ${LRSCHEDULER[@]}; do
+ # change learning rate scheduler in configuration
+ sed -i "s/LR_SCHEDULER\s*=.*/LR_SCHEDULER=$scheduler/" ./climax/main/config.py
+
+ # iterate over weight decay values
+ for lambda in ${LAMBDA[@]}; do
+ # change weight regularization in configuration
+ sed -i "s/'weight_decay':.*/'weight_decay': $lambda/" ./climax/main/config.py
+
+ # run downscaling
+ # python climax/main/downscale.py
+ python climax/main/downscale_train.py
+ python climax/main/downscale_infer.py
+ done
+ done
+ else
+ # iterate over weight decay values
+ for lambda in ${LAMBDA[@]}; do
+ # change weight regularization in configuration
+ sed -i "s/'weight_decay':.*/'weight_decay': $lambda/" ./climax/main/config.py
+
+ # run downscaling
+ # python climax/main/downscale.py
+ python climax/main/downscale_train.py
+ python climax/main/downscale_infer.py
+ done
+ fi
+ done
+ done
+done
diff --git a/climax/main/config.py b/climax/main/config.py
index c2dc303..1ca227b 100644
--- a/climax/main/config.py
+++ b/climax/main/config.py
@@ -123,29 +123,49 @@ LOSS = MSELoss()
# LOSS = BernoulliGammaLoss(min_amount=1)
# LOSS = BernoulliWeibullLoss(min_amount=1)
-# batch size: number of time steps processed by the net in each iteration
-BATCH_SIZE = 16
-
-# base learning rate: constant or CyclicLR policy
-BASE_LR = 1e-4
-
-# maximum learning rate for CyclicLR policy
-MAX_LR = 1e-3
-
# stochastic optimization algorithm
# OPTIM = torch.optim.SGD
OPTIM = torch.optim.Adam
+
+# batch size: number of time steps processed by the net in each iteration
+BATCH_SIZE = 16
+
+# maximum learning rate determined from learning rate range test
+if PREDICTAND == 'tasmin':
+ if isinstance(LOSS, L1Loss):
+ MAX_LR = 0.001 if OPTIM is torch.optim.Adam else 0.004
+ if isinstance(LOSS, MSELoss):
+ MAX_LR = 0.001 if OPTIM is torch.optim.Adam else 0.002
+
+if PREDICTAND == 'tasmax':
+ if isinstance(LOSS, L1Loss):
+ MAX_LR = 0.001
+ if isinstance(LOSS, MSELoss):
+ MAX_LR = 0.001 if OPTIM is torch.optim.Adam else 0.004
+
+if PREDICTAND == 'pr':
+ if isinstance(LOSS, L1Loss):
+ MAX_LR = 0.001
+ if isinstance(LOSS, MSELoss):
+ MAX_LR = 0.0004
+ if isinstance(LOSS, BernoulliGammaLoss):
+ MAX_LR = 0.0005 if OPTIM is torch.optim.Adam else 0.001
+
+# base learning rate: MAX_LR / 4 (Smith L. (2017))
+BASE_LR = MAX_LR / 4
+
+# optimization parameters
OPTIM_PARAMS = {'lr': BASE_LR, 'weight_decay': 0}
-if OPTIM == torch.optim.SGD:
+if OPTIM is torch.optim.SGD:
OPTIM_PARAMS['momentum'] = 0.99 # SGD with momentum
# learning rate scheduler: CyclicLR policy
LR_SCHEDULER = None
# LR_SCHEDULER = torch.optim.lr_scheduler.CyclicLR
-LR_SCHEDULER_PARAMS = {'base_lr': BASE_LR, 'max_lr': MAX_LR,
- 'mode': 'triangular', 'step_size_up': 400,
- 'cycle_momentum': False, 'base_momentum': 0.9,
- 'max_momentum': 0.99}
+LR_SCHEDULER_PARAMS = {
+ 'base_lr': BASE_LR, 'max_lr': MAX_LR, 'mode': 'triangular',
+ 'cycle_momentum': True if OPTIM is torch.optim.SGD else False,
+ 'base_momentum': 0.9, 'max_momentum': 0.99, 'step_size_up': 400}
# whether to randomly shuffle time steps or to conserve time series for model
# training
@@ -157,7 +177,7 @@ NORM = True
# network training configuration
TRAIN_CONFIG = {
'checkpoint_state': {},
- 'epochs': 50,
+ 'epochs': 100,
'save': True,
'save_loaders': False,
'early_stop': True,
--
GitLab