From 7a4d8a183fe25a5bad4b15de6ce4fdb64af33688 Mon Sep 17 00:00:00 2001
From: "Daniel.Frisinghelli" <daniel.frisinghelli@eurac.edu>
Date: Thu, 7 Jan 2021 17:40:49 +0100
Subject: [PATCH] Implemented the interface to the ALCD dataset.
---
pysegcnn/core/constants.py | 104 ++++++++++---------------------------
pysegcnn/core/dataset.py | 104 ++++++++++++++-----------------------
pysegcnn/core/trainer.py | 29 +++++------
pysegcnn/main/config.py | 35 ++++++++-----
4 files changed, 102 insertions(+), 170 deletions(-)
diff --git a/pysegcnn/core/constants.py b/pysegcnn/core/constants.py
index 4161956..08a5a9c 100644
--- a/pysegcnn/core/constants.py
+++ b/pysegcnn/core/constants.py
@@ -105,24 +105,6 @@ class LabelMapping(dict):
return np.array(list(self.items()))
-class Gdal2Numpy(enum.Enum):
- """Data type mapping from gdal to numpy."""
-
- Byte = np.uint8
- UInt8 = np.uint8
- Int8 = np.int8
- UInt16 = np.uint16
- Int16 = np.int16
- UInt32 = np.uint32
- Int32 = np.int32
- Float32 = np.float32
- Float64 = np.float64
- CInt16 = np.complex64
- CInt32 = np.complex64
- CFloat32 = np.complex64
- CFloat64 = np.complex64
-
-
class SparcsLabels(Label):
"""Class labels of the `Sparcs`_ dataset.
@@ -153,67 +135,37 @@ class Cloud95Labels(Label):
Cloud = 1, 'white'
-class ProSnowLabels(Label):
- """Class labels of the ProSnow datasets."""
+class AlcdLabels(Label):
+ """Class labels of the `Alcd`_ dataset.
- Cloud = 0, 'white'
- Snow = 1, 'lightblue'
- Snow_free = 2, 'sienna'
- No_data = 3, 'black'
+ .. _Alcd:
+ https://zenodo.org/record/1460961#.XYCTRzYzaHt
+ """
-def map_labels(source, target):
- """Map labels from a source domain to a target domain.
-
- Parameters
- ----------
- source : :py:class:`enum.EnumMeta`
- The source domain labels, i.e. the labels a model is trained with.
- target : :py:class:`enum.EnumMeta`
- The target domain labels, i.e. the labels of the dataset to apply the
- model to.
+ No_data = 0, 'black'
+ Not_used = 1, 'black'
+ Cloud = 2, 'white'
+ Cirrus = 3, 'white'
+ Shadow = 4, 'grey'
+ Land = 5, 'forestgreen'
+ Water = 6, 'blue'
+ Snow = 7, 'lightblue'
- Raises
- ------
- ValueError
- Raised if no label mapping from ``source`` to ``target`` is defined.
- Returns
- -------
- label_map : `dict` [`int`, `int`]
- Dictionary with source labels as keys and corresponding target labels
- as values.
+class Gdal2Numpy(enum.Enum):
+ """Data type mapping from gdal to numpy."""
- """
- # if source and target labels are equal, the label mapping is the
- # identity
- if source is target:
- label_map = None
-
- # mapping from Sparcs to ProSnow
- elif source is SparcsLabels and target is ProSnowLabels:
- # label transformation mapping
- label_map = {
- # Shadow = Snow Free
- SparcsLabels.Shadow.id: ProSnowLabels.Snow_free.id,
- # Shadow ow = Snow Free
- SparcsLabels.Shadow_over_water.id: ProSnowLabels.Snow_free.id,
- # Water = Snow Free
- SparcsLabels.Water.id: ProSnowLabels.Snow_free.id,
- # Snow = Snow
- SparcsLabels.Snow.id: ProSnowLabels.Snow.id,
- # Land = Snow Free
- SparcsLabels.Land.id: ProSnowLabels.Snow_free.id,
- # Cloud = Cloud
- SparcsLabels.Cloud.id: ProSnowLabels.Cloud.id,
- # Flooded = Snow Free
- SparcsLabels.Flooded.id: ProSnowLabels.Snow_free.id,
- # No data = No data
- SparcsLabels.No_data.id: ProSnowLabels.No_data.id
- }
-
- else:
- raise ValueError('Unknown label mapping from {} to {}'.
- format(source.__name__, target.__name__))
-
- return label_map
+ Byte = np.uint8
+ UInt8 = np.uint8
+ Int8 = np.int8
+ UInt16 = np.uint16
+ Int16 = np.int16
+ UInt32 = np.uint32
+ Int32 = np.int32
+ Float32 = np.float32
+ Float64 = np.float64
+ CInt16 = np.complex64
+ CInt32 = np.complex64
+ CFloat32 = np.complex64
+ CFloat64 = np.complex64
diff --git a/pysegcnn/core/dataset.py b/pysegcnn/core/dataset.py
index da0c02d..fdd0df9 100644
--- a/pysegcnn/core/dataset.py
+++ b/pysegcnn/core/dataset.py
@@ -28,6 +28,7 @@ import csv
import enum
import itertools
import logging
+import pathlib
# externals
import numpy as np
@@ -35,9 +36,9 @@ import torch
from torch.utils.data import Dataset
# locals
-from pysegcnn.core.constants import (Landsat8, Sentinel2, Label, SparcsLabels,
- Cloud95Labels, ProSnowLabels,
- MultiSpectralSensor)
+from pysegcnn.core.constants import (MultiSpectralSensor, Landsat8, Sentinel2,
+ Label, SparcsLabels, Cloud95Labels,
+ AlcdLabels)
from pysegcnn.core.utils import (img2np, is_divisible, tile_topleft_corner,
parse_landsat_scene, parse_sentinel2_scene)
@@ -54,7 +55,7 @@ class ImageDataset(Dataset):
Attributes
----------
- root_dir : `str`
+ root_dir : `str` or :py:class:`pathlib.Path`
The root directory, path to the dataset.
use_bands : `list` [`str`]
List of the spectral bands to use during model training.
@@ -118,7 +119,7 @@ class ImageDataset(Dataset):
Parameters
----------
- root_dir : `str`
+ root_dir : `str` or :py:class:`pathlib.Path`
The root directory, path to the dataset.
use_bands : `list` [`str`], optional
A list of the spectral bands to use. The default is `[]`.
@@ -160,7 +161,7 @@ class ImageDataset(Dataset):
super().__init__()
# dataset configuration
- self.root = root_dir
+ self.root = pathlib.Path(root_dir)
self.use_bands = use_bands
self.tile_size = tile_size
self.pad = pad
@@ -222,7 +223,7 @@ class ImageDataset(Dataset):
LOGGER.info('Adding label "No data" with value={} to ground truth.'
.format(self.cval))
else:
- self._labels.pop(self.cval)
+ # self._labels.pop(self.cval)
self.cval = 0
# remove labels to merge from dataset instance labels
@@ -812,8 +813,7 @@ class StandardEoDataset(ImageDataset):
# check if the current directory name matches a scene identifier
scene = self.parse_scene_id(dirpath)
-
- if scene is not None:
+ if scene:
# get the date of the current scene
date = scene['date']
@@ -897,7 +897,7 @@ class SparcsDataset(StandardEoDataset):
@staticmethod
def get_sensor():
- """Landsat 8 bands of the Sparcs dataset.
+ """Landsat-8 bands of the Sparcs dataset.
Returns
-------
@@ -921,7 +921,7 @@ class SparcsDataset(StandardEoDataset):
@staticmethod
def parse_scene_id(scene_id):
- """Parse Sparcs scene identifiers (Landsat 8).
+ """Parse Sparcs scene identifiers (Landsat-8).
Parameters
----------
@@ -938,8 +938,16 @@ class SparcsDataset(StandardEoDataset):
return parse_landsat_scene(scene_id)
-class ProSnowDataset(StandardEoDataset):
- """Class for the ProSnow datasets."""
+class AlcdDataset(StandardEoDataset):
+ """Class for the `Alcd`_ dataset by `Baetens et al. (2019)`_.
+
+ .. _Alcd:
+ https://zenodo.org/record/1460961#.XYCTRzYzaHt
+
+ .. _Baetens et al. (2019):
+ https://www.mdpi.com/2072-4292/11/4/433
+
+ """
def __init__(self, root_dir, use_bands=[], tile_size=None, pad=False,
gt_pattern='(.*)gt\\.tif', sort=False, seed=0, transforms=[],
@@ -948,9 +956,21 @@ class ProSnowDataset(StandardEoDataset):
super().__init__(root_dir, use_bands, tile_size, pad, gt_pattern,
sort, seed, transforms, merge_labels)
+ @staticmethod
+ def get_size():
+ """Image size of the Alcd dataset.
+
+ Returns
+ -------
+ size : `tuple`
+ The image size (height, width).
+
+ """
+ return (1830, 1830)
+
@staticmethod
def get_sensor():
- """Sentinel 2 bands of the ProSnow datasets.
+ """Sentinel-2 bands of the Alcd dataset.
Returns
-------
@@ -962,7 +982,7 @@ class ProSnowDataset(StandardEoDataset):
@staticmethod
def get_labels():
- """Class labels of the ProSnow datasets.
+ """Class labels of the Alcd dataset.
Returns
-------
@@ -970,11 +990,11 @@ class ProSnowDataset(StandardEoDataset):
The class labels.
"""
- return ProSnowLabels
+ return AlcdLabels
@staticmethod
def parse_scene_id(scene_id):
- """Parse ProSnow scene identifiers (Sentinel 2).
+ """Parse Alcd scene identifiers (Sentinel-2).
Parameters
----------
@@ -991,57 +1011,12 @@ class ProSnowDataset(StandardEoDataset):
return parse_sentinel2_scene(scene_id)
-class ProSnowGarmisch(ProSnowDataset):
- """Class for the ProSnow Garmisch dataset."""
-
- def __init__(self, root_dir, use_bands=[], tile_size=None, pad=False,
- gt_pattern='(.*)gt\\.tif', sort=False, seed=0, transforms=[],
- merge_labels={}):
- # initialize super class StandardEoDataset
- super().__init__(root_dir, use_bands, tile_size, pad, gt_pattern,
- sort, seed, transforms, merge_labels)
-
- @staticmethod
- def get_size():
- """Image size of the ProSnow Garmisch dataset.
-
- Returns
- -------
- size : `tuple`
- The image size (height, width).
-
- """
- return (615, 543)
-
-
-class ProSnowObergurgl(ProSnowDataset):
- """Class for the ProSnow Obergurgl dataset."""
-
- def __init__(self, root_dir, use_bands=[], tile_size=None, pad=False,
- gt_pattern='(.*)gt\\.tif', sort=False, seed=0, transforms=[],
- merge_labels={}):
- # initialize super class StandardEoDataset
- super().__init__(root_dir, use_bands, tile_size, pad, gt_pattern,
- sort, seed, transforms, merge_labels)
-
- @staticmethod
- def get_size():
- """Image size of the ProSnow Obergurgl dataset.
-
- Returns
- -------
- size : `tuple`
- The image size (height, width).
-
- """
- return (310, 270)
-
-
class Cloud95Dataset(ImageDataset):
"""Class for the `Cloud-95`_ dataset by `Mohajerani & Saeedi (2020)`_.
.. _Cloud-95:
https://github.com/SorourMo/95-Cloud-An-Extension-to-38-Cloud-Dataset
+
.. _Mohajerani & Saeedi (2020):
https://arxiv.org/abs/2001.08768
@@ -1218,6 +1193,5 @@ class SupportedDatasets(enum.Enum):
"""Names and corresponding classes of the implemented datasets."""
Sparcs = SparcsDataset
+ Alcd = AlcdDataset
Cloud95 = Cloud95Dataset
- Garmisch = ProSnowGarmisch
- Obergurgl = ProSnowObergurgl
diff --git a/pysegcnn/core/trainer.py b/pysegcnn/core/trainer.py
index a9c5d6c..1d894c6 100644
--- a/pysegcnn/core/trainer.py
+++ b/pysegcnn/core/trainer.py
@@ -50,7 +50,6 @@ from pysegcnn.core.layers import Conv2dSame
from pysegcnn.core.logging import log_conf
from pysegcnn.core.graphics import (plot_loss, plot_confusion_matrix,
plot_sample, Animate)
-from pysegcnn.core.constants import map_labels
from pysegcnn.main.config import HERE, DRIVE_PATH
# module level logger
@@ -1370,23 +1369,23 @@ class NetworkInference(BaseConfig):
"""
return self.trg_ds.dataset.labels
- @property
- def label_map(self):
- """Label mapping from the source to the target domain.
+ # @property
+ # def label_map(self):
+ # """Label mapping from the source to the target domain.
- See :py:func:`pysegcnn.core.constants.map_labels`.
+ # See :py:func:`pysegcnn.core.constants.map_labels`.
- Returns
- -------
- label_map : `dict` [`int`, `int`]
- Dictionary with source labels as keys and corresponding target
- labels as values.
+ # Returns
+ # -------
+ # label_map : `dict` [`int`, `int`]
+ # Dictionary with source labels as keys and corresponding target
+ # labels as values.
- """
- # check whether the source domain labels are the same as the target
- # domain labels
- return map_labels(self.src_ds.get_labels(),
- self.trg_ds.dataset.get_labels())
+ # """
+ # # check whether the source domain labels are the same as the target
+ # # domain labels
+ # return map_labels(self.src_ds.get_labels(),
+ # self.trg_ds.dataset.get_labels())
@property
def source_is_target(self):
diff --git a/pysegcnn/main/config.py b/pysegcnn/main/config.py
index 9068f6b..bab13b0 100644
--- a/pysegcnn/main/config.py
+++ b/pysegcnn/main/config.py
@@ -19,27 +19,33 @@ License
# -*- coding: utf-8 -*-
# builtins
-import os
+import pathlib
# from pysegcnn.core.transforms import Augment, FlipLr, FlipUd, Noise
# path to this file
-HERE = os.path.abspath(os.path.dirname(__file__))
+HERE = pathlib.Path(__file__).resolve().parent
# path to the datasets on the current machine
-DRIVE_PATH = 'C:/Eurac/2020/CCISNOW/_Datasets/'
-# DRIVE_PATH = '/mnt/CEPH_PROJECTS/cci_snow/dfrisinghelli/_Datasets/'
+DRIVE_PATH = pathlib.Path('C:/Eurac/2020/CCISNOW/_Datasets/')
+# DRIVE_PATH = pathlib.Path('/mnt/CEPH_PROJECTS/cci_snow/dfrisinghelli/_Datasets/')
# name and paths to the datasets
-DATASETS = {'Sparcs': os.path.join(DRIVE_PATH, 'Sparcs'),
- 'Cloud95': os.path.join(DRIVE_PATH, 'Cloud95/Training'),
- 'Garmisch': os.path.join(DRIVE_PATH, 'ProSnow/Garmisch')}
+DATASETS = {'Sparcs': DRIVE_PATH.joinpath('Sparcs'),
+ 'Alcd': DRIVE_PATH.joinpath('Alcd/60m')
+ }
# name of the source domain dataset
SRC_DS = 'Sparcs'
# name of the target domain dataset
-TRG_DS = 'Garmisch'
+TRG_DS = 'Alcd'
+
+# spectral bands to use for training
+BANDS = ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']
+
+# tile size of a single sample
+TILE_SIZE = None
# the source dataset configuration dictionary
src_ds_config = {
@@ -63,11 +69,11 @@ src_ds_config = {
# or [], which corresponds to using all available bands
# IMPORTANT: the list of bands should be equal for the source and target
# domains, when using any sort of transfer learning
- 'bands': ['red', 'green', 'blue', 'nir', 'swir1', 'swir2'],
+ 'bands': BANDS,
# define the size of the network input
# if None, the size will default to the size of a scene
- 'tile_size': 128,
+ 'tile_size': TILE_SIZE,
# whether to central pad the scenes with a constant value
# if True, padding is used if the scenes are not evenly divisible into
@@ -146,14 +152,15 @@ src_ds_config = {
trg_ds_config = {
'dataset_name': TRG_DS,
'root_dir': DATASETS[TRG_DS],
- 'gt_pattern': '(.*)class\\.img',
- 'bands': ['red', 'green', 'blue', 'nir', 'swir1', 'swir2'],
- 'tile_size': 128,
+ 'gt_pattern': '(.*)Labels\\.tif',
+ 'bands': BANDS,
+ 'tile_size': TILE_SIZE,
'pad': True,
'seed': 0,
'sort': True,
'transforms': [],
- 'merge_labels': {}
+ 'merge_labels': {'Cirrus': 'Cloud',
+ 'Not_used': 'No_data'}
}
# the source dataset split configuration dictionary
--
GitLab