Commit 0c9074f0 authored by Pietro Zambelli's avatar Pietro Zambelli
Browse files

Add code and repository configuration files

parent 8b79fe9b
[flake8]
max-line-length = 88
select = B,C,E,F,W,T4,B9,B950
ignore = E501, W503, E203
*.tif filter=lfs diff=lfs merge=lfs -text
# Created by https://www.toptal.com/developers/gitignore/api/vim,pycharm+iml,git,python
# Edit at https://www.toptal.com/developers/gitignore?templates=vim,pycharm+iml,git,python
### Git ###
# Created by git for backups. To disable backups in Git:
# $ git config --global mergetool.keepBackup false
*.orig
# Created by git when using merge tools for conflicts
*.BACKUP.*
*.BASE.*
*.LOCAL.*
*.REMOTE.*
*_BACKUP_*.txt
*_BASE_*.txt
*_LOCAL_*.txt
*_REMOTE_*.txt
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
pythonenv*
# Authorized and unauthorized licenses in LOWER CASE
[Licenses]
authorized_licenses:
BSD
new BSD
BSD license
new BDS license
simplified BSD
Apache
Apache 2.0
Apache software license
gnu LGPL
LGPL with exceptions or zpl
ISC license
ISC license (ISCL)
MIT
MIT license
python software foundation license
zpl 2.1
unauthorized_licenses:
GPL v3
GPL
GNU General Public License v2 or later (GPLv2+)
[Authorized Packages]
# Python software license (see http://zesty.ca/python/uuid.README.txt)
uuid: 1.25,>=1.30
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/timothycrosley/isort
rev: 5.6.4
hooks:
- id: isort
files: .*.py
args: ['--profile', 'black']
- repo: https://github.com/psf/black
rev: 21.9b0
hooks:
- id: black
args: [--experimental-string-processing]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.1.0
hooks:
- id: check-ast
- id: check-case-conflict
- id: trailing-whitespace
- id: end-of-file-fixer
- id: debug-statements
- id: check-docstring-first
- id: check-yaml
- repo: https://github.com/PyCQA/flake8
rev: 3.9.2
hooks:
- id: flake8
- repo: https://github.com/PyCQA/bandit
rev: 1.7.0
hooks:
- id: bandit
import uuid
from datetime import datetime
from pathlib import Path
from typing import Any, Dict
import frictionless
import rasterio
def curdir() -> Path:
return Path(__file__).resolve().parent
def projdir() -> Path:
return curdir().parent
def datadir() -> Path:
return projdir() / "data"
def datapackage() -> Path:
return projdir() / "datapackage.json"
def describe_resource(resrc: Dict[str, Any]):
path = Path(resrc.path)
# remove encoding
if "encoding" in resrc:
resrc.pop("encoding")
if "stats" in resrc:
resrc.pop("stats")
*_, sim_type, dd_type, Tb, aggr_window, method, gname = path.parts
resrc["simulation_type"] = sim_type
resrc["dd_type"] = dd_type
resrc["base_temperature"] = Tb
resrc["temporal_aggregation"] = aggr_window
resrc["aggregation_method"] = method
resrc["base_temperature"] = Tb
resrc["unit"] = "°C"
if aggr_window == "monthly":
year, month = gname[:-4].split("_")
resrc["year"] = int(year)
resrc["month"] = int(month)
elif aggr_window == "yearly":
resrc["year"] = int(gname[:4])
else:
raise ValueError(f"Temporal aggregation not supported: {aggr_window}")
stat = path.stat()
resrc["bytes"] = stat.st_size
raster = {}
with rasterio.open(path.as_posix()) as rast:
raster["crs"] = str(rast.crs)
raster["num_of_bands"] = rast.count
raster["dimension"] = dict(columns=rast.width, rows=rast.height)
raster["transformation"] = dict(operator="*", value=10)
raster["nodata"] = rast.nodata
raster["bounds"] = {
k: getattr(rast.bounds, k) for k in ("left", "right", "top", "bottom")
}
raster["resolution"] = rast.res
bands = {}
for i in range(1, rast.count + 1):
bnd = {}
bnd["nodata"] = rast.nodatavals[i - 1]
bnd["dtype"] = rast.dtypes[i - 1]
arr = rast.read(i, masked=True)
bnd["statistics"] = dict(
min=float(arr.min()),
max=float(arr.max()),
mean=float(arr.mean()),
# median=arr.median(),
std=float(arr.std()),
count=int(arr.count()),
)
bands[i] = bnd
raster["bands"] = bands
resrc["raster"] = raster
return resrc
def main():
print("Generating the first package structure...")
package = frictionless.describe(
source=(datadir() / "**" / "*.tif").as_posix(), type="package"
)
package.title = (
"Heating and Cooling Degree Days extracted from CORDEX climate simulations"
)
package.description = (
"HDDs and CDDs are computed using different base temperature and under"
" different simulations assumptions"
)
print("Enrich the information of the main resources...")
start_yr, start_mt, end_yr, end_mt = 5000, 50, 0, 0
lenres = len(package.resources)
resources = []
for j, res in enumerate(package.resources):
print(f"{j*100./lenres:5.2f}% – {res.path}", end="\r", flush=True)
# {'path': 'evaluation/cdd/24.0/monthly/average/1991_01.tif',
# 'name': '1991_01', 'profile': 'data-resource',
# 'scheme': 'file', 'format': 'tif', 'hashing': 'md5',
# 'encoding': 'utf-8'}
res = describe_resource(res)
yr, mt = res["year"], res["month"]
if yr > end_yr:
end_yr = yr
elif yr == end_yr:
end_mt = mt if mt > end_mt else end_mt
elif yr < start_yr:
start_yr = yr
elif yr == start_yr:
start_mt = mt if mt < start_mt else start_mt
else:
continue
resources.append(res)
package.resources = resources
print(f"\n{start_yr}/{start_mt:0d}{end_yr}/{end_mt:0d}", flush=True)
package["temporal"] = dict(
start=f"{start_yr}-{start_mt}-01",
end=f"{end_yr}-{end_mt}-31",
)
package.licenses.append(
dict(
id="CC-BY-4.0",
title="Creative Commons Attribution 4.0 International",
url="https://creativecommons.org/licenses/by/4.0/legalcode",
)
)
package.sources.append(dict(name="CORDEX", path="https://cordex.org/"))
package.contributors.append(
dict(
title="Pietro Zambelli",
email="pietro.zambelli@eurac.edu",
role="author",
organization="eurac",
)
)
package.keywords.extend(
[
"Heating Degree Days",
"HDD",
"Cooling Degree Days",
"CDD",
"Europe",
"EU28",
"CORDEX",
"Raster",
]
)
package.created = f"{datetime.now():%Y-%m-%d %H:%M:%S}"
package.id = str(uuid.uuid4())
package.profile = "raster-data-resource"
print("Exporting the datapackage")
package.to_yaml(projdir() / "hdd_cdd.package.yaml")
package.to_json(projdir() / "hdd_cdd.package.json")
print("✨ Done! ✨")
if __name__ == "__main__":
main()
bandit
black
flake8
isort
pre-commit
rasterio
frictionless
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment