Commit 703e04bc authored by Greifeneder Felix's avatar Greifeneder Felix

initial commit

parents
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyUnboundLocalVariableInspection" enabled="false" level="WARNING" enabled_by_default="false" />
</profile>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7 (1)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/pysmm.iml" filepath="$PROJECT_DIR$/.idea/pysmm.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="0b304d2a-93b9-4b4d-ab03-85bd0c9bb482" name="Default" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file leaf-file-name="test_GEE_wrappers.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/pysmm/tests/test_GEE_wrappers.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="85">
<caret line="5" column="10" selection-start-line="5" selection-start-column="10" selection-end-line="5" selection-end-column="10" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="setup.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/setup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="26" column="29" lean-forward="true" selection-start-line="26" selection-start-column="29" selection-end-line="26" selection-end-column="29" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="README.md" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="544">
<caret line="32" selection-start-line="32" selection-end-line="32" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/pysmm/tests/test_GEE_wrappers.py" />
<option value="$PROJECT_DIR$/README.md" />
<option value="$PROJECT_DIR$/setup.py" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="x" value="-8" />
<option name="y" value="-8" />
<option name="width" value="1936" />
<option name="height" value="1056" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="pysmm" type="b2602c69:ProjectViewProjectNode" />
<item name="pysmm" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="pysmm" type="b2602c69:ProjectViewProjectNode" />
<item name="pysmm" type="462c0819:PsiDirectoryNode" />
<item name="pysmm" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="pysmm" type="b2602c69:ProjectViewProjectNode" />
<item name="pysmm" type="462c0819:PsiDirectoryNode" />
<item name="pysmm" type="462c0819:PsiDirectoryNode" />
<item name="tests" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="settings.editor.selected.configurable" value="reference.idesettings.debugger.python" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="0b304d2a-93b9-4b4d-ab03-85bd0c9bb482" name="Default" comment="" />
<created>1523270434090</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1523270434090</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="6" />
<layout>
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Database Changes" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" show_stripe_button="false" weight="0.32900432" />
<window_info active="true" anchor="bottom" id="Python Console" visible="true" weight="0.32900432" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Terminal" weight="0.32900432" />
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.25" />
<window_info anchor="bottom" id="Docker" show_stripe_button="false" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="SciView" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" side_tool="true" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="right" id="Commander" order="0" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="VcsManagerConfiguration">
<ignored-roots>
<path value="$PROJECT_DIR$" />
<path value="$PROJECT_DIR$/../test" />
</ignored-roots>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/pysmm/utils.py">
<provider selected="true" editor-type-id="text-editor">
<state>
<folding>
<element signature="e#0#37#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pysmm/__init__.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/pysmm/tests/test_GEE_wrappers.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="85">
<caret line="5" column="10" selection-start-line="5" selection-start-column="10" selection-end-line="5" selection-end-column="10" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="544">
<caret line="32" selection-start-line="32" selection-end-line="32" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/setup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="26" column="29" lean-forward="true" selection-start-line="26" selection-start-column="29" selection-end-line="26" selection-end-column="29" />
</state>
</provider>
</entry>
</component>
</project>
\ No newline at end of file
# pysmm
PYthon Sentinel-1 soil-Moisture Mapping (PYSMM)
<p>This package acts as an interface to Google Earth Engine for the estimation of surface soil moisture based
on Copernicus Sentinel-1 intensity data. It is meant as a supplement to the following publication: <i> Greifeneder, F.,
C. Notarnicola, W. Wagner. Machine learning based mapping of high resolution, global surface soil moisture content. </i>
The estimation of soil moisture is based on a Support-Vector-Regression machine learning approach. The model training
was performed based on in-situ data from the International Soil Moisture Network (ISMN). PYSMM all processing steps
for spatial and temporal mapping of surface soil moisture are fully executed online on GEE - none of the input data-sets
needs to be downloaded.
Acknowledgements: This study was made possibly through funding within the Horizon 2020 project "Ecopotential: Improving
Future Ecosystem Benefits through Earth Observations"
Please follow the installation instructions below:
</p>
<h2>Installation</h2>
<p>Most of the data processing is executed on-line on Google Earth Engine.
Therefore, the execution of this script requires a Google account and access to Google Earth Engine -
we are working on an updated version that will utilize a GEE Application Key rather than a personal account.</p>
<h3>Installation of the Google Earth Engine API</h3>
<p>To allow the script to talk to Google Earh Engine the API has to be installed. Please follow the instructions at
this link: <a href="https://developers.google.com/earth-engine/python_install_manual">GEE API</a></p>
<h3>Installtion of the Google Drive API</h3>
<p>After the computation inside Google Earth Engine is finished, the results are exported to your Google Drive.
To let the script access and download the results to you local computer,
the Google Drive API has to be installed as well. Please follow the instructions here:
<a href="https://developers.google.com/drive/v3/web/quickstart/python">Google Drive API</a></p>
<p>As described in the manual, for the first run, the authentication can be initiated by running the
quickstart.py script. To enable the download of data please modify the following line of the script:
SCOPES = 'https://www.googleapis.com/auth/drive.metadata.readonly' -->
SCOPES = 'https://www.googleapis.com/auth/drive'</p>
<h3>Python Installation</h3>
<p>Clone the git repository and run <i>python setup.py install</i></p>
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
from GEE_wrappers import GEE_extent
from GEE_wrappers import GEE_pt
import numpy as np
import os
import matplotlib.pyplot as plt
def get_map(minlon, minlat, maxlon, maxlat, outpath,
sampling=100,
year=None, month=None, day=None,
tracknr=None,
overwrite=False,
ex_t_mask=None,
tempfilter=True,
mask='Globcover',
masksnow=True):
maskcorine=False
maskglobcover=False
if mask == 'Globcover':
maskglobcover = True
elif mask == 'Corine':
maskcorine = True
else:
print(mask + ' is not recognised as a valid land-cover classification')
return
if year is not None:
GEE_interface = GEE_extent(minlon, minlat, maxlon, maxlat, outpath, sampling=sampling)
# retrieve S1
GEE_interface.get_S1(year, month, day,
tempfilter=tempfilter,
applylcmask=maskcorine,
mask_globcover=maskglobcover,
trackflt=tracknr,
masksnow=masksnow)
# retrieve GLDAS
GEE_interface.get_gldas()
# get Globcover
GEE_interface.get_globcover()
# get the SRTM
GEE_interface.get_terrain()
outname = 'SMCmap_' + \
str(GEE_interface.S1_DATE.year) + '_' + \
str(GEE_interface.S1_DATE.month) + '_' + \
str(GEE_interface.S1_DATE.day)
# Estimate soil moisture
GEE_interface.estimate_SM()
GEE_interface.GEE_2_disk(name=outname, timeout=False)
GEE_interface = None
else:
# if no specific date was specified extract entire time series
GEE_interface = GEE_extent(minlon, minlat, maxlon, maxlat, outpath, sampling=sampling)
# get list of S1 dates
dates = GEE_interface.get_S1_dates(tracknr=tracknr)
dates = np.unique(dates)
for dateI in dates:
# retrieve S1
GEE_interface.get_S1(dateI.year, dateI.month, dateI.day,
tempfilter=tempfilter,
applylcmask=maskcorine,
mask_globcover=maskglobcover,
trackflt=tracknr,
masksnow=masksnow)
# retrieve GLDAS
GEE_interface.get_gldas()
# get Globcover
GEE_interface.get_globcover()
# get the SRTM
GEE_interface.get_terrain()
outname = 'SMCmap_' + \
str(GEE_interface.S1_DATE.year) + '_' + \
str(GEE_interface.S1_DATE.month) + '_' + \
str(GEE_interface.S1_DATE.day)
if overwrite == False and os.path.exists(outpath + outname + '.tif'):
print(outname + ' already done')
continue
GEE_interface.GEE_2_disk(name=outname, timeout=False)
GEE_interface = None
def get_ts(lon, lat,
workpath,
footprint=50,
masksnow=True,
calc_anomalies=False,
create_plots=False):
"""Get S1 soil moisture time-series"""
# initialize GEE point object
gee_pt_obj = GEE_pt(lon, lat, workpath, buffer=footprint)
sm_ts = gee_pt_obj.extr_SM(masksnow=masksnow, calc_anomalies=calc_anomalies)
# create plots
if create_plots == True:
if calc_anomalies == False:
# plot s1 soil moisture vs gldas_downscaled
fig, ax = plt.subplots(figsize=(6.5,2.7))
line1, = ax.plot(sm_ts.index, sm_ts,
color='b',
linestyle='-',
marker='+',
label='S1 Soil Moisture',
linewidth=0.2)
ax.set_ylabel('Soil Moisture [%-Vol.]')
plotname = 's1_sm_' + str(lon) + '_' + str(lat) + '.png'
else:
fig, ax = plt.subplots(figsize=(6.5,2.7))
line1, = ax.plot(sm_ts.index, sm_ts['ANOM'].values,
color='r',
linestyle='-',
marker='+',
label='S1 Soil Moisture Anomaly',
linewidth=0.2)
x0 = [sm_ts.index[0], sm_ts.index[-1]]
y0 = [0, 0]
line2, = ax.plot(x0, y0,
color='k',
linestyle='--',
linewidth=0.2)
ax.set_ylabel('Soil Moisture Anomaly [%-Vol.]')
#plt.legend(handles=[line1, line2])
plotname = 's1_sm_anom' + str(lon) + '_' + str(lat) + '.png'
plt.setp(ax.get_xticklabels(), fontsize=6)
plt.savefig(workpath + plotname, dpi=300)
return sm_ts
from unittest import TestCase
import tempfile
import os
import pandas as pd
from pysmm.GEE_wrappers import GEE_pt
class TestGEEWrappers(TestCase):
def test_LC(self):
testlon = 12.31
testlat = 48.89
tempdir = tempfile.mkdtemp()
test_inst = GEE_pt(testlon, testlat, tempdir)
test_inst.extr_LC()
self.assertTrue(isinstance(test_inst.LC, float))
os.rmdir(tempdir)
def test_MOD13Q1(self):
testlon = 12.31
testlat = 48.89
tempdir = tempfile.mkdtemp()
test_inst = GEE_pt(testlon, testlat, tempdir)
test_ser = test_inst.extr_MODIS_MOD13Q1()
self.assertTrue(isinstance(test_ser, pd.Series))
# TODO complete tests for all methods in GEE_wrappers
\ No newline at end of file
from __future__ import print_function
import httplib2
import os
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
from googleapiclient.http import MediaIoBaseDownload
import io
import numpy as np
class gdrive(object):
def __init__(self):
# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/drive-python-quickstart.json
self.SCOPES = 'https://www.googleapis.com/auth/drive'
self.CLIENT_SECRET_FILE = 'client_secret.json'
self.APPLICATION_NAME = 'Drive API Python Quickstart'
def _get_credentials(self):
"""Gets valid user credentials from storage.
If nothing has been stored, or if the stored credentials are invalid,
the OAuth2 flow is completed to obtain the new credentials.
Returns:
Credentials, the obtained credential.
"""
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'drive-python-quickstart.json')
store = Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(self.CLIENT_SECRET_FILE, self.SCOPES)
flow.user_agent = self.APPLICATION_NAME
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def _init_connection(self):
credentials = self._get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('drive', 'v3', http=http)
return(http, service)
def print_file_list(self):
http, service = self._init_connection()
results = service.files().list(
pageSize=30, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
if not items:
print('No files found.')
else:
print('Files:')
for item in items:
print('{0} ({1})'.format(item['name'], item['id']))
def get_id(self, filename):
http, service = self._init_connection()
# get list of files
results = service.files().list(
pageSize=50, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
# extract list of names and id and find the wanted file
namelist = np.array([items[i]['name'] for i in range(len(items))])
idlist = np.array([items[i]['id'] for i in range(len(items))])
file_pos = np.where(namelist == filename)
if len(file_pos[0]) == 0:
return(0, filename + ' not found')
else:
return(1, idlist[file_pos])
def download_file(self, filename, localpath):
http, service = self._init_connection()
# get file id
success, fId = self.get_id(filename)
if success == 0:
print(filename + ' not found')
return
request = service.files().get_media(fileId=fId[0])
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print('Download %d%%.' % int(status.progress() * 100))
fo = open(localpath, 'wb')
fo.write(fh.getvalue())
fo.close()
def delete_file(self, filename):
http, service = self._init_connection()
# get file id
success, fId = self.get_id(filename)
if success == 0:
print(filename + ' not found')
service.files().delete(fileId=fId[0]).execute()
\ No newline at end of file
from setuptools import setup
setup(name='pysmm',
version='0.1',
description='Python Sentinel Soil-Moisture Mapping',
classifiers=[
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Programming Language :: Python :: 2.7',
],
url='https://gitlab.inf.unibz.it/Felix.Greifeneder/pysmm.git',
author='Felix Greifeneder',
author_email='felix.greifeneder@eurac.edu',
license='GPLv3',
packages=['gloma'],
install_requires=['earthengine-api',
'google-api-python-client',
'cryptography',
'matplotlib',
'numpy',
'pandas',
'pytesmo',
'scikit-learn',
'scipy'],
include_package_data=True,
zip_safe=False,
test_suite='nose.collector',
tests_require=['nose'])
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment