Merge pull request #4 from felixlobert/fl-translate-to-python

translate core funtionality from R to python

.dockerignore

@@ -0,0 +1,7 @@
+data/
+test/
+example/
+.gitignore
+Dockerfile
+LICENSE
+README.md

.gitignore

@@ -1,40 +1,2 @@
-# History files
-.Rhistory
-.Rapp.history
-
-# Session Data files
-.RData
-
-# User-specific files
-.Ruserdata
-
-# Example code in package build process
-*-Ex.R
-
-# Output files from R CMD build
-/*.tar.gz
-
-# Output files from R CMD check
-/*.Rcheck/
-
-# RStudio files
-.Rproj.user/
-
-# produced vignettes
-vignettes/*.html
-vignettes/*.pdf
-
-# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
-.httr-oauth
-
-# knitr and R markdown default cache directories
-*_cache/
-/cache/
-
-# Temporary files created by R markdown
-*.utf8.md
-*.knit.md
-
-# R Environment Variables
-.Renviron
-
+data/
+test/

Dockerfile

@@ -1,31 +1,9 @@
-FROM mundialis/esa-snap:8.0-ubuntu
+FROM mundialis/esa-snap:9.0-ubuntu
 
-# fix locale error
-RUN apt-get update && apt-get -y install locales
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
-    locale-gen
-ENV LANG en_US.UTF-8  
-ENV LANGUAGE en_US:en  
-ENV LC_ALL en_US.UTF-8     
+ADD requirements.txt /force-sar/
+RUN pip install -r /force-sar/requirements.txt
 
-# important R dependencies
-RUN apt-get -y install libssl-dev libcurl4-openssl-dev libxml2-dev
+ADD . /force-sar/
+RUN pip install -e /force-sar/.
 
-# add repo and install  R >= 4.0
-RUN apt-get -y install software-properties-common
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
-RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/'
-
-RUN apt-get update && apt-get -y install r-base
-
-RUN apt-get -y install libgdal-dev libproj-dev libgeos-dev libudunits2-dev libcairo2-dev libnetcdf-dev
-
-# install R-packges
-RUN R -e "install.packages('remotes')"
-RUN R -e "remotes::install_github('felixlobert/rcodede@b4ceac7060ab3e1c2bbb5eddf77e8c8727b1c17a')"
-RUN R -e "install.packages('parallel')"
-RUN R -e "install.packages('lubridate')"
-RUN R -e "install.packages('stringr')"
-
-COPY . /force-sar/
-WORKDIR /force-sar/
+WORKDIR /force-sar/

README.md

@@ -1,11 +1,16 @@
 # force-sar
-This project aims to create a Sentinel-1 SAR data processing module than can be integrated into the [Framework for Operational Radiometric Correction for Environmental monitoring (FORCE)](https://force-eo.readthedocs.io/en/latest/) and is still in the conception phase. It is especially intended to be used on platforms that already have a satellite data repository like a DIAS or the [German CODE-DE cloud environment](https://code-de.org/de/), but will also be abled to download the satellite data.
+This project aims to create a Sentinel-1 SAR data processing module that can be used to integrate SAR data into the [Framework for Operational Radiometric Correction for Environmental monitoring (FORCE)](https://force-eo.readthedocs.io/en/latest/). The project is still in the conception phase and especially intended to be used on platforms that have access to a satellite data repository like a DIAS or the [German CODE-DE cloud environment](https://code-de.org/de/). A submodule to download Sentinel-1 data is planned.
 
-It is based on a [module to query](https://github.com/felixlobert/rcodede) scenes of interest and a [docker container running esa SNAP](https://hub.docker.com/r/mundialis/esa-snap) that takes over the processing of the data with a pre-built but easily customizable S1-GRD to $\gamma^0$ workflow.
+*force-sar* is based on a module to query scenes of interest based on a number of different search criteria and a [docker container running esa SNAP](https://hub.docker.com/r/mundialis/esa-snap) that takes over the processing of the data with a pre-built but easily customizable S1-GRD to $\gamma^0$ workflow. Search criteria and data are both defined in a FORCE-like parameter file (see `example/prm_file.prm`).
+
+*force-sar* uses a custom docker image that is built using the Dockerfile. The module is then started using a bash script (`bash/force-sar.sh`) together with the user-defined parameter-file (e.g., `example/prm_file.prm`):
 
-The docker image is built from the Dockerfile, the module is the started with a bash file (bash/force-sar.sh) and a user-defined parameter-file (data/test_params/prm_file.prm):
 ```bash
+git clone https://github.com/felixlobert/force-sar.git
+
+cd force-sar
+
 docker build -t force-sar .
 
-sh bash/force-sar.sh data/test_params/prm_file.prm
+bash bash/force-sar.sh example/prm_file.prm
 ```

bash/force-sar.sh

@@ -1,28 +1,39 @@
 #!/bin/bash
 
+if [[ -z $1 ]]
+then
+    echo error: path to parameterfile needs to be set.
+    exit 1
+fi
+
 # set command input as path for parameterfile
-PRM_FILE=$1
+if [[ "$1" = /* ]]
+then
+    PRM_FILE=$1
+else
+    PRM_FILE=$PWD/$1
+fi
 
 # grep directories from prm-file, remove whitespace and define in environment
 DIRECTORIES=$(cat $PRM_FILE | grep '^DIR\|^FORCE_GRID\|^FILE\|^RESOLUTION\|^NTHREAD' | grep -v 'NULL' | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g')
 eval $DIRECTORIES
 
-# start docker container with esa SNAP and R and connect paths given in prm file
+# start docker container with esa SNAP
 docker run \
     -t -d \
     --name force-sar-container \
-    -v $PRM_FILE:/force-sar/data/test_params/prm_file.prm \
-    -v $FORCE_GRID:$FORCE_GRID \
+    -v $PRM_FILE:$PRM_FILE \
     -v $DIR_ARCHIVE:$DIR_ARCHIVE \
     -v $DIR_LOWER:$DIR_LOWER \
+    -v $FORCE_GRID:$FORCE_GRID \
     -v $DIR_REPO:$DIR_REPO \
+    -v $FILE_TILE:$FILE_TILE \
     -v $DIR_DEM:/root/.snap/auxdata/dem/SRTM\ 1Sec\ HGT \
     -v $DIR_ORBIT:/root/.snap/auxdata/Orbits/ \
-    -v $FILE_TILE:$FILE_TILE \
     force-sar
 
 # query and process data inside docker container
-docker exec force-sar-container Rscript R/force-sar.R
+docker exec force-sar-container python3 src/main.py $PRM_FILE
 
 gid=$(id -g $USER)
 uid=$(id -u $USER)
@@ -35,15 +46,27 @@ docker exec force-sar-container chown -R $USER:$USER $DIR_ARCHIVE
 docker stop force-sar-container
 docker rm force-sar-container
 
-# # define unique date and orbit pairs from processed scenes
+# start force docker container for mosaicing and cubing files
+docker run \
+    -t -d \
+     --name force-container \
+    -v $DIR_LOWER:$DIR_LOWER \
+    -v $DIR_ARCHIVE:$DIR_ARCHIVE \
+    -u $(id -u):$(id -g) \
+    davidfrantz/force:3.7.10 \
+    /bin/bash
+
+# define unique date and orbit pairs from processed scenes
 ACQUISITIONS=$(ls $DIR_ARCHIVE | grep '.tif' | cut -f1-3 -d'_' | uniq)
 
 # loop over date orbit combinations to create a mosaic and tile into existing force cube
 for ACQUISITION in $ACQUISITIONS
 do
-    gdalbuildvrt -vrtnodata -9999 -srcnodata -9999 $DIR_ARCHIVE/${ACQUISITION}_GAM.vrt $DIR_ARCHIVE/$ACQUISITION*.tif
+    docker exec force-container gdalbuildvrt -vrtnodata -9999 -srcnodata -9999 $DIR_ARCHIVE/${ACQUISITION}_GAM.vrt $DIR_ARCHIVE/$ACQUISITION*.tif
 
-    force-cube -o $DIR_LOWER -n -9999 -t Int16 -r near -j $NTHREAD -s $RESOLUTION $DIR_ARCHIVE/${ACQUISITION}_GAM.vrt 
+    docker exec force-container force-cube -o $DIR_LOWER -n -9999 -t Int16 -r near -j $NTHREAD -s $RESOLUTION $DIR_ARCHIVE/${ACQUISITION}_GAM.vrt 
 done
 
-
+# stop container
+docker stop force-container
+docker rm force-container