Add piscal manager scripts

Dockerfile

@@ -0,0 +1,57 @@
+#Download base image
+FROM ubuntu:latest
+
+# Install and update software
+RUN set -xe \
+	&& apt-get update \
+	&& apt-get upgrade -y \
+	# SSHD install
+	&& apt-get install --no-install-recommends -y openssh-server sudo \
+	# Piscal reqs
+	&& apt-get install make -y \
+	&& apt-get install xutils-dev -y \
+	&& apt-get install gfortran -y \
+	&& apt-get install libopenmpi-dev -y \
+	# utils
+	&& apt-get install iproute2 -y \
+	&& apt-get install vim -y \
+	# cleanup
+	&& apt-get autoclean -y \
+	&& apt-get autoremove -y
+
+# configure sshd, copied from wataken44/ubuntu-latest-sshd
+RUN set -xe \
+    && groupadd launcher \
+    && useradd -g launcher -G sudo -m -s /bin/bash launcher \
+    && echo 'launcher:launcher' | chpasswd
+
+RUN set -xe \
+    && sed -i -e 's/#PasswordAuthentication.*/PasswordAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i -e 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \
+    && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+RUN set -xe \
+    && chown -R launcher:launcher /home/launcher 
+
+# fix for SSHD - "Missing privilege separation directory: /run/sshd"
+RUN set -xe \
+    && mkdir /run/sshd
+
+##### add user l2g
+# RUN set -xe \
+#     && groupadd l2g \
+#     && useradd -g l2g -G sudo -m -s /bin/bash l2g \
+#     && echo 'l2g:pwdpwd' | chpasswd
+# RUN set -xe \
+#     && chown -R l2g:l2g /home/l2g 
+# RUN set -xe \
+# 	&& apt-get install nano
+#####
+
+ADD piscal-manager /srv
+ADD leafres/testrun/piscal /srv
+#RUN chmod R +x /srv/*.sh
+WORKDIR /srv
+
+EXPOSE 22
+CMD ["/usr/sbin/sshd", "-D"]

build-docker.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+sudo docker build -t piscal-server:latest .
+#sudo docker run -it --rm -p 2222:22 --name my-piscal-server piscal-server

piscal-manager/README.txt

@@ -0,0 +1,31 @@
+./ # the working directory for Piscal processing
+./piscal.cfg # Piscal configuration - first line has Photosynthetic Pathway type
+             # followed by a list of all files in ./input
+             # C3_photosynthesis_leafweb, C4_photosynthesis_leafweb
+             # CAM_photosynthesis_leafweb
+./piscal.err # Contains any stderr output from launching Piscal, should be empty
+             # unless there is a problem with the system or executable
+./piscal.pid # file containing one line - the process ID which Piscal
+             # is running under. this is used to check if Piscal is currently
+             # executing or if it has completed
+
+./input/     # directory containing all unprocessed LeafInput files
+./input/Wild_Capsicum1.csv # example input copied from LeafWeb
+./input/Wild_Capsicum2.csv # ""
+
+./output/    # directory piscal creates to store all output
+
+./output/clninput/ # directory containing cleaned leaf input, for the purpose
+                   # of making that data available for search in LeafWeb
+./output/clninput/20160311152630Wild_Capsicum1.csv # example cleaned input
+./output/clninput/20160311152630Wild_Capsicum2.csv # ""
+
+./output/fitresult/ # directory containing fitresult generated by Piscal
+./output/fitresult/touser/    # output results sent to user and to storage
+./output/fitresult/touser/cntrlbestparameters.csv # example output
+./output/fitresult/touser/errormessage   # file containing any error 
+                                         # messages processed by Piscal
+./output/fitresult/touser/warningmessage # file containing any warning
+                                         # messages processed by Piscal
+
+./output/fitresult/nottouser/ # Output results sent to storage only

piscal-manager/piscal_launcher.cfg

@@ -0,0 +1,2 @@
+piscal_executable="/home/piscaladmin/piscal_executable/piscal"
+storage_directory="/home/piscaladmin/LeafWeb_storage"

piscal-manager/piscal_launcher.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+# piscal launcher
+
+usage="$(basename "$0") [-h] -d working_directory [-u notify_url] -- script to launch Piscal
+
+where:
+    -h  show this help text
+    -d  working directory
+    -f  input filename
+    -t  suppress storage copy
+    -u  url to notify on completion"
+
+# http://stackoverflow.com/a/246128/99492
+base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+working_directory=""
+output_directory_base_name="output"
+input_directory_name="input"
+cleaned_input_directory_name="clninput"
+touser_directory_name="fitresult/touser"
+nottouser_directory_name="fitresult/nottouser"
+
+# import the settings from piscal.cfg
+# $piscal_executable and $storage_directory
+. "$base_directory/piscal_launcher.cfg"
+
+while getopts "hd:f:u:t" opt; do
+    case "$opt" in
+    h )
+        echo "$usage"
+        exit
+        ;;
+    d )
+        working_directory=$OPTARG
+        ;;
+    t )
+        suppress_storage_copy=true
+        ;;
+    u )
+        notify_url=$OPTARG
+        ;;
+    \?) printf "illegal option: -%s\n" "$OPTARG" >&2
+        echo "$usage" >&2
+        exit 1
+        ;;
+    esac
+done
+
+if [ -z "$working_directory" ]; then
+    echo "working directory required (-d)"
+    exit 1
+fi
+if [ ! -d "$working_directory" ]; then
+    echo "working directory $working_directory not found"
+    exit 1
+fi
+output_directory_base="$working_directory/$output_directory_base_name"
+output_directory_touser="$output_directory_base/$touser_directory_name"
+output_directory_nottouser="$output_directory_base/$nottouser_directory_name"
+cleaned_input_directory="$output_directory_base/$cleaned_input_directory_name"
+run_directory="$working_directory/run"
+
+# setup output directories
+if [ ! -d "$output_directory_base" ]; then
+    mkdir "$output_directory_base"
+fi
+if [ ! -d "$cleaned_input_directory" ]; then
+    mkdir "$cleaned_input_directory"
+fi
+
+#find "$working_directory/$input_directory_name" -maxdepth 1 -type f\
+#    -exec cp {} "$cleaned_input_directory" \;
+
+if [ ! -d "$output_directory_base/fitresult" ]; then
+    mkdir "$output_directory_base/fitresult"
+fi
+if [ ! -d "$output_directory_touser" ]; then
+    mkdir "$output_directory_touser"
+fi
+if [ ! -d "$output_directory_nottouser" ]; then
+    mkdir "$output_directory_nottouser"
+fi
+if [ ! -d "$run_directory" ]; then
+    mkdir "$run_directory"
+fi
+
+# run piscal
+pushd $run_directory >> /dev/null
+
+if [ -z "$test_output_directory" ]; then
+    eval $piscal_executable
+else
+    cp -r "$test_output_directory"/* "$output_directory_base"/
+fi
+
+popd >> /dev/null
+
+# copy output to storage
+if [ -z "$suppress_storage_copy" ]; then
+    cp "$output_directory_touser"/* "$storage_directory"/output/ 2>/dev/null
+    cp "$output_directory_nottouser"/* "$storage_directory"/output/ 2>/dev/null
+    cp "$cleaned_input_directory"/* "$storage_directory"/input/ 2>/dev/null
+
+    mv_script=$base_directory/subdir_year.sh
+    pushd "$storage_directory"/output >> /dev/null
+    "$mv_script"
+    popd >> /dev/null
+    pushd "$storage_directory"/input >> /dev/null
+    "$mv_script"
+    popd >> /dev/null
+fi
+
+# notify given url of completion
+if [ -n "$notify_url" ]; then
+    wget -qO- -t 1 "$notify_url" # &> /dev/null 2>&1
+fi

piscal-manager/piscal_manager.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+# piscal manager script
+
+usage="$(basename "$0") [-h] 
+    -d directory_name
+    [-s|-c|-k]
+    -p photosynthetic_type
+    -u notify_url
+    -t
+
+where:
+    -h  show this help text
+    -d  working directory name
+    -s  start job
+    -c  cleanup directory
+    -k  kill job
+    -u  url to notify on completion
+    -p  photosynthetic type
+    -t  suppress storage copy"
+
+# Initialize variables:
+# http://stackoverflow.com/a/246128/99492
+base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+directory_name=""
+photosynthetic_type="C3_photosynthesis_leafweb" #default
+task="get_status" # default task
+input_directory_name="input"
+pid_filename="piscal.pid"
+stderr_filename="piscal.err"
+cleaned_input_directory_name="output/clninput"
+output_directory_name="output/fitresult/touser"
+nottouser_directory_name="output/fitresult/nottouser"
+launcher="$base_directory/piscal_launcher.sh"
+
+# http://stackoverflow.com/a/14203146/99492
+# http://wiki.bash-hackers.org/howto/getopts_tutorial
+
+while getopts "hd:f:p:stcku:" opt; do
+    #echo "$opt = $OPTARG"
+    case "$opt" in
+    h )
+        echo "$usage"
+        exit
+        ;;
+    d )
+        directory_name=$OPTARG
+        ;;
+    p )
+        photosynthetic_type=$OPTARG
+        ;;
+    s )
+        task="launch"
+        ;;
+    t )
+        suppress_storage_copy=true
+        ;;
+    c )
+        task="cleanup"
+        ;;
+    k )
+        task="kill"
+        ;;
+    u )
+        notify_url=$OPTARG
+        ;;
+    \?) printf "illegal option: -%s\n" "$OPTARG" >&2
+        echo "$usage" >&2
+        exit 1
+        ;;
+    esac
+done
+
+## Examine directories
+if [ -z "$directory_name" ]; then
+    echo "directory name required (-d)"
+    exit 1
+fi
+working_directory="$base_directory/$directory_name"
+if [ ! -d "$working_directory" ]; then
+    echo "working directory $working_directory not found"
+    exit 1
+fi
+input_directory="$working_directory/$input_directory_name"
+if [ ! -d "$input_directory" ]; then
+    echo "input directory $input_directory not found"
+    exit 1
+fi
+pid_path="$working_directory/$pid_filename"
+
+## Process task
+if [ "$task" = "launch" ]; then
+    # verify process isn't running yet
+    pid=$(head -n 1 $pid_path 2>/dev/null)
+    # if the pid exists, check the process status using ps
+    if ps -p $pid &>/dev/null; then
+        # if it is in ps, then it's still running
+        echo still running
+        exit 1
+    fi
+    
+    piscal_config_file="$working_directory"/piscal.cfg
+    # write config file for piscal
+    echo $photosynthetic_type > "$piscal_config_file"
+    find "$input_directory" -maxdepth 1 -type f\
+        -printf '%P\n'\
+        >> "$piscal_config_file"
+    
+    command="$launcher -d $working_directory -f piscal.cfg"
+    if [ "$suppress_storage_copy" = true ]; then
+        command="$command -t"
+    fi
+
+    if [ -n "$notify_url" ]; then
+        command="$command -u $notify_url"
+    fi
+
+    # launch it, sending error output to file
+    nohup ${command} > $working_directory/$stderr_filename 2>&1 &
+
+    # write the PID to a temp file to check for completion later
+    echo $! > $pid_path
+    echo started
+elif [ "$task" = "get_status" ] || [ "$task" = "get_status_cleaned_input" ]; then
+    # if the pid doesn't exist, then process hasn't started
+    if [ ! -f "$pid_path" ]; then
+        echo "not started"
+        exit
+    fi
+
+    pid=$(head -n 1 $pid_path 2>/dev/null)
+    # if the pid exists, check the process status using ps
+    if ps -p $pid > /dev/null; then
+        # if it is in ps, then it's still running
+        echo running
+    else
+        # otherwise, it is complete, check for runtime errors
+        if [ -s "$working_directory/$stderr_filename" ]; then
+            cat "$working_directory/$stderr_filename"
+            exit 1
+        fi
+        
+        output_directory="$working_directory/$output_directory_name"
+        if [ ! -d "$output_directory" ]; then
+            echo "output directory $output_directory not found"
+            exit 1
+        fi
+        
+        echo complete
+        if [ "$task" = "get_status" ]; then
+            echo "#touser"
+            find "$output_directory"/* 2>/dev/null
+            
+            cleaned_input_directory="$working_directory/$cleaned_input_directory_name"
+            echo "#clninput"
+            find "$cleaned_input_directory"/* 2>/dev/null
+            
+            nottouser_directory="$working_directory/$nottouser_directory_name"
+            echo "#nottouser"
+            find "$nottouser_directory"/* 2>/dev/null
+            exit 0
+       fi
+    fi
+elif [ "$task" = "cleanup" ]; then
+    pid=$(head -n 1 $pid_path 2>/dev/null)
+    # if the pid exists, check the process status using ps
+    if ps -p $pid > /dev/null; then
+        # if it is in ps, then it's still running
+        echo still running
+        exit 1
+    fi
+    rm -rf "$working_directory"
+elif [ "$task" = "kill" ]; then
+    # if the pid doesn't exist, then process hasn't started
+    if [ ! -f "$pid_path" ]; then
+        echo "not started"
+        exit 1
+    fi
+
+    pid=$(head -n 1 $pid_path 2>/dev/null)
+    # if the pid exists, check the process status using ps
+    if ps -p $pid > /dev/null; then
+        # if it is in ps, then it's still running
+        kill $pid
+    fi
+    echo killed
+fi

piscal-manager/subdir_year.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+for file in *; do 
+    ## Skip unless this is a file
+    if [ -f "$file" ]; then
+        ## get the first four characters from filename, aka the year
+        target="${file:0:4}"
+        ## Create the target directory if it doesn't exist
+        [ -d "$target" ] || mkdir  "$target" 2>/dev/null 
+        ## Move the current file
+        mv "$file" "$target" 2>/dev/null
+    fi
+done
+