diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..80457ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +#Download base image +FROM ubuntu:latest + +# Install and update software +RUN set -xe \ + && apt-get update \ + && apt-get upgrade -y \ + # SSHD install + && apt-get install --no-install-recommends -y openssh-server sudo \ + # Piscal reqs + && apt-get install make -y \ + && apt-get install xutils-dev -y \ + && apt-get install gfortran -y \ + && apt-get install libopenmpi-dev -y \ + # utils + && apt-get install iproute2 -y \ + && apt-get install vim -y \ + # cleanup + && apt-get autoclean -y \ + && apt-get autoremove -y + +# configure sshd, copied from wataken44/ubuntu-latest-sshd +RUN set -xe \ + && groupadd launcher \ + && useradd -g launcher -G sudo -m -s /bin/bash launcher \ + && echo 'launcher:launcher' | chpasswd + +RUN set -xe \ + && sed -i -e 's/#PasswordAuthentication.*/PasswordAuthentication yes/g' /etc/ssh/sshd_config \ + && sed -i -e 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \ + && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +RUN set -xe \ + && chown -R launcher:launcher /home/launcher + +# fix for SSHD - "Missing privilege separation directory: /run/sshd" +RUN set -xe \ + && mkdir /run/sshd + +##### add user l2g +# RUN set -xe \ +# && groupadd l2g \ +# && useradd -g l2g -G sudo -m -s /bin/bash l2g \ +# && echo 'l2g:pwdpwd' | chpasswd +# RUN set -xe \ +# && chown -R l2g:l2g /home/l2g +# RUN set -xe \ +# && apt-get install nano +##### + +ADD piscal-manager /srv +ADD leafres/testrun/piscal /srv +#RUN chmod R +x /srv/*.sh +WORKDIR /srv + +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/build-docker.sh b/build-docker.sh new file mode 100755 index 0000000..aa1e68b --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +sudo docker build -t piscal-server:latest . +#sudo docker run -it --rm -p 2222:22 --name my-piscal-server piscal-server diff --git a/piscal-manager/README.txt b/piscal-manager/README.txt new file mode 100644 index 0000000..0833542 --- /dev/null +++ b/piscal-manager/README.txt @@ -0,0 +1,31 @@ +./ # the working directory for Piscal processing +./piscal.cfg # Piscal configuration - first line has Photosynthetic Pathway type + # followed by a list of all files in ./input + # C3_photosynthesis_leafweb, C4_photosynthesis_leafweb + # CAM_photosynthesis_leafweb +./piscal.err # Contains any stderr output from launching Piscal, should be empty + # unless there is a problem with the system or executable +./piscal.pid # file containing one line - the process ID which Piscal + # is running under. this is used to check if Piscal is currently + # executing or if it has completed + +./input/ # directory containing all unprocessed LeafInput files +./input/Wild_Capsicum1.csv # example input copied from LeafWeb +./input/Wild_Capsicum2.csv # "" + +./output/ # directory piscal creates to store all output + +./output/clninput/ # directory containing cleaned leaf input, for the purpose + # of making that data available for search in LeafWeb +./output/clninput/20160311152630Wild_Capsicum1.csv # example cleaned input +./output/clninput/20160311152630Wild_Capsicum2.csv # "" + +./output/fitresult/ # directory containing fitresult generated by Piscal +./output/fitresult/touser/ # output results sent to user and to storage +./output/fitresult/touser/cntrlbestparameters.csv # example output +./output/fitresult/touser/errormessage # file containing any error + # messages processed by Piscal +./output/fitresult/touser/warningmessage # file containing any warning + # messages processed by Piscal + +./output/fitresult/nottouser/ # Output results sent to storage only \ No newline at end of file diff --git a/piscal-manager/piscal_launcher.cfg b/piscal-manager/piscal_launcher.cfg new file mode 100644 index 0000000..22e931c --- /dev/null +++ b/piscal-manager/piscal_launcher.cfg @@ -0,0 +1,2 @@ +piscal_executable="/home/piscaladmin/piscal_executable/piscal" +storage_directory="/home/piscaladmin/LeafWeb_storage" diff --git a/piscal-manager/piscal_launcher.sh b/piscal-manager/piscal_launcher.sh new file mode 100755 index 0000000..368c00b --- /dev/null +++ b/piscal-manager/piscal_launcher.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# piscal launcher + +usage="$(basename "$0") [-h] -d working_directory [-u notify_url] -- script to launch Piscal + +where: + -h show this help text + -d working directory + -f input filename + -t suppress storage copy + -u url to notify on completion" + +# http://stackoverflow.com/a/246128/99492 +base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +working_directory="" +output_directory_base_name="output" +input_directory_name="input" +cleaned_input_directory_name="clninput" +touser_directory_name="fitresult/touser" +nottouser_directory_name="fitresult/nottouser" + +# import the settings from piscal.cfg +# $piscal_executable and $storage_directory +. "$base_directory/piscal_launcher.cfg" + +while getopts "hd:f:u:t" opt; do + case "$opt" in + h ) + echo "$usage" + exit + ;; + d ) + working_directory=$OPTARG + ;; + t ) + suppress_storage_copy=true + ;; + u ) + notify_url=$OPTARG + ;; + \?) printf "illegal option: -%s\n" "$OPTARG" >&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done + +if [ -z "$working_directory" ]; then + echo "working directory required (-d)" + exit 1 +fi +if [ ! -d "$working_directory" ]; then + echo "working directory $working_directory not found" + exit 1 +fi +output_directory_base="$working_directory/$output_directory_base_name" +output_directory_touser="$output_directory_base/$touser_directory_name" +output_directory_nottouser="$output_directory_base/$nottouser_directory_name" +cleaned_input_directory="$output_directory_base/$cleaned_input_directory_name" +run_directory="$working_directory/run" + +# setup output directories +if [ ! -d "$output_directory_base" ]; then + mkdir "$output_directory_base" +fi +if [ ! -d "$cleaned_input_directory" ]; then + mkdir "$cleaned_input_directory" +fi + +#find "$working_directory/$input_directory_name" -maxdepth 1 -type f\ +# -exec cp {} "$cleaned_input_directory" \; + +if [ ! -d "$output_directory_base/fitresult" ]; then + mkdir "$output_directory_base/fitresult" +fi +if [ ! -d "$output_directory_touser" ]; then + mkdir "$output_directory_touser" +fi +if [ ! -d "$output_directory_nottouser" ]; then + mkdir "$output_directory_nottouser" +fi +if [ ! -d "$run_directory" ]; then + mkdir "$run_directory" +fi + +# run piscal +pushd $run_directory >> /dev/null + +if [ -z "$test_output_directory" ]; then + eval $piscal_executable +else + cp -r "$test_output_directory"/* "$output_directory_base"/ +fi + +popd >> /dev/null + +# copy output to storage +if [ -z "$suppress_storage_copy" ]; then + cp "$output_directory_touser"/* "$storage_directory"/output/ 2>/dev/null + cp "$output_directory_nottouser"/* "$storage_directory"/output/ 2>/dev/null + cp "$cleaned_input_directory"/* "$storage_directory"/input/ 2>/dev/null + + mv_script=$base_directory/subdir_year.sh + pushd "$storage_directory"/output >> /dev/null + "$mv_script" + popd >> /dev/null + pushd "$storage_directory"/input >> /dev/null + "$mv_script" + popd >> /dev/null +fi + +# notify given url of completion +if [ -n "$notify_url" ]; then + wget -qO- -t 1 "$notify_url" # &> /dev/null 2>&1 +fi diff --git a/piscal-manager/piscal_manager.sh b/piscal-manager/piscal_manager.sh new file mode 100755 index 0000000..2a7feb5 --- /dev/null +++ b/piscal-manager/piscal_manager.sh @@ -0,0 +1,186 @@ +#!/bin/bash +# piscal manager script + +usage="$(basename "$0") [-h] + -d directory_name + [-s|-c|-k] + -p photosynthetic_type + -u notify_url + -t + +where: + -h show this help text + -d working directory name + -s start job + -c cleanup directory + -k kill job + -u url to notify on completion + -p photosynthetic type + -t suppress storage copy" + +# Initialize variables: +# http://stackoverflow.com/a/246128/99492 +base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +directory_name="" +photosynthetic_type="C3_photosynthesis_leafweb" #default +task="get_status" # default task +input_directory_name="input" +pid_filename="piscal.pid" +stderr_filename="piscal.err" +cleaned_input_directory_name="output/clninput" +output_directory_name="output/fitresult/touser" +nottouser_directory_name="output/fitresult/nottouser" +launcher="$base_directory/piscal_launcher.sh" + +# http://stackoverflow.com/a/14203146/99492 +# http://wiki.bash-hackers.org/howto/getopts_tutorial + +while getopts "hd:f:p:stcku:" opt; do + #echo "$opt = $OPTARG" + case "$opt" in + h ) + echo "$usage" + exit + ;; + d ) + directory_name=$OPTARG + ;; + p ) + photosynthetic_type=$OPTARG + ;; + s ) + task="launch" + ;; + t ) + suppress_storage_copy=true + ;; + c ) + task="cleanup" + ;; + k ) + task="kill" + ;; + u ) + notify_url=$OPTARG + ;; + \?) printf "illegal option: -%s\n" "$OPTARG" >&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done + +## Examine directories +if [ -z "$directory_name" ]; then + echo "directory name required (-d)" + exit 1 +fi +working_directory="$base_directory/$directory_name" +if [ ! -d "$working_directory" ]; then + echo "working directory $working_directory not found" + exit 1 +fi +input_directory="$working_directory/$input_directory_name" +if [ ! -d "$input_directory" ]; then + echo "input directory $input_directory not found" + exit 1 +fi +pid_path="$working_directory/$pid_filename" + +## Process task +if [ "$task" = "launch" ]; then + # verify process isn't running yet + pid=$(head -n 1 $pid_path 2>/dev/null) + # if the pid exists, check the process status using ps + if ps -p $pid &>/dev/null; then + # if it is in ps, then it's still running + echo still running + exit 1 + fi + + piscal_config_file="$working_directory"/piscal.cfg + # write config file for piscal + echo $photosynthetic_type > "$piscal_config_file" + find "$input_directory" -maxdepth 1 -type f\ + -printf '%P\n'\ + >> "$piscal_config_file" + + command="$launcher -d $working_directory -f piscal.cfg" + if [ "$suppress_storage_copy" = true ]; then + command="$command -t" + fi + + if [ -n "$notify_url" ]; then + command="$command -u $notify_url" + fi + + # launch it, sending error output to file + nohup ${command} > $working_directory/$stderr_filename 2>&1 & + + # write the PID to a temp file to check for completion later + echo $! > $pid_path + echo started +elif [ "$task" = "get_status" ] || [ "$task" = "get_status_cleaned_input" ]; then + # if the pid doesn't exist, then process hasn't started + if [ ! -f "$pid_path" ]; then + echo "not started" + exit + fi + + pid=$(head -n 1 $pid_path 2>/dev/null) + # if the pid exists, check the process status using ps + if ps -p $pid > /dev/null; then + # if it is in ps, then it's still running + echo running + else + # otherwise, it is complete, check for runtime errors + if [ -s "$working_directory/$stderr_filename" ]; then + cat "$working_directory/$stderr_filename" + exit 1 + fi + + output_directory="$working_directory/$output_directory_name" + if [ ! -d "$output_directory" ]; then + echo "output directory $output_directory not found" + exit 1 + fi + + echo complete + if [ "$task" = "get_status" ]; then + echo "#touser" + find "$output_directory"/* 2>/dev/null + + cleaned_input_directory="$working_directory/$cleaned_input_directory_name" + echo "#clninput" + find "$cleaned_input_directory"/* 2>/dev/null + + nottouser_directory="$working_directory/$nottouser_directory_name" + echo "#nottouser" + find "$nottouser_directory"/* 2>/dev/null + exit 0 + fi + fi +elif [ "$task" = "cleanup" ]; then + pid=$(head -n 1 $pid_path 2>/dev/null) + # if the pid exists, check the process status using ps + if ps -p $pid > /dev/null; then + # if it is in ps, then it's still running + echo still running + exit 1 + fi + rm -rf "$working_directory" +elif [ "$task" = "kill" ]; then + # if the pid doesn't exist, then process hasn't started + if [ ! -f "$pid_path" ]; then + echo "not started" + exit 1 + fi + + pid=$(head -n 1 $pid_path 2>/dev/null) + # if the pid exists, check the process status using ps + if ps -p $pid > /dev/null; then + # if it is in ps, then it's still running + kill $pid + fi + echo killed +fi diff --git a/piscal-manager/subdir_year.sh b/piscal-manager/subdir_year.sh new file mode 100755 index 0000000..fb16fba --- /dev/null +++ b/piscal-manager/subdir_year.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +for file in *; do + ## Skip unless this is a file + if [ -f "$file" ]; then + ## get the first four characters from filename, aka the year + target="${file:0:4}" + ## Create the target directory if it doesn't exist + [ -d "$target" ] || mkdir "$target" 2>/dev/null + ## Move the current file + mv "$file" "$target" 2>/dev/null + fi +done +