Add piscal manager scripts
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
./ # the working directory for Piscal processing
|
||||
./piscal.cfg # Piscal configuration - first line has Photosynthetic Pathway type
|
||||
# followed by a list of all files in ./input
|
||||
# C3_photosynthesis_leafweb, C4_photosynthesis_leafweb
|
||||
# CAM_photosynthesis_leafweb
|
||||
./piscal.err # Contains any stderr output from launching Piscal, should be empty
|
||||
# unless there is a problem with the system or executable
|
||||
./piscal.pid # file containing one line - the process ID which Piscal
|
||||
# is running under. this is used to check if Piscal is currently
|
||||
# executing or if it has completed
|
||||
|
||||
./input/ # directory containing all unprocessed LeafInput files
|
||||
./input/Wild_Capsicum1.csv # example input copied from LeafWeb
|
||||
./input/Wild_Capsicum2.csv # ""
|
||||
|
||||
./output/ # directory piscal creates to store all output
|
||||
|
||||
./output/clninput/ # directory containing cleaned leaf input, for the purpose
|
||||
# of making that data available for search in LeafWeb
|
||||
./output/clninput/20160311152630Wild_Capsicum1.csv # example cleaned input
|
||||
./output/clninput/20160311152630Wild_Capsicum2.csv # ""
|
||||
|
||||
./output/fitresult/ # directory containing fitresult generated by Piscal
|
||||
./output/fitresult/touser/ # output results sent to user and to storage
|
||||
./output/fitresult/touser/cntrlbestparameters.csv # example output
|
||||
./output/fitresult/touser/errormessage # file containing any error
|
||||
# messages processed by Piscal
|
||||
./output/fitresult/touser/warningmessage # file containing any warning
|
||||
# messages processed by Piscal
|
||||
|
||||
./output/fitresult/nottouser/ # Output results sent to storage only
|
||||
@@ -0,0 +1,2 @@
|
||||
piscal_executable="/home/piscaladmin/piscal_executable/piscal"
|
||||
storage_directory="/home/piscaladmin/LeafWeb_storage"
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/bash
|
||||
# piscal launcher
|
||||
|
||||
usage="$(basename "$0") [-h] -d working_directory [-u notify_url] -- script to launch Piscal
|
||||
|
||||
where:
|
||||
-h show this help text
|
||||
-d working directory
|
||||
-f input filename
|
||||
-t suppress storage copy
|
||||
-u url to notify on completion"
|
||||
|
||||
# http://stackoverflow.com/a/246128/99492
|
||||
base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
working_directory=""
|
||||
output_directory_base_name="output"
|
||||
input_directory_name="input"
|
||||
cleaned_input_directory_name="clninput"
|
||||
touser_directory_name="fitresult/touser"
|
||||
nottouser_directory_name="fitresult/nottouser"
|
||||
|
||||
# import the settings from piscal.cfg
|
||||
# $piscal_executable and $storage_directory
|
||||
. "$base_directory/piscal_launcher.cfg"
|
||||
|
||||
while getopts "hd:f:u:t" opt; do
|
||||
case "$opt" in
|
||||
h )
|
||||
echo "$usage"
|
||||
exit
|
||||
;;
|
||||
d )
|
||||
working_directory=$OPTARG
|
||||
;;
|
||||
t )
|
||||
suppress_storage_copy=true
|
||||
;;
|
||||
u )
|
||||
notify_url=$OPTARG
|
||||
;;
|
||||
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$working_directory" ]; then
|
||||
echo "working directory required (-d)"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -d "$working_directory" ]; then
|
||||
echo "working directory $working_directory not found"
|
||||
exit 1
|
||||
fi
|
||||
output_directory_base="$working_directory/$output_directory_base_name"
|
||||
output_directory_touser="$output_directory_base/$touser_directory_name"
|
||||
output_directory_nottouser="$output_directory_base/$nottouser_directory_name"
|
||||
cleaned_input_directory="$output_directory_base/$cleaned_input_directory_name"
|
||||
run_directory="$working_directory/run"
|
||||
|
||||
# setup output directories
|
||||
if [ ! -d "$output_directory_base" ]; then
|
||||
mkdir "$output_directory_base"
|
||||
fi
|
||||
if [ ! -d "$cleaned_input_directory" ]; then
|
||||
mkdir "$cleaned_input_directory"
|
||||
fi
|
||||
|
||||
#find "$working_directory/$input_directory_name" -maxdepth 1 -type f\
|
||||
# -exec cp {} "$cleaned_input_directory" \;
|
||||
|
||||
if [ ! -d "$output_directory_base/fitresult" ]; then
|
||||
mkdir "$output_directory_base/fitresult"
|
||||
fi
|
||||
if [ ! -d "$output_directory_touser" ]; then
|
||||
mkdir "$output_directory_touser"
|
||||
fi
|
||||
if [ ! -d "$output_directory_nottouser" ]; then
|
||||
mkdir "$output_directory_nottouser"
|
||||
fi
|
||||
if [ ! -d "$run_directory" ]; then
|
||||
mkdir "$run_directory"
|
||||
fi
|
||||
|
||||
# run piscal
|
||||
pushd $run_directory >> /dev/null
|
||||
|
||||
if [ -z "$test_output_directory" ]; then
|
||||
eval $piscal_executable
|
||||
else
|
||||
cp -r "$test_output_directory"/* "$output_directory_base"/
|
||||
fi
|
||||
|
||||
popd >> /dev/null
|
||||
|
||||
# copy output to storage
|
||||
if [ -z "$suppress_storage_copy" ]; then
|
||||
cp "$output_directory_touser"/* "$storage_directory"/output/ 2>/dev/null
|
||||
cp "$output_directory_nottouser"/* "$storage_directory"/output/ 2>/dev/null
|
||||
cp "$cleaned_input_directory"/* "$storage_directory"/input/ 2>/dev/null
|
||||
|
||||
mv_script=$base_directory/subdir_year.sh
|
||||
pushd "$storage_directory"/output >> /dev/null
|
||||
"$mv_script"
|
||||
popd >> /dev/null
|
||||
pushd "$storage_directory"/input >> /dev/null
|
||||
"$mv_script"
|
||||
popd >> /dev/null
|
||||
fi
|
||||
|
||||
# notify given url of completion
|
||||
if [ -n "$notify_url" ]; then
|
||||
wget -qO- -t 1 "$notify_url" # &> /dev/null 2>&1
|
||||
fi
|
||||
Executable
+186
@@ -0,0 +1,186 @@
|
||||
#!/bin/bash
|
||||
# piscal manager script
|
||||
|
||||
usage="$(basename "$0") [-h]
|
||||
-d directory_name
|
||||
[-s|-c|-k]
|
||||
-p photosynthetic_type
|
||||
-u notify_url
|
||||
-t
|
||||
|
||||
where:
|
||||
-h show this help text
|
||||
-d working directory name
|
||||
-s start job
|
||||
-c cleanup directory
|
||||
-k kill job
|
||||
-u url to notify on completion
|
||||
-p photosynthetic type
|
||||
-t suppress storage copy"
|
||||
|
||||
# Initialize variables:
|
||||
# http://stackoverflow.com/a/246128/99492
|
||||
base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
directory_name=""
|
||||
photosynthetic_type="C3_photosynthesis_leafweb" #default
|
||||
task="get_status" # default task
|
||||
input_directory_name="input"
|
||||
pid_filename="piscal.pid"
|
||||
stderr_filename="piscal.err"
|
||||
cleaned_input_directory_name="output/clninput"
|
||||
output_directory_name="output/fitresult/touser"
|
||||
nottouser_directory_name="output/fitresult/nottouser"
|
||||
launcher="$base_directory/piscal_launcher.sh"
|
||||
|
||||
# http://stackoverflow.com/a/14203146/99492
|
||||
# http://wiki.bash-hackers.org/howto/getopts_tutorial
|
||||
|
||||
while getopts "hd:f:p:stcku:" opt; do
|
||||
#echo "$opt = $OPTARG"
|
||||
case "$opt" in
|
||||
h )
|
||||
echo "$usage"
|
||||
exit
|
||||
;;
|
||||
d )
|
||||
directory_name=$OPTARG
|
||||
;;
|
||||
p )
|
||||
photosynthetic_type=$OPTARG
|
||||
;;
|
||||
s )
|
||||
task="launch"
|
||||
;;
|
||||
t )
|
||||
suppress_storage_copy=true
|
||||
;;
|
||||
c )
|
||||
task="cleanup"
|
||||
;;
|
||||
k )
|
||||
task="kill"
|
||||
;;
|
||||
u )
|
||||
notify_url=$OPTARG
|
||||
;;
|
||||
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
## Examine directories
|
||||
if [ -z "$directory_name" ]; then
|
||||
echo "directory name required (-d)"
|
||||
exit 1
|
||||
fi
|
||||
working_directory="$base_directory/$directory_name"
|
||||
if [ ! -d "$working_directory" ]; then
|
||||
echo "working directory $working_directory not found"
|
||||
exit 1
|
||||
fi
|
||||
input_directory="$working_directory/$input_directory_name"
|
||||
if [ ! -d "$input_directory" ]; then
|
||||
echo "input directory $input_directory not found"
|
||||
exit 1
|
||||
fi
|
||||
pid_path="$working_directory/$pid_filename"
|
||||
|
||||
## Process task
|
||||
if [ "$task" = "launch" ]; then
|
||||
# verify process isn't running yet
|
||||
pid=$(head -n 1 $pid_path 2>/dev/null)
|
||||
# if the pid exists, check the process status using ps
|
||||
if ps -p $pid &>/dev/null; then
|
||||
# if it is in ps, then it's still running
|
||||
echo still running
|
||||
exit 1
|
||||
fi
|
||||
|
||||
piscal_config_file="$working_directory"/piscal.cfg
|
||||
# write config file for piscal
|
||||
echo $photosynthetic_type > "$piscal_config_file"
|
||||
find "$input_directory" -maxdepth 1 -type f\
|
||||
-printf '%P\n'\
|
||||
>> "$piscal_config_file"
|
||||
|
||||
command="$launcher -d $working_directory -f piscal.cfg"
|
||||
if [ "$suppress_storage_copy" = true ]; then
|
||||
command="$command -t"
|
||||
fi
|
||||
|
||||
if [ -n "$notify_url" ]; then
|
||||
command="$command -u $notify_url"
|
||||
fi
|
||||
|
||||
# launch it, sending error output to file
|
||||
nohup ${command} > $working_directory/$stderr_filename 2>&1 &
|
||||
|
||||
# write the PID to a temp file to check for completion later
|
||||
echo $! > $pid_path
|
||||
echo started
|
||||
elif [ "$task" = "get_status" ] || [ "$task" = "get_status_cleaned_input" ]; then
|
||||
# if the pid doesn't exist, then process hasn't started
|
||||
if [ ! -f "$pid_path" ]; then
|
||||
echo "not started"
|
||||
exit
|
||||
fi
|
||||
|
||||
pid=$(head -n 1 $pid_path 2>/dev/null)
|
||||
# if the pid exists, check the process status using ps
|
||||
if ps -p $pid > /dev/null; then
|
||||
# if it is in ps, then it's still running
|
||||
echo running
|
||||
else
|
||||
# otherwise, it is complete, check for runtime errors
|
||||
if [ -s "$working_directory/$stderr_filename" ]; then
|
||||
cat "$working_directory/$stderr_filename"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
output_directory="$working_directory/$output_directory_name"
|
||||
if [ ! -d "$output_directory" ]; then
|
||||
echo "output directory $output_directory not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo complete
|
||||
if [ "$task" = "get_status" ]; then
|
||||
echo "#touser"
|
||||
find "$output_directory"/* 2>/dev/null
|
||||
|
||||
cleaned_input_directory="$working_directory/$cleaned_input_directory_name"
|
||||
echo "#clninput"
|
||||
find "$cleaned_input_directory"/* 2>/dev/null
|
||||
|
||||
nottouser_directory="$working_directory/$nottouser_directory_name"
|
||||
echo "#nottouser"
|
||||
find "$nottouser_directory"/* 2>/dev/null
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
elif [ "$task" = "cleanup" ]; then
|
||||
pid=$(head -n 1 $pid_path 2>/dev/null)
|
||||
# if the pid exists, check the process status using ps
|
||||
if ps -p $pid > /dev/null; then
|
||||
# if it is in ps, then it's still running
|
||||
echo still running
|
||||
exit 1
|
||||
fi
|
||||
rm -rf "$working_directory"
|
||||
elif [ "$task" = "kill" ]; then
|
||||
# if the pid doesn't exist, then process hasn't started
|
||||
if [ ! -f "$pid_path" ]; then
|
||||
echo "not started"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pid=$(head -n 1 $pid_path 2>/dev/null)
|
||||
# if the pid exists, check the process status using ps
|
||||
if ps -p $pid > /dev/null; then
|
||||
# if it is in ps, then it's still running
|
||||
kill $pid
|
||||
fi
|
||||
echo killed
|
||||
fi
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
for file in *; do
|
||||
## Skip unless this is a file
|
||||
if [ -f "$file" ]; then
|
||||
## get the first four characters from filename, aka the year
|
||||
target="${file:0:4}"
|
||||
## Create the target directory if it doesn't exist
|
||||
[ -d "$target" ] || mkdir "$target" 2>/dev/null
|
||||
## Move the current file
|
||||
mv "$file" "$target" 2>/dev/null
|
||||
fi
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user