#!/bin/bash

##########################################################################################################
# This acts as a wrapper/filter to qsub. Written by Sreedhar Manchu (sm4082@nyu.edu)
# Added on Sep 24th, 2012
# http://www.clusterresources.com/torquedocs21/a.jqsubwrapper.shtml
##########################################################################################################

# Memory Variables.
# If you want to provide more memory for jobs than the currently allocated memory on
# nodes on certain chassis you just need to change these variables.

# Note: The below mentioned memory limit values based on the physical memory present
# on the node act as limits to the queues too. 

mem48gb_limit=46               		# Right now 46GB memory is allocated for jobs on 48GB nodes.
mem96gb_limit=94		       	# Right now 94GB memory is allocated for jobs on 96GB nodes 
mem192gb_limit=190		       	# Right now 194GB memory is allocated for jobs on 192GB nodes 
mem1024gb_limit=1000		       	# Right now 1000GB memory is allocated for jobs on 1TB node.
bigmem_memory_lower_limit=47   		# Right now 47GB is the minimum memory requirement for bigmem queue.

memory_per_core=2048				# Memory per core for serial jobs in MB.

wtime_p12_max=43200     			# p12 queue maximum walltime in seconds.
wtime_p48_min=43260				# p48 queue minimum walltime in seconds.
wtime_p48_max=172800				# p48 queue maximum walltime in seconds.
wtime_bigmem_max=172800				# bigmem queue maximum walltime in seconds.
wtime_interactive_max=14400 			# interactive queue maximum walltime in seconds.
wtime_gpu_max=172800				# gpu queue maximum walltime in seconds.
wtime_s48_min=60	    			# s48 queue minimum walltime in seconds (1 minute).
wtime_s48_max=172800    			# s48 queue maximum walltime in seconds (48 hours)
wtime_s96_min=172860	    			# s96 queue minimum walltime in seconds (48 hours and 1 minute).
wtime_s96_max=345600    			# s96 queue maximum walltime in seconds (96 hours)
wtime_default=3600				# Deafault walltime for all the queues

pbsq_from_cmdline="false"
pbsq="false"
pbsS_from_cmdline="false"
pbsS="false"
pbsln_from_cmdline="false"
pbsln="false"
pbslw_from_cmdline="false"
pbslw="false"
pbslm_from_cmdline="false"
pbslm="false"
pbsN_from_cmdline="false"
pbsN="false"
pbso_from_cmdline="false"
pbso="false"
pbse_from_cmdline="false"
pbse="false"
pbsd_from_cmdline="false"
pbsd="false"
pbst_from_cmdline="false"
pbst="false"
pbsI="false"

line_count=0

reading_pbs_operatives_is="not_done"
parsing_pbs_oed_lines_is="not_done"
counting_resources_is="not_done"
node_features_are="not_added"

function quit ()
{
	exit -1
}

function print_run_info()
{
	echo -e "\nFor more information please refter to HPC Wiki at" >&2
	echo -e "https://wikis.nyu.edu/display/NYUHPC/Running+jobs\n" >&2
}

function print_run_info_quit ()
{
	print_run_info
	quit
}

function print_queues_info ()
{
	echo -e "\nFor more information please refer to HPC wiki at" >&2
	echo -e "https://wikis.nyu.edu/display/NYUHPC/Queues\n" >&2
}

function print_queues_info_quit ()
{
	print_queues_info
	quit
}

function print_butinah_info ()
{
	echo -e "\nFor more information please refer to HPC Wiki at" >&2
	echo -e "https://wikis.nyu.edu/display/NYUHPC/Bowery\n" >&2
	quit
}

function no_queue_declared ()
{
	if [ $node_count -eq 1 ];then
		if [ $ppn_count -gt 12 ];then
			echo -e "\nMaximum ppn count for a serial job is 12." >&2
			print_queues_info_quit
		fi
		if [ $walltime -lt $wtime_s48_min ];then
			echo -e "\nMinimum walltime for serial jobs is $(($wtime_s48_min/60)) minute." >&2
			print_queues_info_quit
		fi
		if [ -n "$memory" ];then
			if [ $memory -gt $(($mem48gb_limit*1024*1024)) ];then
				echo -e "\nMaximum memory available for serial jobs is ${mem48gb_limit}GB.\nIf you need more than this use bigmem queue." >&2
				print_queues_info_quit
			fi
		else
			echo "#PBS -l mem=$((${memory_per_core}*${ppn_count}))MB"
		fi	
		echo "#PBS -l feature=serial"
	else        
		if [ $walltime -gt $wtime_p48_max ];then
			echo -e "\nThe maximum walltime allowed is $(($wtime_p48_max/3600)) hours." >&2
			print_queues_info_quit
		elif [ $walltime -gt $wtime_p12_max -a $walltime -le $wtime_p48_max ];then
			echo "#PBS -q p48"
			#echo "#PBS -l feature=chassis3-6"
			echo "#PBS -l feature=parallel"
			if [ -z "$memory" ];then
				echo "#PBS -l mem=${mem48gb_limit}GB"
echo
			else
				if [ $memory -gt $(($mem48gb_limit*1024*1024)) ];then
					echo -e "\nIf requested walltime is more than $(($wtime_p12_max/3600)) hours," >&2
					echo -e "without explicit queue declaration, maximum memory available is ${mem48gb_limit}GB." >&2
					print_queues_info_quit
				fi
			fi
		elif [ $walltime -le $wtime_p12_max ];then
			if [ $ppn_count -gt 12 ];then
				echo -e "\nMaximum number of ppn available for parallel jobs is 12." >&2
				print_butinah_info
			else
				if [ -z "$memory" ];then
					echo "#PBS -q p12"
					#echo "#PBS -l feature=chassis7-16"
					echo "#PBS -l feature=parallel"
					echo "#PBS -l mem=${mem48gb_limit}GB"
				else
					if [ $memory -gt $(($mem48gb_limit*1024*1024)) ];then
						echo -e "\nMaximum memory available for parallel runs is ${mem48gb_limit}GB." >&2
						print_butinah_info
					else
						echo "#PBS -q p12"
						#echo "#PBS -l feature=chassis4-9"
						echo "#PBS -l feature=parallel"
					fi
				fi
			fi
		fi
	fi
}

function q_interactive ()
{
	if [ $node_count -gt 2 ];then
		echo -e "\nMaximum node count for an interactive job is 2." >&2
		print_queues_info_quit
	fi
	if [ $ppn_count -gt 12 ];then
		echo -e "\nMaximum number of processors per node (ppn) for interactive queue is 12." >&2
		print_queues_info_quit
	fi
	if [ $walltime -gt $wtime_interactive_max ];then
		echo -e "\nMaximum walltime for interactive queue is $(($wtime_interactive_max/3600)) hours." >&2
		print_queues_info_quit
	fi
	echo "#PBS -l feature=interactive"
}

function q_gpu ()
{
	if [ $ppn_count -gt 12 ];then
		echo -e "\nMaximum number of processors per node (ppn) for queue gpu is 12." >&2
		print_queues_info_quit
	fi
	if [ $walltime -gt $wtime_gpu_max ];then
		echo -e "\nThe maximum walltime for gpu queue is $(($wtime_gpu_max/3600)) hours." >&2
		print_queues_info_quit
	fi
	if [ -n "$memory" ];then
		if [ $memory -gt $(($mem96gb_limit*1024*1024)) ];then
			echo -e "\nMaximum memory available for gpu jobs is ${mem96gb_limit}GB." >&2
			print_queues_info_quit
		fi
	else
		echo "#PBS -l mem=${mem96gb_limit}GB"
	fi	
	echo "#PBS -l feature=gpu"
}

function q_p48 ()
{
	if [ $walltime -lt $wtime_p48_min ];then
		echo -e "\nThe minimum walltime for p48 queue is $(($wtime_p48_min/3600)) hours and $((($wtime_p48_min-3600*($wtime_p48_min/3600))/60)) minute." > /dev/tty
		print_queues_info_quit
	elif [ $walltime -gt $wtime_p48_max ];then
		echo -e "\nThe maximum walltime for p48 jobs is $(($wtime_p48_max/3600)) hours." >&2
		print_queues_info_quit
        fi
	if [ $node_count -eq 1 ];then    
		echo -e "\nMinimum number of nodes needed for queue p48 is 2." >&2
		print_run_info_quit
	fi      
	if [ $ppn_count -gt 12 ];then    
		echo -e "\nMaximum number of processors per node (ppn) for queue p48 is 12." >&2
		print_run_info_quit
	fi
	if [ -n "$memory" ];then
		if [ $memory -gt $(($mem48gb_limit*1024*1024)) ];then
			echo -e "\nThe maximum memory p48 queue can support is ${mem48gb_limit}GB." >&2
			print_run_info_quit
		else
			#echo "#PBS -l feature=chassis0-3"
			echo "#PBS -l feature=parallel"
		fi
	else
		echo "#PBS -l mem=${mem48gb_limit}GB"
		#echo "#PBS -l feature=chassis0-3"
		echo "#PBS -l feature=parallel"
	fi
}

function q_p12 ()
{
	if [ $node_count -eq 1 ];then
		echo -e "\nMinimum number of nodes needed for queue p12 is 2." >&2
		print_run_info_quit
	fi
	if [ $ppn_count -gt 12 ];then
		echo -e "\nMaximum number of processors per node (ppn) for queue p12 is 12." >&2
		print_run_info_quit
	fi
	if [ $walltime -gt $wtime_p12_max ];then
        echo -e "\nThe maximum walltime for p12 jobs is $(($wtime_p12_max/3600)) hours." >&2
        print_queues_info_quit
    fi	
	if [ -n "$memory" ];then
		if [ $memory -le $(($mem48gb_limit*1024*1024)) ];then
			#echo "#PBS -l feature=chassis4-9"
			echo "#PBS -l feature=parallel"
		else
			echo -e "\nThe maximum memory for parallel jobs is ${mem48gb_limit}GB." >&2
			print_run_info_quit
		fi
	else
		echo "#PBS -l mem=${mem48gb_limit}GB"
		#echo "#PBS -l feature=chassis4-9"
		echo "#PBS -l feature=parallel"
	fi
}

function q_bigmem ()
{
	if [ $walltime -gt $wtime_bigmem_max ];then
        echo -e "\nThe maximum walltime for bigmem jobs is $(($wtime_bigmem_max/3600)) hours." >&2
        print_queues_info_quit
    fi
	if [ -z "$memory" ];then
		echo -e "\nBigmem queue needs memory declaration in the pbs script." >&2
		echo -e "For example, if your job needs 47 GB memory, job script should contain a line:" >&2
		echo -e "\n#PBS -l mem=47GB" >&2
		print_run_info_quit
	else
		if [ "$memory" -lt $(($bigmem_memory_lower_limit*1024*1024)) ];then
			echo -e "\nMinimum memory requirement for bigmem queue is ${bigmem_memory_lower_limit}GB." >&2
			print_queues_info_quit
		fi
		if [ $node_count -eq 1 ];then
			if [ $ppn_count -gt 32 ]     # For serial jobs on bigmem queue, maximum processors is 32
			then
				echo -e "\nMaximum number of processors per node (ppn) for bigmem serial job is 32." >&2
				print_run_info_quit
			fi
			if [ $memory -gt $(($mem48gb_limit*1024*1024)) -a $memory -le $(($mem192gb_limit*1024*1024)) ];then
				if [ $ppn_count -gt 12 ];then
					#echo "#PBS -l feature=chassis19"
					echo "#PBS -l feature=superfat"
				else
					#echo "#PBS -l feature=chassis17"
					echo "#PBS -l feature=fat"
				fi
			elif [ $memory -gt $(($mem192gb_limit*1024*1024)) -a $memory -le $(($mem1024gb_limit*1024*1024)) ];then
				#echo "#PBS -l feature=chassis19"
				echo "#PBS -l feature=superfat"
			elif [ $memory -gt $(($mem1024gb_limit*1024*1024)) ];then
				echo -e "\nThe maximum memory for bigmem serial job is ${mem1024gb_limit}GB." >&2
				print_run_info_quit
			fi
		else
			if [ $ppn_count -gt 12 ];then
				echo -e "\nMaximum number of processors per node (ppn) for bigmem parallel job is 12." >&2
				print_run_info_quit
			fi
			if [ $memory -gt $(($mem48gb_limit*1024*1024)) -a $memory -le $(($mem192gb_limit*1024*1024)) ];then
				#echo "#PBS -l feature=chassis9"   
				echo "#PBS -l feature=fat" 
			else
				echo -e "\nThe maximum memory for parallel jobs is ${mem192gb_limit}GB." >&2
				print_run_info_quit
			fi
		fi
	fi
}

function q_s48 ()
{
	if [ $node_count -gt 1 ];then
		echo -e "\nMaximum node count for a serial job is 1." >&2
		print_queues_info_quit
	fi
	if [ $ppn_count -gt 12 ];then
		echo -e "\nMaximum ppn count for a serial job is 12." >&2
		print_queues_info_quit
	fi
	if [ $walltime -lt $wtime_s48_min ];then
		echo -e "\nMinimum walltime for a serial job is $(($wtime_s48_min/60)) minute." >&2
		print_queues_info_quit
	elif [ $walltime -gt $wtime_s48_max ];then
		echo -e "\nMaximum walltime for a serial job is $(($wtime_s48_max/3600)) hours." >&2
		print_queues_info_quit
	fi
	if [ -n "$memory" ];then
		if [ $memory -gt $(($mem48gb_limit*1024*1024)) ];then
			echo -e "\nMaximum memory available for serial jobs under s48\nqueue is ${mem48gb_limit}GB. If you need more than this you can\nuse bigmem queue." >&2
			print_queues_info_quit
		fi
	else
		echo "#PBS -l mem=$((${memory_per_core}*${ppn_count}))MB"
	fi	
	echo "#PBS -l feature=serial"
}

function q_s96 ()
{
	if [ $node_count -gt 1 ];then
		echo -e "\nMaximum node count for a serial job is 1." >&2
		print_queues_info_quit
	fi
	if [ $ppn_count -gt 12 ];then
		echo -e "\nMaximum ppn count for a serial job is 12." >&2
		print_queues_info_quit
	fi
	if [ $walltime -lt $wtime_s96_min ];then
		echo -e "\nThe minimum walltime for s96 queue is $(($wtime_s96_min/3600)) hours and $((($wtime_s96_min-3600*($wtime_s96_min/3600))/60)) minute." > /dev/tty
		print_queues_info_quit
	elif [ $walltime -gt $wtime_s96_max ];then
		echo -e "\nMaximum walltime for a serial job is $(($wtime_s96_max/3600)) hours." >&2
		print_queues_info_quit
	fi
	if [ -n "$memory" ];then
		if [ $memory -gt $(($mem48gb_limit*1024*1024)) ];then
			echo -e "\nMaximum memory available for serial jobs under s96\nqueue is ${mem48gb_limit}GB. If you need more than this you can\nuse bigmem queue." >&2
			print_queues_info_quit
		fi
	else
		echo "#PBS -l mem=$((${memory_per_core}*${ppn_count}))MB"
	fi	
	echo "#PBS -l feature=serial"
}

function process_queue_settings ()
{               
		case "$queue" in
			   s48) q_s48 ;;
			   s96) q_s96 ;;
			bigmem) q_bigmem ;;
			   p12) q_p12 ;;
			   p48) q_p48 ;;
			  gpu) q_gpu ;;
	   interactive) q_interactive ;;
				 *) if [ -z "$queue" ];then
						no_queue_declared
					else    
                        echo -e "\nUnknown queue. $queue doesn't exist." >&2
                        print_queues_info_quit
                    fi
		esac
}

function get_node_features ()
{
	process_queue_settings
}

function apply_defaults ()
{
	if [ -z $node_count ];then
		node_count=1
		ppn_count=1
	fi

	if [ -z "$walltime" ];then
		if [ -z "$queue" ];then
			walltime=$wtime_default
		else
			case "$queue" in
					 *) walltime=$wtime_default
			esac
		fi
	fi
}

function get_memory ()
{
	if echo $1 | egrep " *mem=0*[1-9][0-9]*[a-zA-Z][a-zA-Z]" 2>&1 >/dev/null;then
		if echo $1 | egrep -w " *mem=0*[1-9][0-9]*[a-zA-Z][a-zA-Z]" 2>&1 >/dev/null;then
			mtag=`echo $1 | egrep -o ' *mem=0*[1-9][0-9]*[a-zA-Z][a-zA-Z]' | cut -d= -f2 | egrep -o '[a-zA-Z][a-zA-Z]'`
			memory=`echo $1 | egrep -o 'mem=0*[1-9][0-9]*' | cut -d"=" -f2`
			if echo $mtag | egrep '[gG][bB]' 2>&1 >/dev/null;then
			#if [[ $mtag =~ "[gG][bB]" ]];then
				memory=`expr $memory '*' 1024 '*' 1024`
				#memory=$(($memory*1024*1024))
			elif echo $mtag | egrep '[mM][bB]' 2>&1 >/dev/null;then
			#if [[ $mtag =~ "[mM][bB]" ]];then
				memory=`expr $memory '*' 1024`
				#memory=$(($memory*1024))
			elif echo $mtag | egrep '[kK][bB]' 2>&1 >/dev/null;then
			#if [[ $mtag =~ "[kK][bB]" ]];then
				memory=$memory
			else
				echo -e "\nMemory should be either in Kilo Bytes(kb, KB) or Mega Bytes (mb, MB) or Giga Bytes (gb, GB)." >&2
				print_run_info_quit
			fi
		else
			echo -e "\nMemory tag should not be more than two letters. It can be just one of kb,KB,mb,MB,gb and GB." >&2
			print_run_info_quit
		fi
	fi
}

function get_walltime ()
{
	if echo $1 | egrep -o '( *|,)walltime=[0-9]*:?[0-9]*:?[0-9]*:?[0-9]*($|,)' 2>&1 >/dev/null;then
		wallstring=`echo $1 | egrep -o '( *|,)walltime=[0-9]*:?[0-9]*:?[0-9]*:?[0-9]*($|,)' | cut -f2 -d"="`
		if [ `echo "$wallstring" | egrep -o '^[0-9]*:?[0-9]*:?[0-9]*:?[0-9]*,$'` ];then
			wallstring=`echo "$wallstring" | cut -f1 -d","`
		fi
		if [ -z "$wallstring" ];then
			walltime=""
		elif [ `echo "$wallstring" | egrep -o '^:{1,3}$'` ];then
			wallstring='00:00:00'
		else
			number=`echo "$wallstring" | tr -dc ":" | wc -c`
			for  (( j=$(($number+1)); j>0; j-- ))
			do
				if [ -n "`echo "$wallstring" | cut -f"$j" -d":"`" -a $j -eq $(($number+1)) ];then
					walltime=`echo "$wallstring" | cut -f"$j" -d":"  | sed 's/^0*//;s/^$/0/'`
				elif [ -n "`echo "$wallstring" | cut -f$j -d":"`" -a $j -eq $number ];then
					walltime=$(($walltime+`echo "$wallstring" | cut -f$j -d":"  | sed 's/^0*//;s/^$/0/'`*60))
				elif [ -n "`echo "$wallstring" | cut -f$j -d":"`" -a $j -eq $(($number-1)) ];then
					walltime=$(($walltime+`echo "$wallstring" | cut -f$j -d":"  | sed 's/^0*//;s/^$/0/'`*60*60))
				elif [ -n "`echo "$wallstring" | cut -f$j -d":"`" -a $j -eq $(($number-2)) ];then
					walltime=$(($walltime+`echo "$wallstring" | cut -f$j -d":"  | sed 's/^0*//;s/^$/0/'`*24*60*60))
				fi
			done
		fi
	fi
}

function get_ppn_count ()
{
	if echo $1 | egrep "nodes=0*[1-9][0-9]*(:ppn=0*[1-9][0-9]*)?(,|$)" | egrep -v ",$" 2>&1 >/dev/null;then
		if echo $1 | egrep ":ppn=0*[1-9][0-9]*" 2>&1 >/dev/null;then
			ppn_count=`echo $1 | egrep -o 'ppn=0*[1-9][0-9]*' | cut -d"=" -f2`
		else
			ppn_count=1
		fi
	elif echo $1 | egrep "nodes=((compute-([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])(:ppn=0*[1-9][0-9]*)?\+?)+|(compute-17-[1-8](:ppn=0*[1-9][0-9]*)?\+?)+|compute-19-1(:ppn=0*[1-9][0-9]*)?|(gpu-18-([1-9]|1[0-6])(:ppn=0*[1-9][0-9]*)?\+?)+)($|,)" | egrep -v ",$|\+$|\+,.+$"  2>&1 >/dev/null;then
		node_count=`echo $1 | egrep -o "(compute-([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])|compute-17-[1-8]|compute-19-1|gpu-18-([1-9]|1[0-6]))+" | wc -l` 
		for((count=1;count<=$node_count;count++))
        do
			host_name=`echo $1 | cut -f"$count" -d+ | egrep -o "(compute-(([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])|17-[1-8]|19-1)|gpu-18-([1-9]|1[0-6]))"` 
			if echo $1 | cut -f"$count" -d+ |  egrep "ppn=0*[1-9][0-9]?" 2>&1 >/dev/null;then
				ppn_count=`echo $1 | cut -f"$count" -d+ | egrep -o "ppn=0*[1-9][0-9]?" | cut -f2 -d=`
			else
				ppn_count=1
			fi
			if echo $host_name | egrep "(compute-(([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])|17-[1-8])|gpu-18-([1-9]|1[0-6]))" 2>&1 >/dev/null;then
				if [ $ppn_count -gt 12 ];then
					echo -e "\nMaximum ppn for the requested compute node $host_name is 12" >&2
					print_butinah_info
				fi
			elif echo $host_name | egrep "compute-19-1" 2>&1 >/dev/null;then
				if [ $ppn_count -gt 32 ];then
					echo -e "\nMaximum ppn for the requested compute node $host_name is 32" >&2
					print_butinah_info
				fi
			fi
			if [ $count -gt 1 ];then
				if [ $ppn_count -gt $tmp_val ];then
					ppn_count=$ppn_count
				fi
			fi
			tmp_val=$ppn_count
		done
	fi
}

function get_node_count ()
{
	if echo $1 | egrep "nodes=0*[1-9][0-9]*(:ppn=0*[1-9][0-9]*)?(,|$)" | egrep -v ",$" 2>&1 >/dev/null;then
		node_count=`echo $1 | egrep -o 'nodes=0*[1-9][0-9]*' | cut -d"=" -f2`
	elif echo $1 | egrep "nodes=((compute-([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])(:ppn=0*[1-9][0-9]*)?\+?)+|(compute-17-[1-8](:ppn=0*[1-9][0-9]*)?\+?)+|compute-19-1(:ppn=0*[1-9][0-9]*)?|(gpu-18-([1-9]|1[0-6])(:ppn=0*[1-9][0-9]*)?\+?)+)($|,)" | egrep -v ",$|\+$|\+,.+$"  2>&1 >/dev/null;then
		node_count=`echo $1 | egrep -o "(compute-([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])|compute-17-[1-8]|compute-19-1|gpu-18-([1-9]|1[0-6]))+" | wc -l` 
		if [ $node_count -gt 1 ];then
			for((count=1;count<=$node_count;count++))
			do  
				host_name=`echo $1 | cut -f"$count" -d+ | egrep -o "(compute-(([1-9]|1[0-6])-([1-9]|[1-2][0-9]|3[0-2])|17-[1-8])|gpu-18-([1-9]|1[0-6]))"` 
				duplicate_count=`echo $1 | egrep -o $host_name | wc -l` 
				if [ $duplicate_count -gt 1 ];then
					echo -e "\nRequest cann't have duplicate hostnames." >&2
					print_run_info_quit
				fi  
			done
		fi
	else
		echo -e "\nIllegal -l argument." >&2
		print_run_info_quit
	fi
}

function check_file_path_in_oe_lines ()
{
        filepath=$(echo $1 | sed "s:\${\?HOME}\?:${HOME}:g" | sed "s:\${\?PBS_O_WORKDIR}\?:$PBS_O_WORKDIR:g")
        if [ ! -d $(echo "$filepath" | sed 's/^localhost://') ];then
                if echo $(echo "$filepath" | sed 's/^localhost://') | egrep ".+/$" 2>&1 >/dev/null;then
                        echo -e "\nDirectory $(echo "$filepath" | sed 's/^localhost://') in -o/-e directive(s) doesn't exist.\n" >&2
                        exit -1
                else
                        if [ ! -f $(echo "$filepath" | sed 's/^localhost://') ];then
                                if [ ! -d $(dirname $(echo "$filepath" | sed 's/^localhost://')) ];then
                                        echo -e "\nDirectory $(dirname $(echo "$filepath" | sed 's/^localhost://')) in -o/-e directive(s) doesn't exist.\n" >&2
                                        exit -1
                                fi
                        fi
                fi
        fi
}

function check_for_localhost_in_oe_lines ()
{
        if echo $1 | egrep "[$][{]?PBS_O_WORKDIR[}]?" 2>&1 >/dev/null;then
                if ! echo $1 | egrep "^localhost:[$][{]?PBS_O_WORKDIR[}]?" 2>&1 >/dev/null;then
                        echo -e '\n"localhost:" is missing in front of ${PBS_O_WORKDIR} in #PBS -o/-e statements\n' >&2
                        exit -1
                fi
        fi
}

function parse_pbs_oe_lines ()
{
        if [ "$pbsd" = "true" ];then
                PBS_O_WORKDIR="$darg"
        else
                PBS_O_WORKDIR="`pwd`"
        fi
        if [ "$pbso" = "true" ];then
                check_for_localhost_in_oe_lines "$oarg"
                check_file_path_in_oe_lines "$oarg"
        fi
        if [ "$pbse" = "true" ];then
                check_for_localhost_in_oe_lines "$earg"
                check_file_path_in_oe_lines "$earg"
        fi
}

function parse_pbs_d_line
{
	if [ "$pbsd" = "true" ];then
		if [ ! -d "$darg" ];then
			if [ -f $(echo "$darg" | sed 's/\/$//') ];then
				echo -e "\n$(echo "$darg") in #PBS -d directive is not a directory.\n" >&2
				exit -1
			fi
			echo -e "\nDirectory "$darg" in -d directive doesn't exist.\n" >&2
			exit -1
		else
			darg="$(dirname $darg)/$(basename $darg)"
		fi	
	fi
}

function hashbang ()
{
	if echo $1 | egrep "^#\! */" 2>&1 >/dev/null;then
		if echo $1  | sed 's/#!//' | egrep '^/bin/bash($|  *)' 2>&1 >/dev/null || echo $1  | sed 's/#!//' | egrep '^/bin/sh($|  *)' 2>&1 >/dev/null;then
			user_shell='bash'
		elif echo $1  | sed 's/#!//' | egrep '^/bin/tcsh($|  *)' 2>&1 >/dev/null || echo $1  | sed 's/#!//' | egrep '^/bin/csh($|  *)' 2>&1 >/dev/null;then
			user_shell='tcsh'
		fi
	fi
}

function parse_pbs_operatives ()
{
	if [ $line_count -eq 1 ];then
		hashbang "$i"
	fi

	if [ `echo $i | egrep "^$" 2>&1 >/dev/null` $? -ne 0 -a `echo $i | egrep "^ *#" | grep -v "^ *#PBS *" 2>&1 >/dev/null` $? -ne 0 -a `echo $i | egrep "^ *#PBS *" 2>&1 >/dev/null` $? -ne 0 ];then
		reading_pbs_operatives_is="done"
	else
		if echo $i | egrep "^ *#PBS *-q" 2>&1 >/dev/null;then
			if [ "$pbsq_from_cmdline" = "false" ];then
				qarg=$(echo $i | sed 's/ *#PBS *-q *//')
				queue="$qarg"
				pbsq="true"
			fi
		elif echo $i | egrep "^ *#PBS *-l" 2>&1 >/dev/null;then
			larg=$(echo $i | sed 's/ *#PBS *-l *//')
			if [ "$pbsln_from_cmdline" = "false" ];then
				if echo "$larg" | egrep "nodes=" >/dev/null 2>&1;then
					lnarg="$larg"
					pbsln="true"
				fi
			fi
			if [ "$pbslw_from_cmdline" = "false" ];then
				if echo "$larg" | egrep "walltime=" >/dev/null 2>&1;then
					lwarg="$larg"
					pbslw="true"
				fi
			fi
			if [ "$pbslm_from_cmdline" = "false" ];then
				if echo "$larg" | egrep "mem=" >/dev/null 2>&1;then
					lmarg="$larg"
					pbslm="true"
				fi
			fi
		elif echo $i | egrep "^ *#PBS *-t" 2>&1 >/dev/null;then
			if [ "$pbst_from_cmdline" = "false" ];then
				if echo $i | egrep "(0|[1-9][0-9]*)-[1-9][0-9]*:[1-9][0-9]*$" 2>&1 >/dev/null
				then
					args_to_t=`echo $i | egrep -o "(0|[1-9][0-9]*)-[1-9][0-9]*:[1-9][0-9]*$"`
					args_to_t=$(seq -s, $(echo $args_to_t | cut -f1 -d-) $(echo $args_to_t | cut -f2 -d:) $(echo $args_to_t | cut -f1 -d: | cut -f2 -d-))
					i="#PBS -t ${args_to_t}"
				fi
				pbst="true"
			fi
		elif echo $i | egrep "^ *#PBS *-N" 2>&1 >/dev/null;then
			if [ "$pbsN_from_cmdline" = "false" ];then
				Narg=$(echo $i | sed 's/ *#PBS *-N *//')
				pbsN="true"
			fi
		elif echo $i | egrep "^ *#PBS *-S" 2>&1 >/dev/null;then
			if [ "$pbsS_from_cmdline" = "false" ];then
				Sarg=$(echo $i | sed 's/ *#PBS *-S *//')
				pbsS="true"
			fi
		elif echo $i | egrep "^ *#PBS *-o" 2>&1 >/dev/null;then
			if [ "$pbso_from_cmdline" = "false" ];then
				oarg=$(echo $i | sed 's/ *#PBS *-o *//')
				pbso="true"
			fi
		elif echo $i | egrep "^ *#PBS *-e" 2>&1 >/dev/null;then
			if [ "$pbse_from_cmdline" = "false" ];then
				earg=$(echo $i | sed 's/ *#PBS *-e *//')
				pbse="true"
			fi
		elif echo $i | egrep "^ *#PBS *-d" 2>&1 >/dev/null;then
			if [ "$pbsd_from_cmdline" = "false" ];then
				darg=$(echo $i | sed 's/ *#PBS *-d *//')
				pbsd="true"
			fi
		fi
	fi
}

function read_pbs_script ()
{
	while IFS= read -r i
	do
		line_count=$(($line_count+1))

		if [ "$reading_pbs_operatives_is" = "not_done" ];then
			parse_pbs_operatives
			if [ "$reading_pbs_operatives_is" = "not_done" ];then
				echo "$i"
				continue
			fi
		fi

		if [ "$parsing_pbs_oed_lines_is" = "not_done" ];then
			parse_pbs_d_line				
			parse_pbs_oe_lines				
			parsing_pbs_oed_lines_is="done"
		fi

		if [ "$counting_resources_is" = "not_done" ];then
			if [ "$pbsln" = "true" ];then
				get_node_count "$lnarg"		
				get_ppn_count "$lnarg"
			fi
			if [ "$pbslw" = "true" ];then
				get_walltime "$lwarg"
			fi
			if [ "$pbslm" = "true" ];then
				get_memory "$lmarg"
			fi
			apply_defaults
			counting_resources_is="done"
		fi

		if [ "$node_features_are" = "not_added" ];then
                        echo
                        echo '###-----Wrapper Added-----###'
                        echo
			get_node_features
			echo
			echo '###-----Wrapper Done-----###'
			echo
			node_features_are="added"
		fi
		
		echo "$i"

	done
}

function read_commandline_arguments ()
{
for((arg=0;arg<$arg_count;arg++))
do
	if [ $arg -eq $(($arg_count-1)) ];then
		if [ -f "${args[$arg]}" ];then
			break
		fi
	fi

	if [ "${args[$arg]}" = "-q" -o `echo "${args[$arg]}" | egrep "^-q" >/dev/null 2>&1` $? -eq 0 -o `echo "${args[$arg]}" | egrep "^-Iq" >/dev/null 2>&1` $? -eq 0 ];then 
		pbsq_from_cmdline="true"
		pbsq="true"
		if [ "${args[$arg]}" = "-q" ];then
			qarg="${args[$(($arg+1))]}"
		elif echo "${args[$arg]}" | egrep "^-q" >/dev/null 2>&1;then
			qarg=`echo "${args[$arg]}" | sed 's/-q//'`
		else
			qarg=`echo "${args[$arg]}" | sed 's/-Iq//'`
		fi
		queue="$qarg"
	elif [ "${args[$arg]}" = "-N" -o `echo "${args[$arg]}" | egrep "^-N" >/dev/null 2>&1` $? -eq 0 -o `echo "${args[$arg]}" | egrep "^-IN" >/dev/null 2>&1` $? -eq 0 ];then 
		pbsN_from_cmdline="true"
		pbsN="true"
		if [ "${args[$arg]}" = "-N" ];then
			Narg="${args[$(($arg+1))]}"
		elif echo "${args[$arg]}" | egrep "^-N" >/dev/null 2>&1;then
			Narg=`echo "${args[$arg]}" | sed 's/-N//'`
		else
			Narg=`echo "${args[$arg]}" | sed 's/-IN//'`
		fi
	elif [ "${args[$arg]}" = "-S" -o `echo "${args[$arg]}" | egrep "^-S" >/dev/null 2>&1` $? -eq 0 -o `echo "${args[$arg]}" | egrep "^-IS" >/dev/null 2>&1` $? -eq 0 ];then 
		pbsS_from_cmdline="true"
		pbsS="true"
		if [ "${args[$arg]}" = "-S" ];then
			Sarg="${args[$(($arg+1))]}"
		elif echo "${args[$arg]}" | egrep "^-S" >/dev/null 2>&1;then
			Sarg=`echo "${args[$arg]}" | sed 's/-S//'`
		else
			Sarg=`echo "${args[$arg]}" | sed 's/-IS//'`
		fi
	elif [ "${args[$arg]}" = "-l" -o `echo "${args[$arg]}" | egrep "^-l" >/dev/null 2>&1` $? -eq 0 -o `echo "${args[$arg]}" | egrep "^-Il" >/dev/null 2>&1` $? -eq 0 ];then 
		if [ "${args[$arg]}" = "-l" ];then
			larg="${args[$(($arg+1))]}"
		elif echo "${args[$arg]}" | egrep "^-l" >/dev/null 2>&1;then
			larg=`echo "${args[$arg]}" | sed 's/-l//'`
		else
			larg=`echo "${args[$arg]}" | sed 's/-Il//'`
		fi
		if echo "$larg" | egrep "nodes=" >/dev/null 2>&1;then
			pbsln_from_cmdline="true"
			pbsln="true"
			lnarg="$larg"
		fi
		if echo "$larg" | egrep "walltime=" >/dev/null 2>&1;then
			pbslw_from_cmdline="true"
			pbslw="true"
			lwarg="$larg"
		fi
		if echo "$larg" | egrep "mem=" >/dev/null 2>&1;then
			pbslm_from_cmdline="true"
			pbslm="true"
			lmarg="$larg"
		fi
	elif echo "${args[$arg]}" | egrep "^-I" >/dev/null 2>&1;then
		pbsI="true"
	elif [ "${args[$arg]}" = "-t" -o `echo "${args[$arg]}" | egrep "^-t" >/dev/null 2>&1` $? -eq 0 ];then 
		pbst_from_cmdline="true"
		pbst="true"
	elif [ "${args[$arg]}" = "-o" -o `echo "${args[$arg]}" | egrep "^-o" >/dev/null 2>&1` $? -eq 0 ];then 
		pbso_from_cmdline="true"
		pbso="true"
		if [ "${args[$arg]}" = "-o" ];then
			oarg="${args[$(($arg+1))]}"
		else
			oarg=`echo "${args[$arg]}" | sed 's/-o//'`
		fi
	elif [ "${args[$arg]}" = "-e" -o `echo "${args[$arg]}" | egrep "^-e" >/dev/null 2>&1` $? -eq 0 ];then 
		pbse_from_cmdline="true"
		pbse="true"
		if [ "${args[$arg]}" = "-e" ];then
			earg="${args[$(($arg+1))]}"
		else
			earg=`echo "${args[$arg]}" | sed 's/-e//'`
		fi
	elif [ "${args[$arg]}" = "-d" -o `echo "${args[$arg]}" | egrep "^-d" >/dev/null 2>&1` $? -eq 0 ];then 
		pbsd_from_cmdline="true"
		pbsd="true"
		if [ "${args[$arg]}" = "-d" ];then
			darg="${args[$(($arg+1))]}"
		else
			darg=`echo "${args[$arg]}" | sed 's/-d//'`
		fi
	fi
done

if [ "$pbsI" = "true" ];then
	if [ "$pbsln" = "true" ];then
		get_node_count "$lnarg"		
		get_ppn_count "$lnarg"
	fi
	if [ "$pbslw" = "true" ];then
		get_walltime "$lwarg"
	fi
	if [ "$pbslm" = "true" ];then
		get_memory "$lmarg"
	fi
	apply_defaults
	get_node_features
else
	if [ ! -z "$pbs_script_name" ];then
		read_pbs_script
	fi 
fi
}

if [ $# -eq 0 ];then
    while IFS= read -r i
    do
        echo "$i"
    done    
else
	args=("$@")
	arg_count=$#
	if [ -f "${@:$#}" ];then
		pbs_script_name="${@:$#}"
		pbs_script_line_count=`wc -l < $pbs_script_name`
	fi
	read_commandline_arguments
fi