[root@cluster install]# qrun 57
[ublcr@cluster install]# qhold 57
qhold: something specified didn't exist MSG=MOM rejected hold request: 15204 57.cluster.pac
[root@cluster install]# qtracejob.pbs-server 57
/opt/torque/mom_logs/20110311: No such file or directory
/opt/torque/sched_logs/20110311: No such file or directory
/opt/torque/server_priv/accounting/20110311: No such file or directory
/opt/torque/server_logs/20110311: No such file or directory
/opt/torque/sched_logs/20110311: No such file or directory
cluster.PAC 03/11/2011 15:03:34 A queue=default
cluster.PAC 03/11/2011 15:03:34 S enqueuing into default, state 1 hop 1
cluster.PAC 03/11/2011 15:03:34 S Job Queued at request of ublcr@cluster.PAC, owner = ublcr@cluster.PAC, job name = test.sh, queue = default
cluster.PAC 03/11/2011 15:03:36 S Unauthorized Request, request type: 11, Object: Job, Name: 57.cluster.pac, request from: maui@cluster.PAC
cluster.PAC 03/11/2011 15:03:43 A user=ublcr group=ublcr jobname=test.sh queue=default ctime=1299855814 qtime=1299855814 etime=1299855814 start=1299855823 owner=ublcr@cluster.PAC exec_host=compute-0-0/0 Resource_List.neednodes=compute-0-0 Resource_List.nodect=1 Resource_List.nodes=compute-0-0 Resource_List.walltime=00:10:00 
cluster.PAC 03/11/2011 15:03:43 S Job Run at request of root@cluster.PAC
cluster.PAC 03/11/2011 15:03:43 S Not sending email: User does not want mail of this type.
compute-0-0 03/11/2011 15:03:43 M ready to commit job
compute-0-0 03/11/2011 15:03:43 M ready to commit job completed
compute-0-0 03/11/2011 15:03:43 M committing job
compute-0-0 03/11/2011 15:03:43 M req_commit:starting job execution
compute-0-0 03/11/2011 15:03:43 M evaluating limits for job
compute-0-0 03/11/2011 15:03:43 M about to fork child which will become job
compute-0-0 03/11/2011 15:03:43 M phase 2 of job launch successfully completed
compute-0-0 03/11/2011 15:03:43 M task/session info loaded
compute-0-0 03/11/2011 15:03:43 M saving task (TMomFinalizeJob3)
compute-0-0 03/11/2011 15:03:43 M exec_job_on_ms:job successfully started
compute-0-0 03/11/2011 15:03:43 M start_exec:job 57.cluster.pac reported successful start on 1 node(s)
compute-0-0 03/11/2011 15:03:43 M req_commit:job execution started
compute-0-0 03/11/2011 15:03:43 M encoding "send flagged" attr: session_id
cluster.PAC 03/11/2011 15:03:49 S Holds u set at request of root@cluster.PAC
cluster.PAC 03/11/2011 15:03:49 S MOM rejected hold request: 15204
compute-0-0 03/11/2011 15:03:49 M checking job w/subtask pid=4513 (child pid=4513)
compute-0-0 03/11/2011 15:03:49 M found match with job subtask for pid=4513
cluster.PAC 03/11/2011 15:05:24 A user=ublcr group=ublcr jobname=test.sh queue=default ctime=1299855814 qtime=1299855814 etime=1299855814 start=1299855823 owner=ublcr@cluster.PAC exec_host=compute-0-0/0 Resource_List.neednodes=compute-0-0 Resource_List.nodect=1 Resource_List.nodes=compute-0-0 Resource_List.walltime=00:10:00 session=4484 end=1299855924 Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=2656kb resources_used.vmem=139620kb resources_used.walltime=00:01:41
cluster.PAC 03/11/2011 15:05:24 S Not sending email: User does not want mail of this type.
cluster.PAC 03/11/2011 15:05:24 S Exit_status=0 resources_used.cput=00:00:00 resources_used.mem=2656kb resources_used.vmem=139620kb resources_used.walltime=00:01:41
cluster.PAC 03/11/2011 15:05:24 S dequeuing from default, state COMPLETE
compute-0-0 03/11/2011 15:05:24 M checking job w/subtask pid=0 (child pid=4484)
compute-0-0 03/11/2011 15:05:24 M found match with job task 0 for pid=4484
compute-0-0 03/11/2011 15:05:24 M kill_task: sending signal 9 to task 1, session 4484
compute-0-0 03/11/2011 15:05:24 M kill_task: could not send signal 9 to task 1 (session 4484)--no process was found with this session ID (marking task as killed)!
compute-0-0 03/11/2011 15:05:24 M scan_for_terminated: job 57.cluster.pac task 1 terminated, sid=4484
compute-0-0 03/11/2011 15:05:24 M job was terminated
compute-0-0 03/11/2011 15:05:24 M no sisters contacted - setting job substate to EXITING
compute-0-0 03/11/2011 15:05:24 M task is dead
compute-0-0 03/11/2011 15:05:24 M kill_job done (killed 0 processes)
compute-0-0 03/11/2011 15:05:24 M sending preobit jobstat
compute-0-0 03/11/2011 15:05:24 M performing job clean-up in preobit_reply()
compute-0-0 03/11/2011 15:05:24 M epilog subtask created with pid 4551 - substate set to JOB_SUBSTATE_OBIT - registered post_epilogue
compute-0-0 03/11/2011 15:05:24 M checking job w/subtask pid=4551 (child pid=4551)
compute-0-0 03/11/2011 15:05:24 M found match with job subtask for pid=4551
compute-0-0 03/11/2011 15:05:24 M obit sent to server
compute-0-0 03/11/2011 15:05:24 M job obit acknowledge received - substate set to JOB_SUBSTATE_EXITED
compute-0-0 03/11/2011 15:05:24 M attempting to copy file 'cluster.pac:/home/ublcr/test.sh.o57'
compute-0-0 03/11/2011 15:05:24 M forking to user, uid: 501  gid: 501  homedir: '/home/ublcr'
compute-0-0 03/11/2011 15:05:24 M deleting job
compute-0-0 03/11/2011 15:05:24 M deleting job 57.cluster.pac in state EXITED
compute-0-0 03/11/2011 15:05:24 M removing job
compute-0-0 03/11/2011 15:05:24 M removed job script
compute-0-0 03/11/2011 15:05:24 M removed job file