source: jobq/jobq

Last change on this file was 560, checked in by zerodeux, 10 years ago

jobq 0.8.2 released

  • Property svn:executable set to *
File size: 5.3 KB
Line 
1#!/bin/sh
2
3# Simple job queue, run one process at once per queue and log timings and delays.
4#
5# A jobqueue is a simple list of files as $JOBPATH/<queue-name>/<job-id>
6# which content is a plain command to execute.
7
8# Copyright (C) 2009,2010 Bearstech - http://bearstech.com/
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program.  If not, see <http://www.gnu.org/licenses/>.
22
23PROGRAM=jobq
24VERSION=0.8.2
25
26# Hardcoded value for now, should fit most people
27JOBPATH=/tmp/jobq-$LOGNAME
28
29
30list_jobs() {
31  cd $JOBPATH/$queue 2>/dev/null || exit 0
32  echo "ID     Submission date      Command"
33  ls -1rt | \
34  while read job; do
35    jobt=$(stat -c%y $job 2>/dev/null | cut -d. -f1)
36    jobp=$(sed 's/\\\([^\]\)/\1/g' $job 2>/dev/null)
37    if [ -n "$jobp" ]; then
38      printf "%-5s %20s  %s\n" $job "$jobt" "$jobp"
39    fi
40  done 
41}
42
43add_job() {
44  set -e
45  # Make sure the queue path exists so we can write our job file
46  mkdir -m 0700 -p $JOBPATH/$queue
47  cd $JOBPATH/$queue
48
49  # Command line args re-quote - you don't want to know this shell ugliness.
50  argv=
51  for a in "$@"; do 
52    qa=$(printf %q "$a")
53    argv="$argv$qa "
54  done
55
56  # Hopefully this 'mv' is atomic, thus our joblist is lock-free
57  id=$$
58  echo "$argv" >.$id && mv .$id $id
59 
60  # Automatically start queue runner in backgorund. Make sure to restore
61  # caller's cwd so referring to $0 still works.
62  cd - >/dev/null && queue_runner_daemon
63}
64
65queue_runner() {
66  trap ":" USR1
67  echo "queue runner started for user/queue '$LOGNAME/$queue' (pid $$)" | logger -t $PROGRAM
68
69  cd $JOBPATH/$queue
70
71  while true; do
72    # Fetch job list in time order, older is first
73    jobl=`ls -1rt 2>/dev/null`
74    job=`echo "$jobl" | head -n1`
75
76    if [ -n "$job" ]; then
77      cmd=`cat $job`              # content of job file, the command to run
78      job=`basename "$job"`       # job ID
79      jobt=`stat -c%Y $job`       # job submission time
80      rm $job                     # remove the job, it _will_ be run in a few lines
81      todo=`echo "$jobl" |wc -l`  # new job queue size
82
83      # Run job in a subshell, capture stdin+stderr, send to syslog
84      export cmd jobt todo
85      (
86        t0=`date +%s`
87        late=$(( $t0 - $jobt ))
88        jobp=$(echo "$cmd" | sed 's/\\\([^\]\)/\1/g')
89        echo "start (delay:$late sec, todo:$todo): $jobp"
90
91        if [ -z "$MAILTO" ]; then
92          eval $cmd 2>&1
93        else
94          eval $cmd 2>&1 | mailx -e -s "`hostame -f` job[$LOGNAME/$queue/$job]: $jobp" $MAILTO
95        fi
96        ret=$?
97
98        dt=$(( `date +%s` - $t0 ))
99        echo "done (time:$dt sec, exit:$ret)"
100      ) | logger -t "job[$LOGNAME/$queue/$job]"
101    else
102      # No job, re-scan the queue later: wait 60sec or USR1
103      sleep 60 & wait $!
104      kill $!
105    fi
106  done
107}
108
109queue_runner_daemon() {
110  set +e
111  # Make sure the queue path exists so we can write our pidfile
112  mkdir -m 0700 -p $JOBPATH/$queue
113  pidfile=$JOBPATH/$queue/.runner
114
115  # - s-s-d does the pidfile mess and hopefully proper locking for us
116  # - --chdir PWD is needed to invoke $0 because s-s-d chdir to / as a default
117  /sbin/start-stop-daemon --quiet \
118    --start --startas "$0" \
119    --chdir "$PWD" \
120    --background \
121    --make-pidfile --pidfile $pidfile \
122    -- --runner --queue "$queue"
123  if [ $? = 1 ]; then
124    # Daemon is already running, wake it up (send USR1) to process new job immediatly
125    /sbin/start-stop-daemon --quiet --stop --signal USR1 --pidfile $pidfile
126  fi
127  if [ $? != 0 ]; then
128    echo "Error while trying to start queue runner for user/queue '$LOGNAME/$queue'" >&2
129  fi
130}
131
132help() {
133  cat << EOF
134List queue: $PROGRAM [-q <queue>]
135Submit job: $PROGRAM [-q <queue>] command args ...
136
137The default queue is called, er, 'default'. One 'queue runner' daemon is
138needed per queue and per user, and is started automatically upon first job
139submission. There is no mechanism to stop a queue runner besides killing it.
140
141If MAILTO is set, job output (if any) is sent to this address, otherwise it
142is syslogged to the user.notice facility. This is a queue runner setting,
143either parsed by explicit queue runner start, either while running the first
144job for a given user/queue.
145
146Options:
147  -h, --help         Show this help
148  -q, --queue QUEUE  Select queue (default name is 'default')
149  -r, --runner       Explicitly start queue runner (in foreground)
150  -v, --version      Show this software revision
151EOF
152  exit 1
153}
154
155version() {
156  echo "$PROGRAM $VERSION"
157  exit 0
158}
159
160queue=default
161runner=
162
163parse_opt=run
164while [ $parse_opt != "done"  ] ; do
165  case "$1" in
166    -q|--queue)   shift; queue="$1"; shift;;
167    -r|--runner)  shift; runner=yes;;
168    -h|--help)    help;;
169    -v|--version) version;;
170
171    --) parse_opt=done;;
172    -*) echo "$PROGRAM: unknown option $1, try -h for help" >&2; exit 2;;
173     *) parse_opt=done;;
174  esac
175done
176
177if [ -n "$runner" ]; then
178  queue_runner
179  exit 0
180fi
181if [ $# -gt 0 ]; then
182  add_job "$@"
183else
184  list_jobs
185fi
Note: See TracBrowser for help on using the repository browser.