threadpool.py
author Fabien Ninoles <fabien@tzone.org>
Sun, 21 Mar 2010 21:47:48 -0400
changeset 5 eb1133af54ed
parent 2 a00ae018daf8
permissions -rw-r--r--
Seperate task and scheduler.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     1
#!/usr/bin/env python2.6
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     3
import threading
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     4
import logging
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     5
import Queue
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     6
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     7
class _Worker(threading.Thread):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     8
    def __init__(self, uid, pool):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     9
        threading.Thread.__init__(self, name = "Worker%d" % uid)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    10
        self.pool = pool
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    11
    def run(self):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    12
        logging.info("%s started", self.getName())
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    13
        while True:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    14
            job = self.pool.jobs.get()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    15
            if job:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    16
                logging.debug("%s is running %r", self.getName(), job)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    17
                try:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    18
                    job()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    19
                except:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    20
                    logging.exception("Job %r return an exception", job)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    21
                logging.debug("Job %r completed.", job)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    22
            else:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    23
                self.pool.jobs.put(job)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    24
                break
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    25
        logging.info("%s stopped", self.getName())
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    26
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    27
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    28
##
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    29
# A basic worker pool... should be replaced with a resizable one.
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    30
class ThreadPool(object):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    31
    Worker = _Worker
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    32
    
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    33
    def __init__(self, size):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    34
        super(ThreadPool, self).__init__()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    35
        self.jobs = Queue.Queue()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    36
        self._workers = [self.Worker(x+1, self) for x in xrange(size)]
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    37
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    38
    def Start(self):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    39
        logging.info("Starting Workers pool")
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    40
        for worker in self._workers:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    41
            worker.start()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    42
        logging.info("Workers started")
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    43
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    44
    def Stop(self, now = False):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    45
        logging.info("Stopping Workers")
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    46
        if now:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    47
            self.jobs = Queue.Queue()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    48
        self.jobs.put(None)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    49
        for worker in self._workers:
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    50
            worker.join()
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    51
        logging.info("Workers stopped")
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    52
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    53
    def AddJob(self, job):
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    54
        self.jobs.put(job)
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    55