author | Fabien Ninoles <fabien@tzone.org> |
Mon, 07 Jun 2010 23:56:53 -0400 | |
changeset 6 | 6657247ddbbf |
parent 2 | a00ae018daf8 |
permissions | -rw-r--r-- |
2
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
1 |
#!/usr/bin/env python2.6 |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
2 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
3 |
import threading |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
4 |
import logging |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
5 |
import Queue |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
6 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
7 |
class _Worker(threading.Thread): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
8 |
def __init__(self, uid, pool): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
9 |
threading.Thread.__init__(self, name = "Worker%d" % uid) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
10 |
self.pool = pool |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
11 |
def run(self): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
12 |
logging.info("%s started", self.getName()) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
13 |
while True: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
14 |
job = self.pool.jobs.get() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
15 |
if job: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
16 |
logging.debug("%s is running %r", self.getName(), job) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
17 |
try: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
18 |
job() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
19 |
except: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
20 |
logging.exception("Job %r return an exception", job) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
21 |
logging.debug("Job %r completed.", job) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
22 |
else: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
23 |
self.pool.jobs.put(job) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
24 |
break |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
25 |
logging.info("%s stopped", self.getName()) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
26 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
27 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
28 |
## |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
29 |
# A basic worker pool... should be replaced with a resizable one. |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
30 |
class ThreadPool(object): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
31 |
Worker = _Worker |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
32 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
33 |
def __init__(self, size): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
34 |
super(ThreadPool, self).__init__() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
35 |
self.jobs = Queue.Queue() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
36 |
self._workers = [self.Worker(x+1, self) for x in xrange(size)] |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
37 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
38 |
def Start(self): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
39 |
logging.info("Starting Workers pool") |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
40 |
for worker in self._workers: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
41 |
worker.start() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
42 |
logging.info("Workers started") |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
43 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
44 |
def Stop(self, now = False): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
45 |
logging.info("Stopping Workers") |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
46 |
if now: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
47 |
self.jobs = Queue.Queue() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
48 |
self.jobs.put(None) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
49 |
for worker in self._workers: |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
50 |
worker.join() |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
51 |
logging.info("Workers stopped") |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
52 |
|
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
53 |
def AddJob(self, job): |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
54 |
self.jobs.put(job) |
a00ae018daf8
Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents:
diff
changeset
|
55 |