scheduler.py
author Fabien Ninoles <fabien@tzone.org>
Sun, 21 Mar 2010 20:56:44 -0400
changeset 3 00b6708d1852
parent 2 a00ae018daf8
child 4 76ba9b3a9e1c
permissions -rwxr-xr-x
Rewrite the whole to use linked chain of callback, allowing the reactor to be specified at the very end.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     1
#!/usr/bin/env python2.6
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     2
import sys
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     3
import logging
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     4
import threading
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     5
import Queue
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents: 1
diff changeset
     6
from threadpool import ThreadPool
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     7
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     8
##
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
     9
# A task, representing a series of linked pairs of callback and error
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    10
# handler Concept is similar to twisted, but you need to put all your
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    11
# callback on the task before giving it to the Scheduler.  For this
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    12
# reason, you shouldnt call the callback/errorback method yourself (or
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    13
# at least, don't put it back in the scheduler after that).
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    14
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    15
class _Callback(object):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    16
    def __init__(self, callback, errorback, threaded = False):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    17
        self.callback = callback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    18
        self.errorback = errorback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    19
        self.next = None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    20
        self.threaded = threaded
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    21
    def Chain(self, next):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    22
        # Can only be called once
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    23
        assert(self.next is None)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    24
        self.next = next
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    25
        return next
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    26
    def Next(self):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    27
        return self.next
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    28
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    29
class Task(object):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    30
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    31
    @staticmethod
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    32
    def DefaultCallback(result):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    33
        return result
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    34
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    35
    @staticmethod
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    36
    def DefaultErrorback(error):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    37
        return error
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    38
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    39
    def __init__(self, func = None, *args, **kwargs):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    40
        super(Task, self).__init__()
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    41
        self.head = None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    42
        self.tail = None
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    43
        if func:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    44
            def callback(result):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    45
                return func(*args, **kwargs)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    46
            self.AddCallback(callback)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    47
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    48
    def _AddCallback(self, callback):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    49
        if self.head is None:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    50
            self.head = self.tail = callback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    51
        else:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    52
            self.tail = self.tail.Chain(callback)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    53
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    54
    def _GetNext(self):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    55
        head = self.head
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    56
        if head:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    57
            self.head = head.Next()
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    58
        return head
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    59
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    60
    def AddCallback(self, callback, errorback = None, threaded = False):
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    61
        if errorback == None:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    62
            errorback = self.DefaultErrorback
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    63
        cb = _Callback(callback, errorback, threaded)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    64
        self._AddCallback(cb)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    65
        # permit chained calls
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    66
        return self
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    67
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    68
    def AddThreadedCallback(self, callback, errorback = None):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    69
        return self.AddCallback(callback, errorback, True)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    70
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    71
    def ChainTask(self, task):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    72
        self.tail.Chain(task.head)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    73
        self.tail = task.tail
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    74
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    75
class ThreadedTask(Task):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    76
    def __init__(self, func, *args, **kwargs):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    77
        super(ThreadedTask, self).__init__(func, *args, **kwargs)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    78
        self.head.threaded = True
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    79
    
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    80
class Scheduler(threading.Thread):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    81
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    82
    class _SchedulerStop(Exception):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    83
        pass
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    84
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    85
    def __init__(self, poolSize):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    86
        threading.Thread.__init__(self, name = "Scheduler", target = self.Run)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    87
        self.pool = ThreadPool(poolSize)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    88
        self.tasks = Queue.Queue()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    89
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    90
    def ExecuteOne(self, blocking = True):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    91
        logging.debug("Looking for next task...")
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    92
        try:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    93
            task = self.tasks.get(blocking)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    94
        except Queue.Empty:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
    95
            logging.debug("No task to run")
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    96
            return None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    97
        result = None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    98
        error = None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
    99
        traceback = None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   100
        while True:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   101
            cb = task._GetNext()
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   102
            if not cb:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   103
                # no more callback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   104
                break
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   105
            if cb.threaded:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   106
                # a threaded callback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   107
                self._AddJob(task, cb, result, error, traceback)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   108
                # don't pass Go, don't reclaim $200
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   109
                return None
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   110
            # Run the callback according to the current state
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   111
            try:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   112
                if error:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   113
                    error = cb.errorback(error)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   114
                else:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   115
                    result = cb.callback(result)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   116
            except:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   117
                errtype, error, traceback = sys.exc_info()
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   118
        if error:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   119
            raise error, None, traceback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   120
        else:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   121
            return result
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   122
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   123
    def Run(self):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   124
        logging.info("Scheduler start")
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   125
        while True:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   126
            try:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   127
                self.ExecuteOne()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   128
            except self._SchedulerStop:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   129
                break
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   130
            except:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   131
                logging.exception("Unhandled task exception")
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   132
        logging.info("Scheduler stop")
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   133
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   134
    def Start(self):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   135
        self.pool.Start()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   136
        return self.start()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   137
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   138
    def Stop(self, now = False):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   139
        self.pool.Stop(now)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   140
        if now:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   141
            self.tasks = Queue.Queue()
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   142
        # We raise an exception to find if we stop stop the scheduler.
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   143
        # We could have use a None task, but this make it easier if we
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   144
        # want to add such mechanism public or we want to stop on
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   145
        # other exception
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   146
        def RaiseSchedulerStop():
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   147
            raise self._SchedulerStop
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   148
        self.AddTask(Task(RaiseSchedulerStop))
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   149
        self.join()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   150
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   151
    def AddTask(self, task, blocking = True):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   152
        self.tasks.put(task, blocking)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   153
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   154
    def _AddJob(self, task, cb, result, error, traceback):
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   155
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   156
        def DoIt(task, cb, result, error, traceback):
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   157
            try:
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   158
                if error:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   159
                    error = cb.errorback(error)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   160
                else:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   161
                    result = cb.callback(result)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   162
            except:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   163
                errtype, error, traceback = sys.exc_info()
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   164
            if error:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   165
                def RaiseError():
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   166
                    raise error, None, traceback
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   167
                jobTask = Task(RaiseError)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   168
            else:
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   169
                def ReturnResult():
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   170
                    return result
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   171
                jobTask = Task(ReturnResult)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   172
            jobTask.ChainTask(task)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   173
            self.AddTask(jobTask)
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   174
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   175
        # This double wrap (Job over DoIt) seems necessary to make
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   176
        # error not look like a local of Job...
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   177
        def Job():
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   178
            return DoIt(task, cb, result, error, traceback)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   179
        self.pool.AddJob(Job)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   180
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   181
# The global scheduler
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   182
scheduler = None
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   183
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   184
def StartScheduler(size):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   185
    global scheduler
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   186
    if scheduler:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   187
        StopScheduler()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   188
    scheduler = Scheduler(size)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   189
    scheduler.Start()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   190
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   191
def StopScheduler(now = False):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   192
    global scheduler
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   193
    if scheduler:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   194
        scheduler.Stop(now)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   195
    scheduler = None
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   196
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   197
if __name__ == '__main__':
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   198
    from time import sleep
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents: 1
diff changeset
   199
    logging.getLogger().setLevel(logging.INFO)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   200
    # This function is a sample and shouldn't know about the scheduler
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   201
    count = 0
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents: 1
diff changeset
   202
    def AsyncCall(name, seconds):
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   203
        global count
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   204
        count += 1
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   205
        
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   206
        # Probably a bad example, since the callback
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   207
        # doesn't return the exact same type...
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents: 1
diff changeset
   208
        def Initialize(name, seconds):
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   209
            print "Here", name
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents: 1
diff changeset
   210
            return name, seconds
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   211
        def Blocking(args):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   212
            name, time = args
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   213
            print name, "goes to bed"
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   214
            sleep(time)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   215
            print name, ": ZZZ..."
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   216
            return name
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   217
        def Finalize(name):
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   218
            global count
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   219
            print name, "wakes up!"
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   220
            count -= 1
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   221
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   222
        task = Task(Initialize, name, seconds)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   223
        task.AddThreadedCallback(Blocking)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   224
        task.AddCallback(Finalize)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   225
        return task
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   226
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   227
    logging.info("Starting scheduler with 10 workers")
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   228
    StartScheduler(10)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   229
    logging.info("Adding asynccall task")
2
a00ae018daf8 Separate thread pool in another file and fix some issues.
Fabien Ninoles <fabien@tzone.org>
parents: 1
diff changeset
   230
    for x in xrange(int(sys.argv[1])):
3
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   231
        task = AsyncCall("Toto%d" % (x+1), (x % 10)/10.0)
00b6708d1852 Rewrite the whole to use linked chain of callback, allowing the
Fabien Ninoles <fabien@tzone.org>
parents: 2
diff changeset
   232
        scheduler.AddTask(task)
0
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   233
    while count > 0:
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   234
        logging.debug("Count = %d", count)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   235
        sleep(1)
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   236
    logging.info("Stopping scheduler")
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   237
    StopScheduler()
8ae7370093db First version.
Fabien Ninoles <fabien@tzone.org>
parents:
diff changeset
   238
    logging.info("The End.")