mincepie

launcher module

launch_local() start server and client processes.

def launch_local():
    """ launches both the server and the clients on the local machine.

    The number of clients is defined in FLAGS.num_clients.
    """
    server, client = mince.Server(), mince.Client()
    serverprocess = Process(target = server.run_server, args = ())
    serverprocess.start()
    clientprocess = []
    for i in range(FLAGS.num_clients):
        clientprocess.append(
                Process(target = client.run_client, args = ()))
        clientprocess[i].start()
    serverprocess.join()
    logging.info("Waiting for processes to finish.")
    # some clients might still be running, and we will wait for them
    for i in range(FLAGS.num_clients): 
        clientprocess[i].join()
    return

server/client module

def run_server(self):
    # read data source
    self.datasource = mapreducer.READER(FLAGS.reader)().read(FLAGS.input)

    ...
    self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
    self.bind(("", FLAGS.port))
    self.listen(1)
    asyncore.loop()
    ...

    # write results
    mapreducer.WRITER(FLAGS.writer)().write(self.taskmanager.results)

mapreduce module

register object:

def _register(target_dict, object_to_register):
    name = object_to_register.__name__
    if name in target_dict:
        logging.fatal('Name ' + name + ' already registered.')
        sys.exit(1)
    target_dict[name] = object_to_register

register default object:

_DEFAULT_NAME = '_default'

def _register_default(target_dict, object_to_register):
    if not object_to_register.__name__ in target_dict:
        _register(target_dict, object_to_register)
    target_dict[_DEFAULT_NAME] = object_to_register

get registered object:

def _get_registered(source_dict, name):
try:
    if name is None or name == "":
        return source_dict[_DEFAULT_NAME]
    else:
        return source_dict[name]
except KeyError:
    logging.fatal("Can't find registered name " + name + " from:")
    logging.fatal(str(source_dict))
    sys.exit(1)

define internal dicts to store registered methods:

_MAPPERS  = {}
_REDUCERS = {}
_READERS  = {}
_WRITERS  = {}

define register macros:

REGISTER_MAPPER  = lambda x: _register(_MAPPERS, x)
REGISTER_REDUCER = lambda x: _register(_REDUCERS, x)
REGISTER_READER  = lambda x: _register(_READERS, x)
REGISTER_WRITER  = lambda x: _register(_WRITERS, x)
REGISTER_DEFAULT_MAPPER  = lambda x: _register_default(_MAPPERS,  x)
REGISTER_DEFAULT_REDUCER = lambda x: _register_default(_REDUCERS, x)
REGISTER_DEFAULT_READER  = lambda x: _register_default(_READERS,  x)
REGISTER_DEFAULT_WRITER  = lambda x: _register_default(_WRITERS,  x)
MAPPER  = lambda name: _get_registered(_MAPPERS,  name)
REDUCER = lambda name: _get_registered(_REDUCERS, name)
READER  = lambda name: _get_registered(_READERS,  name)
WRITER  = lambda name: _get_registered(_WRITERS,  name)

reader and writer interfaces:

class BasicReader(object):
    ...
    def read(self, input_string):
    """
    Input: a string passed by command line argument --input
    Output: a dictionary containing (key, value) pairs
    """
    inputlist = glob.glob(input_string)
    inputlist.sort()
    return dict(enumerate(inputlist))
    ...

REGISTER_DEFAULT_READER(BasicReader)

class BasicWriter(object):
    ...
    def write(self, result):
    """
    input: a dictionary containing (key, value) pairs.
    """
    for key in result:
        print repr(key), ':', repr(result[key])

REGISTER_DEFAULT_WRITER(BasicWriter)

mapper and reducer interfaces:

class BasicMapper(object):
    ...
    def map(self, key, value):
    """
    input: one key and one value.
    output: a list of (key, value) pairs, or a yield command that emits (key, value) pairs.
    """
    raise NotImplementedError

REGISTER_MAPPER(BasicMapper)

class BasicReducer(object):
    ...
    def reduce(self, key, values):
    """
    input: one key and one value
    output: a list of (key, value) pairs, or a yield command that emits (key, value) pairs.
    """
    raise NotImplementedError

REGISTER_REDUCER(BasicReducer)