multiprocessing(2)---communication between subprocesses

---

Queue

Used among multiple processes.

Program

from multiprocessing import Process, Queue
import os, random, time


# define the write function
def proc_write(q,urls):
        # print the process information
    print('Process {} is writing.'.format(os.getpid()))
    for url in urls:
        # put the url into the queue
        q.put(url)
        print('Put {} into queue.'.format(url))
        # make the proces wait for random secs
        time.sleep(random.random())

# define the read function
def proc_read(q):
    print('Process {} is reaing.'.format(os.getpid()))
    while True:
        # get the urls from the queue
        url=q.get()
        print('Get {} from queue.'.format(url))


if __name__ == '__main__':
        # father process create a queue and pass it to subprocesses
        q=Queue()
        # create the write1 write2 read1 subprocesses
        proc_write1=Process(target=proc_write,args=(q,['url1','url2','url3']))
        proc_write2=Process(target=proc_write,args=(q,['url4','url5','url6']))
        proc_read1=Process(target=proc_read,args=(q,))
        # start the subprocesses
        proc_write1.start()
        proc_write2.start()
        proc_read1.start()
        # bolck the main process, join() can only called by started process
        proc_write1.join()
        proc_write2.join()
        # proc_read is a dead loop, so it can only be terminated
        proc_read1.terminate()

#  Annotation:
# 
# 
#  queue.put(blocked, timeout),when blocked is defaulted to True 
#  and timeout is positive, if the queue is not full, the method put the data into 
#  the queue, otherwise during the timeout the method will block to wait until 
#  there is an opening in the queue and put the data, if it pass the timeout,
#  and there is still no opening, the method will raise a Queue.Full exception.
#  When bolcked is defaulted to False, if the queue is full, the method will 
#  raise the exception immediately.
# 
# 
#  queue.get(blocked, timeout)
#  When blocked is defaulted to True, and timeout is positive,
#  the method will try to return a data of queue and delete it from the queue,
#  if pass the timeout, the method still get no data, it will raise the Queue.Empty
#  exception.
#  When blocked is set to False, the method will raise the exception immeditely
#  if there is no data to return.
# 
# 
# 
# 
# 

Output

Process 15884 is writing.
Process 15684 is writing.
Put url1 into queue.
Put url4 into queue.
Process 7528 is reaing.
Get url4 from queue.
Get url1 from queue.
Put url2 into queue.
Get url2 from queue.
Put url5 into queue.
Get url5 from queue.
Put url3 into queue.
Get url3 from queue.
Put url6 into queue.
Get url6 from queue.

Pipe

Used between two processes.

Program

# Annotation:
# 
# 
# Pipe(duplex=True) method of multiprocessing.context.DefaultContext instance
# Returns two connection objects(conn1,conn2)(tuple) connected by a pipe
# Has a optional para duplex defaulted to be True, which means the pipe can pass
# data both directions,that is conn1 and con2 can both send(send) and recieve(recv).
# If duplex is False, conn1 can only send and conn2 can only recieve
# If there is nothing to recieve, recv will block all the way until the pipe closed
# and raise a EOFErrorf


from multiprocessing import Pipe, Process
import time, os, random

def proc_send(conn,urls):
    for url in urls:
        print('Process {} sending: {}'.format(os.getpid(),url))
        conn.send(url)
        time.sleep(random.random())


def pro_recv(conn):
    while True:
        print('Process {} recving: {}'.format(os.getpid(),conn.recv()))
        time.sleep(random.random())


if __name__=="__main__":
    pipe=Pipe()
    p1=Process(target=proc_send,args=(pipe[0],['url_'+str(i) for i in range(10)]))
    p2=Process(target=pro_recv,args=(pipe[1],))
    p1.start()
    p2.start()
    p1.join()
    p2.terminate()

Output

Process 19352 sending: url_0
Process 1620 recving: url_0
Process 19352 sending: url_1
Process 1620 recving: url_1
Process 19352 sending: url_2
Process 1620 recving: url_2
Process 19352 sending: url_3
Process 1620 recving: url_3
Process 19352 sending: url_4
Process 1620 recving: url_4
Process 19352 sending: url_5
Process 1620 recving: url_5
Process 19352 sending: url_6
Process 1620 recving: url_6
Process 19352 sending: url_7
Process 19352 sending: url_8
Process 1620 recving: url_7
Process 19352 sending: url_9