multiprocessing研究

最新推荐文章于 2022-10-10 21:08:27 发布

转载最新推荐文章于 2022-10-10 21:08:27 发布 · 667 阅读

本文介绍Python的multiprocessing模块，展示如何创建并行进程进行任务处理。包括基本进程创建、带参数的进程启动、进程命名及使用守护进程等高级特性。

前言：multiprocessing是一个使用方法类似threading模块的进程模块，允许程序员做并行开发，从python2.6开始内置

import multiprocessing
import logging,sys
def worker():
    print 'Worker'
    sys.stdout.flush() 
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        multiprocessing.log_to_stderr(logging.DEBUG)  #记录日志,也可以使用一下注释的三句
        #multiprocessing.log_to_stderr() 
        #logger = multiprocessing.get_logger() 
        #logger.setLevel(logging.INFO)
        p = multiprocessing.Process(target=worker)  #实例化Process
        jobs.append(p)
        p.start() #调用start()开始，类似于threading.Thread

import multiprocessing

def worker(num): #带参数的函数
    print 'Worker:', num
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,)) #带参数的目标
        jobs.append(p)
        p.start()

import multiprocessing
import time

def worker():
    name = multiprocessing.current_process().name  #当前进程名字
    print name, 'Starting'
    time.sleep(2)
    print name, 'Exiting'

def my_service():
    name = multiprocessing.current_process().name
    print name, 'Starting'
    time.sleep(3)
    print name, 'Exiting'

if __name__ == '__main__':
    service = multiprocessing.Process(name='my_service', target=my_service) #使用自定义的函数名
    worker_1 = multiprocessing.Process(name='worker 1', target=worker)
    worker_2 = multiprocessing.Process(target=worker) # use default name  #使用默认的函数名，即Process-3

    worker_1.start()
    worker_2.start()
    service.start()

import multiprocessing
import time
import sys

def daemon():
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    time.sleep(2)
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

def non_daemon():  #默认情况下，主程序将不会退出，直到所有的孩子都已经退出
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False   #默认是不是守护进程，以便通过真正轮流守护模式

    d.start()
    time.sleep(1)
    n.start()

import multiprocessing
import time
import sys

def daemon():
    print 'Starting:', multiprocessing.current_process().name
    time.sleep(2)
    print 'Exiting :', multiprocessing.current_process().name

def non_daemon():
    print 'Starting:', multiprocessing.current_process().name
    print 'Exiting :', multiprocessing.current_process().name

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False

    d.start()
    time.sleep(1)
    n.start()

    d.join() #会使主调线程堵塞，直到被调用线程运行结束或超时,join函数可以设置超时， 如果进程在超时期限内没有完成就返回 比如join(1)，可以通过d.is_alive()查看进程是否存活
    n.join()

import multiprocessing
import time

def slow_worker():
    print 'Starting worker'
    time.sleep(0.1)
    print 'Finished worker'

if __name__ == '__main__':
    p = multiprocessing.Process(target=slow_worker)
    print 'BEFORE:', p, p.is_alive()

    p.start()
    print 'DURING:', p, p.is_alive()

    p.terminate()  #发送结束进程信号
    print 'TERMINATED:', p, p.is_alive()

    p.join()  #更新状态以反应终止后的状态
    print 'JOINED:', p, p.is_alive()

前言：multiprocessing是一个使用方法类似threading模块的进程模块，允许程序员做并行开发，从python2.6开始内置

import multiprocessing
import logging,sys
def worker():
    print 'Worker'
    sys.stdout.flush() 
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        multiprocessing.log_to_stderr(logging.DEBUG)  #记录日志,也可以使用一下注释的三句
        #multiprocessing.log_to_stderr() 
        #logger = multiprocessing.get_logger() 
        #logger.setLevel(logging.INFO)
        p = multiprocessing.Process(target=worker)  #实例化Process
        jobs.append(p)
        p.start() #调用start()开始，类似于threading.Thread

import multiprocessing

def worker(num): #带参数的函数
    print 'Worker:', num
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,)) #带参数的目标
        jobs.append(p)
        p.start()

import multiprocessing
import time

def worker():
    name = multiprocessing.current_process().name  #当前进程名字
    print name, 'Starting'
    time.sleep(2)
    print name, 'Exiting'

def my_service():
    name = multiprocessing.current_process().name
    print name, 'Starting'
    time.sleep(3)
    print name, 'Exiting'

if __name__ == '__main__':
    service = multiprocessing.Process(name='my_service', target=my_service) #使用自定义的函数名
    worker_1 = multiprocessing.Process(name='worker 1', target=worker)
    worker_2 = multiprocessing.Process(target=worker) # use default name  #使用默认的函数名，即Process-3

    worker_1.start()
    worker_2.start()
    service.start()

import multiprocessing
import time
import sys

def daemon():
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    time.sleep(2)
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

def non_daemon():  #默认情况下，主程序将不会退出，直到所有的孩子都已经退出
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False   #默认是不是守护进程，以便通过真正轮流守护模式

    d.start()
    time.sleep(1)
    n.start()

import multiprocessing
import time
import sys

def daemon():
    print 'Starting:', multiprocessing.current_process().name
    time.sleep(2)
    print 'Exiting :', multiprocessing.current_process().name

def non_daemon():
    print 'Starting:', multiprocessing.current_process().name
    print 'Exiting :', multiprocessing.current_process().name

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False

    d.start()
    time.sleep(1)
    n.start()

    d.join() #会使主调线程堵塞，直到被调用线程运行结束或超时,join函数可以设置超时， 如果进程在超时期限内没有完成就返回 比如join(1)，可以通过d.is_alive()查看进程是否存活
    n.join()

import multiprocessing
import time

def slow_worker():
    print 'Starting worker'
    time.sleep(0.1)
    print 'Finished worker'

if __name__ == '__main__':
    p = multiprocessing.Process(target=slow_worker)
    print 'BEFORE:', p, p.is_alive()

    p.start()
    print 'DURING:', p, p.is_alive()

    p.terminate()  #发送结束进程信号
    print 'TERMINATED:', p, p.is_alive()

    p.join()  #更新状态以反应终止后的状态
    print 'JOINED:', p, p.is_alive()

前言：multiprocessing是一个使用方法类似threading模块的进程模块，允许程序员做并行开发，从python2.6开始内置

import multiprocessing
import logging,sys
def worker():
    print 'Worker'
    sys.stdout.flush() 
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        multiprocessing.log_to_stderr(logging.DEBUG)  #记录日志,也可以使用一下注释的三句
        #multiprocessing.log_to_stderr() 
        #logger = multiprocessing.get_logger() 
        #logger.setLevel(logging.INFO)
        p = multiprocessing.Process(target=worker)  #实例化Process
        jobs.append(p)
        p.start() #调用start()开始，类似于threading.Thread

import multiprocessing

def worker(num): #带参数的函数
    print 'Worker:', num
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,)) #带参数的目标
        jobs.append(p)
        p.start()

import multiprocessing
import time

def worker():
    name = multiprocessing.current_process().name  #当前进程名字
    print name, 'Starting'
    time.sleep(2)
    print name, 'Exiting'

def my_service():
    name = multiprocessing.current_process().name
    print name, 'Starting'
    time.sleep(3)
    print name, 'Exiting'

if __name__ == '__main__':
    service = multiprocessing.Process(name='my_service', target=my_service) #使用自定义的函数名
    worker_1 = multiprocessing.Process(name='worker 1', target=worker)
    worker_2 = multiprocessing.Process(target=worker) # use default name  #使用默认的函数名，即Process-3

    worker_1.start()
    worker_2.start()
    service.start()

import multiprocessing
import time
import sys

def daemon():
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    time.sleep(2)
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

def non_daemon():  #默认情况下，主程序将不会退出，直到所有的孩子都已经退出
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False   #默认是不是守护进程，以便通过真正轮流守护模式

    d.start()
    time.sleep(1)
    n.start()

import multiprocessing
import time
import sys

def daemon():
    print 'Starting:', multiprocessing.current_process().name
    time.sleep(2)
    print 'Exiting :', multiprocessing.current_process().name

def non_daemon():
    print 'Starting:', multiprocessing.current_process().name
    print 'Exiting :', multiprocessing.current_process().name

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False

    d.start()
    time.sleep(1)
    n.start()

    d.join() #会使主调线程堵塞，直到被调用线程运行结束或超时,join函数可以设置超时， 如果进程在超时期限内没有完成就返回 比如join(1)，可以通过d.is_alive()查看进程是否存活
    n.join()

import multiprocessing
import time

def slow_worker():
    print 'Starting worker'
    time.sleep(0.1)
    print 'Finished worker'

if __name__ == '__main__':
    p = multiprocessing.Process(target=slow_worker)
    print 'BEFORE:', p, p.is_alive()

    p.start()
    print 'DURING:', p, p.is_alive()

    p.terminate()  #发送结束进程信号
    print 'TERMINATED:', p, p.is_alive()

    p.join()  #更新状态以反应终止后的状态
    print 'JOINED:', p, p.is_alive()

前言：multiprocessing是一个使用方法类似threading模块的进程模块，允许程序员做并行开发，从python2.6开始内置

import multiprocessing
import logging,sys
def worker():
    print 'Worker'
    sys.stdout.flush() 
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        multiprocessing.log_to_stderr(logging.DEBUG)  #记录日志,也可以使用一下注释的三句
        #multiprocessing.log_to_stderr() 
        #logger = multiprocessing.get_logger() 
        #logger.setLevel(logging.INFO)
        p = multiprocessing.Process(target=worker)  #实例化Process
        jobs.append(p)
        p.start() #调用start()开始，类似于threading.Thread

import multiprocessing

def worker(num): #带参数的函数
    print 'Worker:', num
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,)) #带参数的目标
        jobs.append(p)
        p.start()

import multiprocessing
import time

def worker():
    name = multiprocessing.current_process().name  #当前进程名字
    print name, 'Starting'
    time.sleep(2)
    print name, 'Exiting'

def my_service():
    name = multiprocessing.current_process().name
    print name, 'Starting'
    time.sleep(3)
    print name, 'Exiting'

if __name__ == '__main__':
    service = multiprocessing.Process(name='my_service', target=my_service) #使用自定义的函数名
    worker_1 = multiprocessing.Process(name='worker 1', target=worker)
    worker_2 = multiprocessing.Process(target=worker) # use default name  #使用默认的函数名，即Process-3

    worker_1.start()
    worker_2.start()
    service.start()

import multiprocessing
import time
import sys

def daemon():
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    time.sleep(2)
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

def non_daemon():  #默认情况下，主程序将不会退出，直到所有的孩子都已经退出
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False   #默认是不是守护进程，以便通过真正轮流守护模式

    d.start()
    time.sleep(1)
    n.start()

import multiprocessing
import time
import sys

def daemon():
    print 'Starting:', multiprocessing.current_process().name
    time.sleep(2)
    print 'Exiting :', multiprocessing.current_process().name

def non_daemon():
    print 'Starting:', multiprocessing.current_process().name
    print 'Exiting :', multiprocessing.current_process().name

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False

    d.start()
    time.sleep(1)
    n.start()

    d.join() #会使主调线程堵塞，直到被调用线程运行结束或超时,join函数可以设置超时， 如果进程在超时期限内没有完成就返回 比如join(1)，可以通过d.is_alive()查看进程是否存活
    n.join()

import multiprocessing
import time

def slow_worker():
    print 'Starting worker'
    time.sleep(0.1)
    print 'Finished worker'

if __name__ == '__main__':
    p = multiprocessing.Process(target=slow_worker)
    print 'BEFORE:', p, p.is_alive()

    p.start()
    print 'DURING:', p, p.is_alive()

    p.terminate()  #发送结束进程信号
    print 'TERMINATED:', p, p.is_alive()

    p.join()  #更新状态以反应终止后的状态
    print 'JOINED:', p, p.is_alive()

前言：multiprocessing是一个使用方法类似threading模块的进程模块，允许程序员做并行开发，从python2.6开始内置

import multiprocessing
import logging,sys
def worker():
    print 'Worker'
    sys.stdout.flush() 
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        multiprocessing.log_to_stderr(logging.DEBUG)  #记录日志,也可以使用一下注释的三句
        #multiprocessing.log_to_stderr() 
        #logger = multiprocessing.get_logger() 
        #logger.setLevel(logging.INFO)
        p = multiprocessing.Process(target=worker)  #实例化Process
        jobs.append(p)
        p.start() #调用start()开始，类似于threading.Thread

import multiprocessing

def worker(num): #带参数的函数
    print 'Worker:', num
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,)) #带参数的目标
        jobs.append(p)
        p.start()

import multiprocessing
import time

def worker():
    name = multiprocessing.current_process().name  #当前进程名字
    print name, 'Starting'
    time.sleep(2)
    print name, 'Exiting'

def my_service():
    name = multiprocessing.current_process().name
    print name, 'Starting'
    time.sleep(3)
    print name, 'Exiting'

if __name__ == '__main__':
    service = multiprocessing.Process(name='my_service', target=my_service) #使用自定义的函数名
    worker_1 = multiprocessing.Process(name='worker 1', target=worker)
    worker_2 = multiprocessing.Process(target=worker) # use default name  #使用默认的函数名，即Process-3

    worker_1.start()
    worker_2.start()
    service.start()

import multiprocessing
import time
import sys

def daemon():
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    time.sleep(2)
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

def non_daemon():  #默认情况下，主程序将不会退出，直到所有的孩子都已经退出
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False   #默认是不是守护进程，以便通过真正轮流守护模式

    d.start()
    time.sleep(1)
    n.start()

import multiprocessing
import time
import sys

def daemon():
    print 'Starting:', multiprocessing.current_process().name
    time.sleep(2)
    print 'Exiting :', multiprocessing.current_process().name

def non_daemon():
    print 'Starting:', multiprocessing.current_process().name
    print 'Exiting :', multiprocessing.current_process().name

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False

    d.start()
    time.sleep(1)
    n.start()

    d.join() #会使主调线程堵塞，直到被调用线程运行结束或超时,join函数可以设置超时， 如果进程在超时期限内没有完成就返回 比如join(1)，可以通过d.is_alive()查看进程是否存活
    n.join()

import multiprocessing
import time

def slow_worker():
    print 'Starting worker'
    time.sleep(0.1)
    print 'Finished worker'

if __name__ == '__main__':
    p = multiprocessing.Process(target=slow_worker)
    print 'BEFORE:', p, p.is_alive()

    p.start()
    print 'DURING:', p, p.is_alive()

    p.terminate()  #发送结束进程信号
    print 'TERMINATED:', p, p.is_alive()

    p.join()  #更新状态以反应终止后的状态
    print 'JOINED:', p, p.is_alive()

前言：multiprocessing是一个使用方法类似threading模块的进程模块，允许程序员做并行开发，从python2.6开始内置

import multiprocessing
import logging,sys
def worker():
    print 'Worker'
    sys.stdout.flush() 
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        multiprocessing.log_to_stderr(logging.DEBUG)  #记录日志,也可以使用一下注释的三句
        #multiprocessing.log_to_stderr() 
        #logger = multiprocessing.get_logger() 
        #logger.setLevel(logging.INFO)
        p = multiprocessing.Process(target=worker)  #实例化Process
        jobs.append(p)
        p.start() #调用start()开始，类似于threading.Thread

import multiprocessing

def worker(num): #带参数的函数
    print 'Worker:', num
    return

if __name__ == '__main__':
    jobs = []
    for i in range(5):
        p = multiprocessing.Process(target=worker, args=(i,)) #带参数的目标
        jobs.append(p)
        p.start()

import multiprocessing
import time

def worker():
    name = multiprocessing.current_process().name  #当前进程名字
    print name, 'Starting'
    time.sleep(2)
    print name, 'Exiting'

def my_service():
    name = multiprocessing.current_process().name
    print name, 'Starting'
    time.sleep(3)
    print name, 'Exiting'

if __name__ == '__main__':
    service = multiprocessing.Process(name='my_service', target=my_service) #使用自定义的函数名
    worker_1 = multiprocessing.Process(name='worker 1', target=worker)
    worker_2 = multiprocessing.Process(target=worker) # use default name  #使用默认的函数名，即Process-3

    worker_1.start()
    worker_2.start()
    service.start()

import multiprocessing
import time
import sys

def daemon():
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    time.sleep(2)
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

def non_daemon():  #默认情况下，主程序将不会退出，直到所有的孩子都已经退出
    p = multiprocessing.current_process()
    print 'Starting:', p.name, p.pid
    sys.stdout.flush()
    print 'Exiting :', p.name, p.pid
    sys.stdout.flush()

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False   #默认是不是守护进程，以便通过真正轮流守护模式

    d.start()
    time.sleep(1)
    n.start()

import multiprocessing
import time
import sys

def daemon():
    print 'Starting:', multiprocessing.current_process().name
    time.sleep(2)
    print 'Exiting :', multiprocessing.current_process().name

def non_daemon():
    print 'Starting:', multiprocessing.current_process().name
    print 'Exiting :', multiprocessing.current_process().name

if __name__ == '__main__':
    d = multiprocessing.Process(name='daemon', target=daemon)
    d.daemon = True

    n = multiprocessing.Process(name='non-daemon', target=non_daemon)
    n.daemon = False

    d.start()
    time.sleep(1)
    n.start()

    d.join() #会使主调线程堵塞，直到被调用线程运行结束或超时,join函数可以设置超时， 如果进程在超时期限内没有完成就返回 比如join(1)，可以通过d.is_alive()查看进程是否存活
    n.join()

import multiprocessing
import time

def slow_worker():
    print 'Starting worker'
    time.sleep(0.1)
    print 'Finished worker'

if __name__ == '__main__':
    p = multiprocessing.Process(target=slow_worker)
    print 'BEFORE:', p, p.is_alive()

    p.start()
    print 'DURING:', p, p.is_alive()

    p.terminate()  #发送结束进程信号
    print 'TERMINATED:', p, p.is_alive()

    p.join()  #更新状态以反应终止后的状态
    print 'JOINED:', p, p.is_alive()