在跑廖雪峰的一段代码的时候出了两个问题,第一个问题,经过调研有解决方案,第二个是找到了问题的地方,但是没找到原因 问题1: _pickle.PicklingError: Can't pickle <function <lambda> at 0x000002BAAEF12F28>: attribute lookup <lambda> on __main__ failed 代码如下:
import random
import time
import queue
from multiprocessing.managers import BaseManager
task_queue = queue.Queue()
result_queue = queue.Queue()
def re_task_queue():
global task_queue
return task_queue
def re_result_queue():
global result_queue
return result_queue
class QueueManager(BaseManager):
pass
if __name__ == '__main__':
QueueManager.register("get_task_queue", callable=lambda :task_queue)
QueueManager.register("get_result_queue", callable=lambda :result_queue)
manager = QueueManager(address=("127.0.0.1", 5000), authkey=b'abc')
manager.start()
task = manager.get_task_queue()
result = manager.get_result_queue()
for i in range(10):
n = random.randint(0, 10000)
print "Put task %d..." % n
task.put(n)
print "Try to get results..."
for i in range(10):
r = result.get(timeout=10)
print "Result: %d" % r
manager.shutdown()
print "Master exit."
报错信息:
Traceback (most recent call last):
File "C:/Users/v_wangweihong/Desktop/supervisor-easy-master/task_master.py", line 39, in <module>
manager.start()
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\managers.py", line 524, in start
self._process.start()
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\process.py", line 130, in start
self._popen = Popen(self)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\forking.py", line 277, in __init__
dump(process_obj, to_child, HIGHEST_PROTOCOL)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\forking.py", line 199, in dump
ForkingPickler(file, protocol).dump(obj)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 224, in dump
self.save(obj)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 331, in save
self.save_reduce(obj=obj, *rv)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 425, in save_reduce
save(state)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 687, in _batch_setitems
save(v)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 568, in save_tuple
save(element)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 655, in save_dict
self._batch_setitems(obj.iteritems())
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 687, in _batch_setitems
save(v)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 568, in save_tuple
save(element)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 286, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\pickle.py", line 754, in save_global
(obj, module, name))
pickle.PicklingError: Can't pickle <function <lambda> at 0x00000000035899E8>: it's not found as __main__.<lambda>
解决办法: pickle模块不能序列化lambda,需要自定义函数 修改后代码:
import random
import time
import queue
from multiprocessing.managers import BaseManager
task_queue = queue.Queue()
result_queue = queue.Queue()
def re_task_queue():
global task_queue
return task_queue
def re_result_queue():
global result_queue
return result_queue
class QueueManager(BaseManager):
pass
if __name__ == '__main__':
QueueManager.register("get_task_queue", callable=re_task_queue)
QueueManager.register("get_result_queue", callable=re_result_queue)
manager = QueueManager(address=("127.0.0.1", 5000), authkey=b'abc')
manager.start()
task = manager.get_task_queue()
result = manager.get_result_queue()
for i in range(10):
n = random.randint(0, 10000)
print "Put task %d..." % n
task.put(n)
print "Try to get results..."
for i in range(10):
r = result.get(timeout=10)
print "Result: %d" % r
manager.shutdown()
print "Master exit."
问题2: 如果缺少以下这行代码会抛异常(EOFERROR): if __name__ == '__main__':
报错信息:
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\forking.py", line 380, in main
prepare(preparation_data)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\forking.py", line 510, in prepare
'__parents_main__', file, path_name, etc
File "C:\Users\v_wangweihong\Desktop\supervisor-easy-master\task_master.py", line 39, in <module>
manager.start()
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\managers.py", line 524, in start
self._process.start()
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\process.py", line 130, in start
self._popen = Popen(self)
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\forking.py", line 258, in __init__
cmd = get_command_line() + [rhandle]
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\forking.py", line 358, in get_command_line
is not going to be frozen to produce a Windows executable.''')
RuntimeError:
Attempt to start a new process before the current process
has finished its bootstrapping phase.
This probably means that you are on Windows and you have
forgotten to use the proper idiom in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce a Windows executable.
Traceback (most recent call last):
File "C:/Users/v_wangweihong/Desktop/supervisor-easy-master/task_master.py", line 39, in <module>
manager.start()
File "C:\Users\v_wangweihong\AppData\Local\Continuum\anaconda2\lib\multiprocessing\managers.py", line 528, in start
self._address = reader.recv()
EOFError
很奇葩的问题:
有谁遇到 可以留言
在尝试使用pickle模块在Windows上进行分布式进程时,遇到了一个异常问题:无法序列化lambda函数。这导致了运行时错误,目前寻求解决方案。
1062

被折叠的 条评论
为什么被折叠?



