协程 及yield回顾

yield回顾

import time
import queue


def consumer(name):
    print("--->starting eating baozi...")
    while True:
        print('befor yield')
        new_baozi = yield
        print('after yield')
        print("[%s] is eating baozi %s" % (name, new_baozi))
        # time.sleep(1)


def producer():
    next(con)
    next(con2)
    n = 0

    time.sleep(5)
    while n < 2:
        n += 1
        con.send(n)
        con2.send(n)
        print("\033[32;1m[producer]\033[0m is making baozi %s" % n)


if __name__ == '__main__':
    con = consumer("c1")
    con2 = consumer("c2")
    producer()

程序主要目的为实现producer和consumer两个函数在一个线程同时运行，并通过yield交互，达到单进程中的并行效果。
new_baozi = yield
这句的意思是，将send的结果通过yield返回给neu_baozi，此时程序阻塞在此，知道send执行。
进而实现了并行。当consumer继续运行后又在此阻塞在次，进而等待send

已知的问题，当producer sleep时整个协程会直接block阻塞。故使用gevent解决。

GEVENT

import gevent

def func1():
    print('\033[31;1min func1 start...\033[0m')
    gevent.sleep(2)
    print('\033[31;1mback to func1...\033[0m')


def func2():
    print('\033[32;1min func2 start...\033[0m')
    gevent.sleep(1)
    print('\033[32;1mback to func2...\033[0m')

def func3():
    print('\033[32;1min func3 start...\033[0m')
    gevent.sleep(1)
    print('\033[32;1mback to func3...\033[0m')

gevent.joinall([
    gevent.spawn(func1),
    gevent.spawn(func2),
    gevent.spawn(func3),
])


# C:\Python35\python.exe D:/python/oldboy/DAY09/coroutine_01.py
# in func1 start...
# in func2 start...
# in func3 start...
# back to func2...
# back to func3...
# back to func1...
#
# Process finished with exit code 0

gevent.joinall([
gevent.spawn(func1),
gevent.spawn(func2),
gevent.spawn(func3),
])
程序采用如上方法调用各个函数，通过
gevent.sleep(1)
通过gevent的sleep来实现sleep不阻塞。

单线程实现爬虫并发

from gevent import monkey;

monkey.patch_all()
import gevent
from  urllib.request import urlopen


def f(url):
    print('GET: %s' % url)
    resp = urlopen(url)
    data = resp.read()
    print('%d bytes received from %s.' % (len(data), url))


gevent.joinall([
    gevent.spawn(f, 'https://weibo.com/'),
    gevent.spawn(f, 'https://www.baidu.com/'),
    gevent.spawn(f, 'https://github.com/'),
])

其中如下语句为取消所有阻塞，从而达到并发。

monkey.patch_all()

同步与异步效率对比

import gevent

def task(pid):
    gevent.sleep(0.5)
    print('Task %s done' % pid)

def synchronous():
    for i in range(1, 10):
        task(i)

def asynchronous():
    threads = [gevent.spawn(task, i) for i in range(1, 10)]
    gevent.joinall(threads)

print('Synchronous:')
synchronous()

print('Asynchronous:')
asynchronous()

GEVENT实现socket并发

#   SERV
# import sys
import time
import socket

import gevent
from gevent import monkey
from gevent import socket

monkey.patch_all()

def server(port):
    s = socket.socket()
    s.bind(('0.0.0.0', port))
    s.listen(20)

    while True:
        cli, addr = s.accept()
        gevent.spawn(handle_request, cli)

def handle_request(conn):
    try:
        while True:
            data = conn.recv(1024)
            print("recv:", data)
            conn.send(data)
            if not data:
                conn.shutdown(socket.SHUT_WR)
    except Exception as ex:
        print(ex)

    finally:
        conn.close()

if __name__ == '__main__':
    server(8001)

#   CLI
import socket

HOST = 'localhost'
PORT = 8001
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))

while True:
    msg = bytes(input(">>:"), encoding='utf-8')
    s.sendall(msg)
    data = s.recv(1024)

    print('Recieve', data)

s.close()