Python多进程+gevent实例,协程


利用多进程(multiprocessing)+多协程(gevent)实现同时请求12个网站url

import gevent
from gevent import monkey;monkey.patch_socket();monkey.patch_ssl()

from multiprocessing import Pool
import requests
import time
from collections import deque
from functools import wraps

list = [
    "http://www.baidu.com",
    "http://www.qq.com",
    "http://www.weibo.com",
    "http://www.tencent.com",
    "http://www.jd.com",
    "http://www.meituan.com",
    "http://www.douban.com",
    "http://www.hao123.com",
    "http://www.vip.com",
    "http://www.sohu.com",
    "http://www.alibaba.com",
    "http://www.jumei.com",
]


def getUrlContent(url):
    '''
    获取url内容
    '''
    content = requests.post(url)
    print(content)
    return content

def coroutine(*url_args):
    '''
    创建协程
    '''
    spawnList = []
    for url in url_args:
        spawn = gevent.spawn(getUrlContent, url)
        spawnList.append(spawn)

    gevent.joinall(spawnList)

def printTime(func):
    '''
    包装函数
    '''
    @wraps(func)
    def wrapper(*args):
        startTime = time.time()
        r = func(*args)
        endTime = time.time()
        print("time %s" % str(endTime - startTime))
        return r
    return wrapper

@printTime
def createMultiProcess(processNumber, coroutineNumber):
    '''
    创建进程
    '''
    urlList = deque(list)
    p = Pool()
    for i in range(processNumber):
        urlBlock = []
        for process in range(coroutineNumber):
            url = urlList.pop()
            urlBlock.append(url)
        p.apply_async(coroutine, args=urlBlock)
    p.close()
    p.join()
    

一、使用6个进程 + 2个协程

http://www.jd.com
<Response [200]>
http://www.meituan.com
<Response [200]>
http://www.sohu.com
<Response [405]>
http://www.vip.com
<Response [403]>
http://www.tencent.com
<Response [200]>
http://www.weibo.com
<Response [200]>
http://www.douban.com
<Response [418]>
http://www.hao123.com
<Response [200]>
http://www.baidu.com
<Response [302]>
http://www.qq.com
<Response [200]>
http://www.jumei.com
<Response [403]>
http://www.alibaba.com
<Response [200]>

time 1.625

二、使用2个进程 + 6个协程

http://www.baidu.com
<Response [302]>
http://www.tencent.com
<Response [200]>
http://www.jd.com
<Response [200]>
http://www.qq.com
<Response [200]>
http://www.meituan.com
<Response [200]>
http://www.weibo.com
<Response [200]>
http://www.sohu.com
<Response [405]>
http://www.vip.com
<Response [403]>
http://www.jumei.com
<Response [403]>
http://www.douban.com
<Response [418]>
http://www.hao123.com
<Response [200]>
http://www.alibaba.com
<Response [200]>

time 1.505000114440918