Python Http 百万并发客户端实现 _ V1.0

最新推荐文章于 2025-05-19 16:09:24 发布

原创最新推荐文章于 2025-05-19 16:09:24 发布 · 1.4k 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#Python #socket #IP地址 #并发 #源代码

我的微博专栏收录该内容

17 篇文章

订阅专栏

1. 源代码

#!/usr/bin/env python
# coding=utf8
# si.hairui  ：  @2016.11.24
# Http Client：  @Python 2.7.5
# Platform   ：  @windows 7
# ---------------------------------------------
# 0.先设置机器注册表中TIME_WAIT的值，再运行本程序：
#   reg /?
#   reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\Tcpip\Parameters" /v "TcpTimedWaitDelay" /t REG_DWORD /d 2 /f
#   reg delete "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\Tcpip\Parameters" /v "TcpTimedWaitDelay" /f
# 1.上电之后读取配置文件，获取客户端列表和服务器列表；
# 2.校验配置文件中客户端IP的真实性（有效性）；
# 3.解析服务端URL字段内容
# 4.注册线程，开始并发请求
# ---------------------------------------------


# ---------------------------------------------
#    模块导入
# ---------------------------------------------
import socket
import httplib
import threading
from time import sleep, ctime


# ---------------------------------------------
#    配置开关
# ---------------------------------------------
PRINT_SW = 0  # 业务打印开关，0-关，1-开
PRINT_THR_SW = 0  # 线程打印开关，0-关，1-开
WHILE_TRUE_SW = 1  # 死循环开关，0-关，1-开


# ---------------------------------------------
#    全局变量
# ---------------------------------------------
#TCP_REQUEST = 0  # TCP请求总数，线程需要互斥锁访问
#TCP_REQUEST_LOCK = threading.Lock()  # 互斥锁访问
#TCP_ESTABLISHED = 0  # TCP建链成功计数器，线程需要互斥锁访问
#TCP_ESTABLISHED_LOCK = threading.Lock()  # 互斥锁访问


# 客户端信息列表
HTTP_CLIENT_LIST = []
'''
HTTP_CLIENT_LIST = [
    {'ip': '192.168.0.218', 'portbegin': 464,   'portend': 464},
    {'ip': '192.168.0.220', 'portbegin': 34684, 'portend': 34685}
]
'''
# 服务器信息
HTTP_SERVER_LIST = []
'''
HTTP_SERVER_LIST = [
    'http://192.168.0.236/web/page/',
    'http://192.168.0.235:80/web/video/aboutHFS/HFS.txt'
]
'''


# ---------------------------------------------
#    函数实现
# ---------------------------------------------
# 修改默认buff（详见socket.py Line:238, default_bufsize = 8192）
def buffInit():
	socket._fileobject.default_bufsize = 8192 * 10
	socket._fileobject._wbuf_len = 8192 * 10
	
	
# 读取配置文件，返回配置文件字符串
def readCfgFile():
	try:
		with open('config.cfg', 'r') as f:
			return f.read()  # 返回字符串
	except:
		print "Error: ReadCfgFile fail!"
		return None


# 获取本地业务地址,放入序列clientIps
def getServiceIp(clientIps):
	hostname = socket.gethostname()
	localIP = socket.gethostbyname(hostname)
	if PRINT_SW:
		print "Hostname:\n    %s" % hostname
		print "Local IP:\n    %s" % localIP
	ipList = socket.gethostbyname_ex(hostname)
	if PRINT_SW: print "ipList", ipList, "\n"  # ('Harry-PC', [], ['192.168.1.103', '192.168.1.250', '192.168.1.251', ...)
	for ips in ipList:
		# 过滤空序列、主机名
		if ips and (not isinstance(ips, str)):  # ips是一个序列，其中每个元素是IP地址字符串
			if PRINT_SW: print "External IP:"
			for ip in ips:
				if PRINT_SW: print "    %s" % ip
				clientIps.append(ip)




# 获取客户端列表,eg: HTTP_CLIENT_LIST = list(getClientInfo())
def getClientInfo():
	cfginfo = eval(readCfgFile())
	# 校验配置文件中的IP
	# a.首先获取本地业务IP地址
	locIps = []
	getServiceIp(locIps)
	# b.然后对比配置文件中的地址
	validDicts = []  # 保存配置文件中的合法元素
	for sockdict in cfginfo['clients']:
		try:
			locIps.index(sockdict['ip'])
		except ValueError:  # 如果发生异常，说明sockdict中的IP地址不在本PC上，则丢掉
			print "Error: " + sockdict['ip'] + " in <config.cfg> is not valid, it will be discarded!!!"
			continue
		else:  # 没有异常，则保存该字典
			validDicts.append(sockdict)
	if PRINT_SW: print validDicts
	return validDicts




# 获取服务端列表,eg: HTTP_SERVER_LIST = list(getServerInfo())
def getServerInfo():
	cfginfo = eval(readCfgFile())
	return cfginfo['servers']




# 根据URL获取Server的IP、port、filepath,以字典的形式返回
def getServerSocket(url):
	strIp = ""
	port = 0
	filepath = ""
	if 0 != url.find("http://"):  # URL必须以“http://”开头（暂时不考虑https的场景）
		print "Error: URL is invalid: %s !!!" % url
		return None
	# 首先，去掉“http://”
	urlUnhttp = url[7:]
	# 然后判断有无“/”
	firstXieGangLoc = urlUnhttp.find("/")  # urlUnhttp中第一个“/”下标
	firstMaoHaoLoc = urlUnhttp.find(":")  # urlUnhttp中第一个“:”下标
	if (-1) == firstXieGangLoc and (-1) == firstMaoHaoLoc:  # "192.168.0.235"
		strIp = urlUnhttp
		port = 80
		filepath = "/"
	elif (-1) == firstXieGangLoc and firstMaoHaoLoc > 0:  # "192.168.0.235:8988"
		strIp = urlUnhttp[: firstMaoHaoLoc]
		port = int(urlUnhttp[firstMaoHaoLoc + 1:])
		filepath = "/"
	elif firstXieGangLoc > 0 and (-1) == firstMaoHaoLoc:  # "192.168.0.235/..."
		strIp = urlUnhttp[: firstXieGangLoc]
		port = 80
		filepath = urlUnhttp[firstXieGangLoc:]
	elif firstXieGangLoc > 0 and firstMaoHaoLoc > 0:  # "192.168.0.235:8988/..."
		strIp = urlUnhttp[: firstMaoHaoLoc]  # 这种情况下，默认URL中“:”在“/”左边
		port = int(urlUnhttp[firstMaoHaoLoc + 1: firstXieGangLoc])
		filepath = urlUnhttp[firstXieGangLoc:]
	else:
		print "Error: URL is unable reslute: %s!!!" % url
	
	svrdict = {}.fromkeys(["ip", "port", "filepath"])
	svrdict["ip"] = strIp
	svrdict["port"] = port
	svrdict["filepath"] = filepath
	return svrdict




# 串行轮询服务器和客户端发起HTTP请求，eg: runHttpCliet(HTTP_CLIENT_LIST, HTTP_SERVER_LIST)
'''
def runHttpCliet(clients=[], servers=[]):
    if (None == clients) or (None == servers):  # 保护一下
        print "Error: clients or servers is None!!!"
        return
    for urlstr in servers:
        svrdict = getServerSocket(urlstr)  # 解析URL
        if None == svrdict or None == svrdict["ip"] or None == svrdict["port"]:
            print "Error: Server is invalid!!!", svrdict
            continue  # 执行下一个server
        if PRINT_SW:
            print "svrdict =", svrdict
        for httpClient in clients:
            if PRINT_SW:
                print "httpClient =", httpClient
            for clientPort in range(httpClient['portbegin'], httpClient['portend'] + 1):
                try:
                    myHttpConn = httplib.HTTPConnection(svrdict["ip"], svrdict["port"],
                        timeout = 2, # 设置连接超时等待（单位：s）
                        source_address = (httpClient["ip"], clientPort))
                    myHttpConn.debuglevel = 0  # 调试模式开关
                    if 0 == myHttpConn.debuglevel:
                        print "    ---- Http Request ----"
                        print "    " + httpClient["ip"] + ":", clientPort, \
                            " --> " + svrdict["ip"], svrdict["port"], ": " + svrdict['filepath']
                    myHttpConn.request("GET", svrdict['filepath'])  # 资源路径以“/”开始
                    myHttpResp = myHttpConn.getresponse()
                    if 0 == myHttpConn.debuglevel:
                        print "   ", myHttpResp.status, myHttpResp.reason  # 响应码和状态信息
                        # 此处必须读响应内容,如果不读取，buff将溢出！！！
                        data1 = myHttpResp.read()
                        pass
                except:
                    print "!!!---------- Exception ------------------------"
                    continue
                finally:
                    # print data1   # 打印响应内容
                    print "    Http(%s:%s --> %s:%s) will be closed!" % \
                          (httpClient["ip"], str(clientPort), svrdict["ip"], str(svrdict["port"]))
                    myHttpConn.close()  # 关闭连接
'''




# 并发轮训服务器发起HTTP请求，eg: runHttpCliet(clientsDic, HTTP_SERVER_LIST, delaySeconds)
# delaySeconds: 0-5，从程序开始运行时到发起链接的时延，单位：秒
def runHttpCliet(clientsDic={}, servers=[], delaySeconds=0, whileTrue=0):
	'''
	关于函数中sleep(m)的解释：


	在TCP/IP终止连接的四次握手中，当最后的ACK回复发出后，有个2MSL的时间等待，MSL指一个片段在网络中最大的存活时间，
	这个时间一般是30秒，所以基本上过60秒后就可以重新连接,详见：
	http://www.bkjia.com/Pythonjc/868893.html
	http://www.dewen.net.cn/q/8606
	http://www.51testing.com/html/48/202848-249774.html
	http://blog.youkuaiyun.com/mhfh611/article/details/8769617/
	对于 Windows，修改注册表（win+R运行输入regedit），在HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters
	上添加一个DWORD类型的值TcpTimedWaitDelay，一般认为不要少于60，不然可能会有麻烦。


	进入Windows命令行窗口：
	reg /?
	reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\Tcpip\Parameters" /v "TcpTimedWaitDelay" /t REG_DWORD /d 30 /f
	reg delete "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\Tcpip\Parameters" /v "TcpTimedWaitDelay" /f
	'''
	if (None == clientsDic) or (None == servers):  # 保护一下
		print "Error: clients or servers is None!!!"
		return
	for urlstr in servers:
		svrdict = getServerSocket(urlstr)  # 解析URL
		if None == svrdict or None == svrdict["ip"] or None == svrdict["port"]:
			print "Error: Server is invalid!!!", svrdict
			continue  # 执行下一个server
		if PRINT_SW: print "svrdict =", svrdict
		sleep(delaySeconds)  # tcp建链时间间隔
		whileTrueSw = 1
		while whileTrueSw:  # 开始死循环
			for clientPort in range(clientsDic['portbegin'], clientsDic['portend'] + 1):
				try:
					# TCP建链统计
					#global TCP_REQUEST, TCP_REQUEST_LOCK  # 多线程是共享资源的，使用全局变量
					#if TCP_REQUEST_LOCK.acquire():
						#TCP_REQUEST += 1
						#print "    TCP_REQUEST=", TCP_REQUEST
						#TCP_REQUEST_LOCK.release()
					
					myHttpConn = httplib.HTTPConnection(svrdict["ip"], svrdict["port"],
					                                    # timeout = 3, # 设置连接超时等待（单位：s）
					                                    source_address=(clientsDic["ip"], clientPort))
					myHttpConn.debuglevel = 0  # 调试模式开关
					if 0 == myHttpConn.debuglevel:
						print "    ---- Http Request ----"
						print "    " + clientsDic["ip"] + ":", clientPort, \
							" --> " + svrdict["ip"], svrdict["port"], ": " + svrdict['filepath']
					try:
						myHttpConn.request("GET", svrdict['filepath'])  # 资源路径以“/”开始
					except:
						print "    !!!---------- Exception: myHttpConn.request ----------------"
						continue
					
					# TCP成功建链统计
					#global TCP_ESTABLISHED, TCP_ESTABLISHED_LOCK  # 多线程是共享资源的，使用全局变量
					#if TCP_ESTABLISHED_LOCK.acquire():
						#TCP_ESTABLISHED += 1
						#print "    TCP_ESTABLISHED=", TCP_ESTABLISHED
						#TCP_ESTABLISHED_LOCK.release()
					try:
						myHttpResp = myHttpConn.getresponse()
					except:
						print "    !!!---------- Exception: myHttpConn.getresponse ----------------"
						continue
					if 0 == myHttpConn.debuglevel:
						print "   ", myHttpResp.status, myHttpResp.reason  # 响应码和状态信息
						# 此处必须读响应内容,如果不读取，buff将溢出！！！
						data1 = myHttpResp.read()
						pass
				except (), e:
					print "    !!!---------- Exception: %s ------------------------" % e
					continue
				finally:
					# print data1   # 打印响应内容
					print "    Http(%s:%s --> %s:%s) will be closed!" % \
					      (clientsDic["ip"], str(clientPort), svrdict["ip"], str(svrdict["port"]))
					myHttpConn.close()  # 关闭连接
					sleep(20)
			if 0 == WHILE_TRUE_SW:
				whileTrueSw -= 1
			else:
				pass




# 客户端并发线程子类，父类：threading.Thread
# 每一个客户端IP独占一个线程，线程对客户端列表和服务端列表数据只读不写
class clientThread(threading.Thread):
	def __init__(self, func, args, name='', printThrSw=0):
		threading.Thread.__init__(self)
		self.func = func
		self.args = args
		self.name = name
		self.printThrSw = printThrSw
	
	def getResult(self):
		return self.res
	
	def run(self):
		if self.printThrSw:
			print 'starting', self.name, 'at:', ctime(), ". delaySeconds=", self.args[2]
		self.res = apply(self.func, self.args)
		if self.printThrSw:
			print self.name, 'finished at:', ctime()




# 客户端注册线程
def clientRegThr():
	HTTP_CLIENT_LIST = list(getClientInfo())  # 构造客户端
	if PRINT_SW: print "HTTP_CLIENT_LIST =", HTTP_CLIENT_LIST
	HTTP_SERVER_LIST = list(getServerInfo())  # 构造服务端
	if PRINT_SW: print "HTTP_SERVER_LIST =", HTTP_SERVER_LIST
	
	threadNum = len(HTTP_CLIENT_LIST)  # 计算线程个数,也就是客户端的个数
	if PRINT_SW: print "threadNum =", threadNum
	clientThreads = []  # 线程列表
	for i in range(threadNum):  # 线程编号从0开始
		thr = clientThread(
			func=runHttpCliet,
			# args = (HTTP_CLIENT_LIST[i], HTTP_SERVER_LIST, random.randint(1,5)), # random.randint(1,5) == [1,5]
			args=(HTTP_CLIENT_LIST[i], HTTP_SERVER_LIST, i, WHILE_TRUE_SW),  # 打散并发时间
			name="clientThread_" + str(HTTP_CLIENT_LIST[i]['ip']),
			printThrSw=PRINT_THR_SW
		)
		clientThreads.append(thr)  # 添加线程至列表
	for i in range(threadNum):
		clientThreads[i].start()
	for i in range(threadNum):
		clientThreads[i].join()




# App的main()入口
def main():
	buffInit()
	clientRegThr()




# ---------------------------------------------
#    main模块
# ---------------------------------------------
if __name__ == "__main__":
	main()

2.配置文件

# ----------------------------------------------------------------------------
# config.cfg
# clients: {"ip": "客户端的IP地址", "portbegin": 起始端口号, "portend": 结尾端口号}
# servers: 暂时只支持 “http://IP地址”的格式的URL 
# 同一个IP地址不允许在下面出现2次或多次
# ----------------------------------------------------------------------------

{
    "clients" : [
            { "ip": "192.168.1.254",   "portbegin": 2343,    "portend": 12343  },
            { "ip": "192.168.1.253",   "portbegin": 16357,   "portend": 26357  },
            { "ip": "192.168.1.252",   "portbegin": 27134,   "portend": 37134  },
            { "ip": "192.168.1.251",   "portbegin": 38900,   "portend": 50000  }
    ],

    "servers" : [
        #"http://192.168.1.103/KMP/yfwy.mp3",
        #"http://192.168.1.103/KMP/16.jpg"
        "http://www.cplusplus.com",
        "http://www.appinn.com/",
        "http://www.downxia.com/"
    ]
}

3.客户端运行时

4.服务器运行时