发一个多线程通过 HTTP 下载文件的类(Linux下的实现)

多线程下载文件这个话题已经是老汤了。

在HTTP协议1.1中，允许通过增加一个HTTP Header“Range”来指定下载文件的区间。
所以一般的做法都是：

• 首先获取到文件大小（通过Content-Length）
• 开一个线程池在进行分块下载。

而在具体怎么实现的流程上，还是有差别的。

1. 标准的做法是：首先用一个线程发送HTTP GET指令，服务器会返回Content-Length，并能够根据协议判断出服务器是否支持Range。如果支持Range，则调配其它线程对后续部分分块下载。第一个线程继续下载第1块。

2. 还一种做法，首先发送HTTP HEAD指令，通过返回的Content-Length进行分块，调配线程进行下载。

这里提供一个类，属于第2种。
为了提高IO性能，类中可以使用内存映射文件方式进行操作。

1. #ifndefCHTTPFILEDOWNLOADER_H_
2. #defineCHTTPFILEDOWNLOADER_H_
4. #include"Generic.h"
6. classCHttpFileDownloader{
7. public:
8. CHttpFileDownloader();
9. virtual~CHttpFileDownloader();
11. booldownloadUrlToFile(constchar*lpszUrl,constchar*lpszFile);
12. boolwaitForCompletion(void);
14. private:
15. stringm_strLocalFile;
16. pthread_tm_lLeaderThread;
17. structsockaddr_inm_stServerAddr;
18. charm_szResourceURI[1024];
19. charm_szDomain[1024];
20. charm_szHost[1024];
21. charm_szUrl[1024];
23. sem_tm_stDownSem;
24. pthread_mutex_tm_stDownloadThreadMutex;
25. intm_nDownloadThreadCnt;
26. boolm_bFailed;
27. sem_tm_stCompleteSem;
28. boolm_bSuccess;
30. staticvoid*leaderThread(void*param);
31. staticvoid*downloadThread(void*param);
32. booldownloadProcess(void);
33. voiddownloadBlock(unsignedchar*pMemory,intnRangeStart,intnRangeSize);
35. boolsendBuffer(intnSocket,char*pBuf,intnSize);
36. boolsendStringStream(intnSocket,stringstream&oStream);
37. intrecvStringStream(intnSocket,stringstream&oStream);
38. std::vector<string>parseResponse(stringstrResponse);
39. boolisHttpStatusSuccess(string&strHttpResponse);
40. stringgetHeaderValueByName(constchar*lpszHeader,std::vector<string>&vItems);
42. };
44. #endif/*CHTTPFILEDOWNLOADER_H_*/

1. /*
2. *CHttpFileDownloader.cpp
3. *
4. *Createdon:2008-12-15
5. *Author:root
6. */
7. #include"Generic.h"
8. #include"CMainApp.h"
9. #include"CHttpFileDownloader.h"
11. //64K
12. #defineDOWNLOAD_BLOCK_SIZE1024*512
13. #defineMAX_DOWNLOAD_THREAD5
16. typedefstruct_tagDownloadTask
17. {
18. CHttpFileDownloader*pThis;
19. unsignedchar*pStart;
20. intnSize;
21. intnRangeStart;
22. }tagDownloadTask,*LPDownloadTask;
25. CHttpFileDownloader::CHttpFileDownloader(){
26. sem_init(&m_stCompleteSem,0,0);
27. }
29. CHttpFileDownloader::~CHttpFileDownloader(){
30. sem_destroy(&m_stCompleteSem);
31. }
33. boolCHttpFileDownloader::sendStringStream(intnSocket,stringstream&oStream)
34. {
35. intnSize=oStream.str().length()*sizeof(char);
36. char*pBuffer=newchar[nSize];
37. memcpy(pBuffer,oStream.str().c_str(),nSize);
39. intnSent=0;
40. while(nSent<nSize)
41. {
42. intnRet=send(nSocket,(char*)(pBuffer+nSent),nSize-nSent,0);
43. if(nRet==-1)
44. break;
46. nSent+=nRet;
47. }
49. delete[]pBuffer;
51. return(nSent==nSize);
52. }
54. boolCHttpFileDownloader::sendBuffer(intnSocket,char*pBuf,intnSize)
55. {
56. intnSent=0;
57. while(nSent<nSize)
58. {
59. intnRet=send(nSocket,(char*)(pBuf+nSent),nSize-nSent,0);
60. if(nRet==-1)
61. break;
63. nSent+=nRet;
64. }
66. return(nSent==nSize);
67. }
69. intCHttpFileDownloader::recvStringStream(intnSocket,stringstream&oStream)
70. {
71. intnReceived=0;
72. while(true)
73. {
74. charszBuf[1025]={0};
75. intnRet=recv(nSocket,szBuf,1024,0);
76. if(nRet==0)
77. break;
79. if(nRet<0)
80. break;
82. oStream<<szBuf;
83. nReceived+=nRet;
85. if(oStream.str().find("\r\n\r\n")!=string::npos)
86. break;
87. }
88. returnnReceived;
89. }
91. std::vector<string>CHttpFileDownloader::parseResponse(stringstrResponse)
92. {
93. std::vector<string>vItems;
94. size_tnLast=strResponse.find("\r\n\r\n");
95. if(nLast>=strResponse.length())
96. returnvItems;
99. size_tnPos=0;
100. while(nPos<nLast)
101. {
102. size_tnFind=strResponse.find("\r\n",nPos);
103. if(nFind>nLast)
104. break;
106. vItems.push_back(strResponse.substr(nPos,nFind-nPos));
107. nPos=nFind+2;
108. }
109. returnvItems;
110. }
112. boolCHttpFileDownloader::isHttpStatusSuccess(string&strHttpResponse)
113. {
114. size_tnBegin=strHttpResponse.find('');
115. if(nBegin>strHttpResponse.length())
116. returnfalse;
118. size_tnEnd=strHttpResponse.find_last_of('');
119. if(nEnd>strHttpResponse.length())
120. returnfalse;
122. stringstrStatusCode=strHttpResponse.substr(nBegin+1,nEnd-nBegin-1);
123. intnStatusCode=atoi(strStatusCode.c_str());
124. return(nStatusCode>=200&&nStatusCode<300);
125. }
127. stringCHttpFileDownloader::getHeaderValueByName(constchar*lpszHeader,std::vector<string>&vItems)
128. {
129. stringstrHeader=lpszHeader;
130. std::transform(strHeader.begin(),strHeader.end(),strHeader.begin(),(int(*)(int))std::tolower);
131. strHeader.append(":");
133. stringstrValue="";
135. std::vector<string>::iteratoriter;
136. for(iter=vItems.begin();iter!=vItems.end();iter++)
137. {
138. stringstrItem=*iter;
139. std::transform(strItem.begin(),strItem.end(),strItem.begin(),(int(*)(int))std::tolower);
141. if(strItem.find(strHeader)!=0)
142. continue;
144. strValue=strItem.substr(strHeader.length());
145. break;
146. }
147. returnstrValue.erase(0,strValue.find_first_not_of(''));
148. }
151. boolCHttpFileDownloader::downloadUrlToFile(constchar*lpszUrl,constchar*lpszFile)
152. {
153. memset(m_szUrl,0,1024);
154. memcpy(m_szUrl,lpszUrl,strlen(lpszUrl));
155. m_strLocalFile=lpszFile;
157. //createthread
158. intnErr=pthread_create(&m_lLeaderThread
159. ,NULL
160. ,&leaderThread
161. ,this
162. );
163. if(nErr!=0)
164. {
165. CMainApp::getSingleton()->log("Error:pthread_createdownloadleaderthreadfailed.Return=%d,Message=%s"
166. ,nErr
167. ,strerror(nErr)
168. );
169. returnfalse;
170. }
171. returntrue;
172. }
174. boolCHttpFileDownloader::waitForCompletion(void)
175. {
176. sem_wait(&m_stCompleteSem);
177. returnm_bSuccess;
178. }
180. void*CHttpFileDownloader::leaderThread(void*param)
181. {
182. CHttpFileDownloader*pThis=static_cast<CHttpFileDownloader*>(param);
184. CMainApp::getSingleton()->log("Info:downloadfile\"%s\"start..."
185. ,pThis->m_szUrl
186. );
187. pThis->m_bSuccess=pThis->downloadProcess();
188. sem_post(&pThis->m_stCompleteSem);
189. CMainApp::getSingleton()->log("Info:downloadfile\"%s\"%s..."
190. ,pThis->m_szUrl
191. ,pThis->m_bSuccess?"success":"failed"
192. );
193. returnNULL;
194. }
197. boolCHttpFileDownloader::downloadProcess(void)
198. {
199. //parsetheurlandport
200. stringstrUrl=m_szUrl;
201. std::transform(strUrl.begin(),strUrl.end(),strUrl.begin(),(int(*)(int))std::tolower);
203. size_tuFind=strUrl.find("http://");
204. if(uFind!=0)
205. {
206. CMainApp::getSingleton()->log("Error:InvalidURL:%s"
207. ,m_szUrl
208. );
209. returnfalse;
210. }
212. intnLen=string("http://").length();
213. uFind=strUrl.find('/',nLen);
214. if(uFind>strUrl.length())
215. {
216. CMainApp::getSingleton()->log("Error:InvalidURL:%s"
217. ,m_szUrl
218. );
219. returnfalse;
220. }
222. strUrl=m_szUrl;
223. stringstrHost=strUrl.substr(nLen,uFind-nLen);
224. stringstrResourceURI=strUrl.substr(uFind);
226. stringstrDomain=strHost;
227. uintuPort=80;
228. uFind=strHost.find(':');
229. if(uFind<strHost.length())
230. {
231. strDomain=strHost.substr(0,uFind);
232. uPort=atoi(strHost.substr(uFind+1).c_str());
233. }
235. structhostent*pHostent=gethostbyname(strDomain.c_str());
236. if(pHostent==NULL)
237. {
238. CMainApp::getSingleton()->log("Error:failedtoresolvetheIPaddressfortheURL:%s"
239. ,m_szUrl
240. );
241. returnfalse;
242. }
244. memset(&m_stServerAddr,0,sizeof(m_stServerAddr));
245. m_stServerAddr.sin_family=AF_INET;
246. m_stServerAddr.sin_port=htons((short)uPort);
247. memcpy((char*)&m_stServerAddr.sin_addr.s_addr,pHostent->h_addr_list[0],pHostent->h_length);
249. intsock=socket(AF_INET,SOCK_STREAM,0);
250. if(sock==-1)
251. {
252. CMainApp::getSingleton()->log("Error:socketfailed.error=%s"
253. ,strerror(errno)
254. );
255. returnfalse;
256. }
258. memset(m_szResourceURI,0,1024);
259. memcpy(m_szResourceURI,strResourceURI.c_str(),strlen(strResourceURI.c_str()));
260. memset(m_szHost,0,1024);
261. memcpy(m_szHost,strHost.c_str(),strlen(strHost.c_str()));
262. memset(m_szDomain,0,1024);
263. memcpy(m_szDomain,strDomain.c_str(),strlen(strDomain.c_str()));
265. //populatetheHTTPHEADrequest
266. stringstreamstrHttp;
267. strHttp<<"HEAD"<<m_szResourceURI<<"HTTP/1.1\r\n";
268. strHttp<<"User-Agent:Mozilla/4.0(compatible;MSIE5.00;Windows98)\r\n";
269. strHttp<<"Host:"<<m_szHost<<"\r\n";
270. strHttp<<"Cache-Control:no-cache\r\n";
271. strHttp<<"Pragma:no-cache\r\n";
272. strHttp<<"Connection:Keep-Alive\r\n";
273. strHttp<<"Accept:*/*\r\n";
274. strHttp<<"\r\n";
276. intnRet=connect(sock
277. ,(structsockaddr*)&m_stServerAddr
278. ,sizeof(structsockaddr)
279. );
280. if(nRet==-1)
281. {
282. CMainApp::getSingleton()->log("Error:failedtoconnecttoURL:%s"
283. ,m_szUrl
284. );
285. returnfalse;
286. }
288. structtimevaltv={0};
289. tv.tv_sec=15;
290. if(setsockopt(sock,SOL_SOCKET,SO_RCVTIMEO,(char*)&tv,sizeof(tv)))
291. {
292. CMainApp::getSingleton()->log("Error:setsockoptfailed(1).error=%s"
293. ,strerror(errno)
294. );
295. returnfalse;
296. }
298. if(!sendStringStream(sock,strHttp))
299. {
300. CMainApp::getSingleton()->log("Error:failedtosendtheHTTPHEADrequesttoURL:%s"
301. ,m_szUrl
302. );
303. returnfalse;
304. }
306. stringstreamstrResponse;
307. recvStringStream(sock,strResponse);
308. shutdown(sock,SHUT_RDWR);
309. close(sock);
311. //parsetheresponse
312. std::vector<string>vItems=parseResponse(strResponse.str());
313. if(vItems.size()==0)
314. {
315. CMainApp::getSingleton()->log("Error:theHTTPHEADresponsecontainsnothing.URL:%s"
316. ,m_szUrl
317. );
318. returnfalse;
319. }
321. if(!isHttpStatusSuccess(vItems[0]))
322. {
323. CMainApp::getSingleton()->log("Error:%s.URL:%s"
324. ,vItems[0].c_str()
325. ,m_szUrl
326. );
327. returnfalse;
328. }
330. stringstrContentLen=getHeaderValueByName("Content-Length",vItems);
331. if(strContentLen.length()==0)
332. {
333. CMainApp::getSingleton()->log("Error:InvalidContent-LengthinHTTPHEADresponse.URL:%s"
334. ,m_szUrl
335. );
336. returnfalse;
337. }
339. intnContentLength=atoi(strContentLen.c_str());
341. //createfile
342. std::ofstreamoutStream;
343. outStream.open(m_strLocalFile.c_str(),ios_base::out|ios_base::binary|ios_base::trunc);
344. outStream.seekp(nContentLength-1);
345. outStream.put('\0');
346. outStream.close();
348. intfd=open(m_strLocalFile.c_str(),O_RDWR);
349. if(fd==-1)
350. {
351. CMainApp::getSingleton()->log("Error:cannotcreatefile\"%s\".%s"
352. ,m_strLocalFile.c_str()
353. ,strerror(errno)
354. );
355. returnfalse;
356. }
358. unsignedchar*pMemory=(unsignedchar*)mmap(NULL,nContentLength,PROT_WRITE,MAP_SHARED|MAP_POPULATE|MAP_NONBLOCK,fd,0);
359. close(fd);
360. if(pMemory==MAP_FAILED)
361. {
362. CMainApp::getSingleton()->log("Error:failedtomapthefile\"%s\"intomemory;size=%d;error=%s"
363. ,m_strLocalFile.c_str()
364. ,nContentLength
365. ,strerror(errno)
366. );
367. returnfalse;
368. }
369. mlock(pMemory,nContentLength);
371. //createthread
372. sem_init(&m_stDownSem,0,MAX_DOWNLOAD_THREAD);
373. pthread_mutex_init(&m_stDownloadThreadMutex,NULL);
374. m_bFailed=false;
376. intnDownloadLength=0;
377. m_nDownloadThreadCnt=0;
378. while(true)
379. {
380. sem_wait(&m_stDownSem);
381. if(nDownloadLength>=nContentLength||
382. m_bFailed)
383. {
384. if(m_nDownloadThreadCnt==0)
385. break;
386. else
387. continue;
388. }
390. LPDownloadTaskpTask=(LPDownloadTask)malloc(sizeof(tagDownloadTask));
391. pTask->pStart=(unsignedchar*)(pMemory+nDownloadLength);
392. pTask->nSize=((DOWNLOAD_BLOCK_SIZE+nDownloadLength)>nContentLength)
393. ?(nContentLength-nDownloadLength):DOWNLOAD_BLOCK_SIZE;
394. pTask->nRangeStart=nDownloadLength;
395. pTask->pThis=this;
397. nDownloadLength+=pTask->nSize;
400. //createthread
401. pthread_tlThread;
402. intnErr=pthread_create(&lThread
403. ,NULL
404. ,&downloadThread
405. ,pTask
406. );
407. if(nErr!=0)
408. {
409. CMainApp::getSingleton()->log("Error:pthread_createdownloadthreadfailed.Error=%d,Message=%s"
410. ,nErr
411. ,strerror(nErr)
412. );
413. m_bFailed=true;
414. }
415. else
416. {
417. pthread_mutex_lock(&m_stDownloadThreadMutex);
418. m_nDownloadThreadCnt++;
419. pthread_mutex_unlock(&m_stDownloadThreadMutex);
420. }
421. }
423. pthread_mutex_destroy(&m_stDownloadThreadMutex);
424. sem_destroy(&m_stDownSem);
427. if(msync(pMemory,nContentLength,MS_SYNC)==-1)
428. {
429. CMainApp::getSingleton()->log("Error:failedtomsyncthefile\"%s\"frommemory;size=%d;error=%s"
430. ,m_strLocalFile.c_str()
431. ,nContentLength
432. ,strerror(errno)
433. );
434. m_bFailed=true;
435. }
437. munlock(pMemory,nContentLength);
438. munmap(pMemory,nContentLength);
440. return!m_bFailed;
441. }
445. void*CHttpFileDownloader::downloadThread(void*param)
446. {
447. LPDownloadTaskpTask=static_cast<LPDownloadTask>(param);
449. pTask->pThis->downloadBlock(pTask->pStart
450. ,pTask->nRangeStart
451. ,pTask->nSize
452. );
454. pthread_mutex_lock(&(pTask->pThis->m_stDownloadThreadMutex));
455. pTask->pThis->m_nDownloadThreadCnt--;
456. pthread_mutex_unlock(&(pTask->pThis->m_stDownloadThreadMutex));
458. sem_post(&(pTask->pThis->m_stDownSem));
459. free(pTask);
461. returnNULL;
462. }
464. voidCHttpFileDownloader::downloadBlock(unsignedchar*pMemory,intnRangeStart,intnRangeSize)
465. {
466. CMainApp::getSingleton()->log("Info:downloadblock\"%s\"[%08d-%08d]start..."
467. ,m_szUrl
468. ,nRangeStart
469. ,nRangeStart+nRangeSize-1
470. );
472. intnReceived=0;
473. intnErrorTimes=0;
474. while(nReceived<nRangeSize&&nErrorTimes<5&&!m_bFailed)
475. {
476. intnSocket=socket(AF_INET,SOCK_STREAM,0);
477. if(nSocket==-1)
478. {
479. nErrorTimes++;
480. continue;
481. }
483. intnRet=connect(nSocket
484. ,(structsockaddr*)&m_stServerAddr
485. ,sizeof(structsockaddr)
486. );
487. if(nRet==-1)
488. {
489. nErrorTimes++;
490. CMainApp::getSingleton()->log("Error:failedtoconnecttoURL:%s"
491. ,m_szUrl
492. );
493. continue;
494. }
496. {
497. structtimevaltv={0};
498. tv.tv_sec=15;
499. if(setsockopt(nSocket,SOL_SOCKET,SO_RCVTIMEO,(char*)&tv,sizeof(tv)))
500. {
501. nErrorTimes++;
502. CMainApp::getSingleton()->log("Error:setsockoptfailed(2).error=%s"
503. ,strerror(errno)
504. );
505. continue;
506. }
507. }
509. {
510. charszRequest[4096]={0};
511. sprintf(szRequest,"GET%sHTTP/1.1\r\n"
512. "User-Agent:Mozilla/4.0(compatible;MSIE5.00;Windows98)\r\n"
513. "Host:%s\r\n"
514. "Cache-Control:no-cache\r\n"
515. "Pragma:no-cache\r\n"
516. "Connection:Keep-Alive\r\n"
517. "Accept:*/*\r\n"
518. "Range:bytes=%d-%d\r\n"
519. "\r\n"
520. ,m_szResourceURI
521. ,m_szHost
522. ,nRangeStart+nReceived
523. ,nRangeStart+nRangeSize-1
524. );
526. if(!sendBuffer(nSocket,szRequest,strlen(szRequest)))
527. {
528. shutdown(nSocket,SHUT_RDWR);
529. close(nSocket);
530. CMainApp::getSingleton()->log("Error:failedtosendtheHTTPGETrequesttoURL:%s"
531. ,m_szUrl
532. );
533. nErrorTimes++;
534. continue;
535. }
536. }
538. charszBuf[1024]={0};
539. nRet=recv(nSocket,szBuf,1024,0);
540. if(nRet<=0)
541. {
542. shutdown(nSocket,SHUT_RDWR);
543. close(nSocket);
544. CMainApp::getSingleton()->log("Error:recvfailed(1).returncode=%d,error=%s,URL=%s"
545. ,nRet
546. ,strerror(errno)
547. ,m_szUrl
548. );
549. nErrorTimes++;
550. continue;
551. }
553. stringstrHttpResponse;
554. intnRemain=0;
555. intnIndex=0;
556. for(nIndex=0;nIndex<nRet;nIndex++)
557. {
558. if(szBuf[nIndex]=='\r'&&
559. szBuf[nIndex+1]=='\n'&&
560. szBuf[nIndex+2]=='\r'&&
561. szBuf[nIndex+3]=='\n')
562. {
563. charszTemp[1025]={0};
564. memcpy(szTemp,szBuf,nIndex+4);
565. strHttpResponse=szTemp;
566. nRemain=nRet-(nIndex+4);
567. break;
568. }
569. }
571. if(strHttpResponse.length()==0)
572. {
573. shutdown(nSocket,SHUT_RDWR);
574. close(nSocket);
575. nErrorTimes++;
576. CMainApp::getSingleton()->log("Error:theresponsedoesnotcontainaHTTPheader(1),URL:%s"
577. ,m_szUrl
578. );
579. continue;
580. }
582. std::vector<string>vItems=parseResponse(strHttpResponse);
583. if(vItems.size()==0)
584. {
585. shutdown(nSocket,SHUT_RDWR);
586. close(nSocket);
587. nErrorTimes++;
588. CMainApp::getSingleton()->log("Error:theresponsedoesnotcontainaHTTPheader(2).URL:%s"
589. ,m_szUrl
590. );
591. continue;
592. }
594. if(!isHttpStatusSuccess(vItems[0]))
595. {
596. shutdown(nSocket,SHUT_RDWR);
597. close(nSocket);
598. nErrorTimes++;
599. CMainApp::getSingleton()->log("Error:%s.URL:%s"
600. ,vItems[0].c_str()
601. ,m_szUrl
602. );
603. continue;
604. }
606. if(nRemain>0)
607. {
608. memcpy((unsignedchar*)(pMemory+nReceived),&(szBuf[nIndex+4]),nRemain);
609. nReceived+=nRemain;
610. }
612. while((nReceived<nRangeSize)&&!m_bFailed)
613. {
614. nRet=recv(nSocket,(unsignedchar*)(pMemory+nReceived),nRangeSize-nReceived,0);
615. if(nRet<=0)
616. {
617. CMainApp::getSingleton()->log("Error:recvfalied(2).returncode=%d,error=%s,URL=%s"
618. ,nRet
619. ,strerror(errno)
620. ,m_szUrl
621. );
622. nErrorTimes++;
623. break;
624. }
625. nReceived+=nRet;
626. }
627. shutdown(nSocket,SHUT_RDWR);
628. close(nSocket);
629. }//while
631. m_bFailed=m_bFailed?m_bFailed:(nReceived!=nRangeSize);
633. CMainApp::getSingleton()->log("Info:downloadblock\"%s\"[%08d-%08d]%s."
634. ,m_szUrl
635. ,nRangeStart
636. ,nRangeStart+nRangeSize-1
637. ,(nReceived!=nRangeSize)?"Failed":"Success"
638. );
639. }