为什么WININET 和 SOCKET 实现网页下载速度相差很大

测试下来,下载3个页面WININET大概需要1秒钟就行
SOCKET确大概需要1分钟,晕跟踪程序运行,发现SOCKET方式,时间都花在recv上
(当然,recv的时间也包括了send的时间,不能完全测出,SOCKET在recv的时候,本身就要检查发送缓冲区是否空,如果有数据,会先等发送完,再进行真正的recv)

socket 方式代码写的有问题

那个肯定是你代码问题。
WinInet底层应该也是socket，进行了一层封装都可以达到那种速度，如果直接写，写得好的话，至少不会慢。

SOCKET确大概需要1分钟,晕估计是recv部分有问题，网页那边已经接发送了，你这边还在接收，1分钟后可能别人网站强制断开连接，你这边recv才能返回
你是不是循环recv的？试试看用一个大Buff一次接收完

在网速恒定的情况下我的感觉是直接操作socket快

我也是这样想地,SOCKET 总比WININET 快
结果测试下来让我跌破眼镜贴代码上来?

发送HTTP请求:
// construct request header
m_strHeaders="GET ";
m_strHeaders+=m_strObject;
m_strHeaders+=" HTTP/1.1\r\n";
m_strHeaders+="Host: ";
m_strHeaders+=m_strServer;
m_strHeaders+="\r\n";
m_strHeaders+="Accept: */*\r\n";
m_strHeaders += "Pragma: no-cache\r\n";
m_strHeaders += "Cache-Control: no-cache\r\n";
m_strHeaders+="User-Agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT;)\r\n";
m_strHeaders+="Connection: Keep-Alive\r\n";
// m_strHeaders+="Proxy-Connection: Keep-Alive\r\n";
// m_strHeaders+="Range: bytes=100-\r\n";
// m_strHeaders+="Content Type: text/html;charset=GBK\r\n";
if(lpszHeaders) m_strHeaders+=lpszHeaders;
m_strHeaders+="\r\n";
int nSend=send(m_socket,(LPCTSTR)m_strHeaders,m_strHeaders.GetLength(),0);接收:
while(1)
{
nRecv=recv(m_socket,szData,4096,0);
if(SOCKET_ERROR==nRecv)
{
CString strErr;
strErr.Format("Error while recv(ErrorCode: %d)",::GetLastError());
SetErrorText(strErr);
CloseSocket();
return FALSE;
}
...
}

SOCKET肯定比wininet快，但是可能比不过winhttp，尤其是访问多域名并发环境

我想知道的是,问题在哪儿,没有人用SOCKET实现过网页下载???
没有发现速度不可以忍受?

while(1)
{
nRecv=recv(m_socket,szData,4096,0);
if(SOCKET_ERROR==nRecv)
{
CString strErr;
strErr.Format("Error while recv(ErrorCode: %d)",::GetLastError());
SetErrorText(strErr);
CloseSocket();
return FALSE;
}
...
}这里不对，http协议头有长度Content-Length，或者Transfer-Encoding：chunked。如果你的请求没有包含Connection: Close 。服务器是不会断开的。这样 SOCKET_ERROR==nRecv 不会成立。

#include "stdafx.h"
#include "HttpClient.h"
#include "../utils/ZvUtils.h"
#include "UriCodec.h"
#include "HttpDefine.h"namespace ZvLibs
{ CHttpClient::CHttpClient()
:CHttpSocket(true)
{
m_sClient = INVALID_SOCKET;
m_dwLenFiles = 0;
} CHttpClient::~CHttpClient()
{
} void CHttpClient::SetKeepAlive()
{
AddHeader("Connection", "keep-alive");
} bool CHttpClient::IsPost()
{
return m_mapParamPost.size()>0 || m_mapFiles.size()>0;
} bool CHttpClient::Connect(const string & szIp, int nPort)
{
AddHeader("Host", szIp);
return CHttpSocket::Connect(szIp, nPort);
} void CHttpClient::Close()
{
Clearup();
CHttpSocket::Close();
} string CHttpClient::GetPostLength()
{
int nLen = 0;
map<string, string>::iterator it; if(!m_bUploadFile)
{
for(it=m_mapParamPost.begin(); it!=m_mapParamPost.end(); it++)
{
nLen += it->first.length()+it->second.length()+1;
}
if(m_mapParamPost.size()>0)
{
nLen += m_mapParamPost.size()-1;
}
}
else
{
for(it=m_mapParamPost.begin(); it!=m_mapParamPost.end(); it++)
{
nLen += 42+41+it->first.length()+it->second.length()+4;
} for(it=m_mapFiles.begin(); it!=m_mapFiles.end(); it++)
{
nLen += 42+54+it->first.length()+it->second.length()+4+40;
}
if(nLen>0)
nLen+=m_dwLenFiles+42+2;
} ostringstream ss;
ss << nLen;
return ss.str();
} void CHttpClient::AddHeader( const string &  szName, const string &  szValue )
{
m_mapHeaderOut[szName] = szValue;
} void CHttpClient::AddParamUrl( const string &  szName, const string &  szValue )
{
m_mapParamUrl[szName] = szValue;
} void CHttpClient::AddParamPost( const string &  szName, const string &  szValue )
{
m_mapParamPost[szName] = szValue;
} bool CHttpClient::AddFile(const string &  szName, const string &  szFilePath)
{
if(m_mapFiles.find(szName)!=m_mapFiles.end())
{
SetErr("file name exist");
return false;
} WIN32_FIND_DATA fd; if(!FileExist(szFilePath, &fd))
{
SetErr("file not exist");
return false;
}
m_dwLenFiles += fd.nFileSizeLow;
m_mapFiles[szName] = szFilePath; return true;
} bool CHttpClient::SendData()
{
ostringstream ss;
string str;
map<string, string>::iterator it; if(!m_bUploadFile)
{
for(it=m_mapParamPost.begin(); it!=m_mapParamPost.end(); it++)
{
ss << it->first << "=" << it->second;
} if(!Send(ss.str())) return false;
}
else
{
for(it=m_mapParamPost.begin(); it!=m_mapParamPost.end(); it++)
{
ss.str("");
ss << m_szBoundary << CRLF;
ss << "Content-Disposition: form-data; name=\"" << it->first << "\"" << CRLFCRLF;
ss << it->second << CRLF; if(!Send(ss.str())) return false;
} for(it=m_mapFiles.begin(); it!=m_mapFiles.end(); it++)
{
ss.str("");
ss << m_szBoundary << CRLF;
ss << "Content-Disposition: form-data; name=\"" << it->first << "\"; filename=\"" << it->second << "\"" << CRLF;
ss << "Content-Type: application/octet-stream" << CRLFCRLF;
if(!Send(ss.str())) return false; HANDLE hFile; hFile = CreateFile(it->second.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if(hFile==INVALID_HANDLE_VALUE)
{
SetErr(GetWinErr());
return false;
} DWORD dwSize = GetFileSize(hFile, NULL); if(dwSize==0xFFFFFFFF)
{
CloseHandle(hFile);
return false;
} if(!TransmitFile(m_sClient, hFile, dwSize, 0, NULL, NULL, 0))
{
SetErr(GetWinErr(), "TransmitFile");
CloseHandle(hFile);
return false;
}
CloseHandle(hFile); Send(CRLF);
} Send(m_szBoundary+"--\r\n");
} return true;
} bool CHttpClient::SendHeader()
{
ostringstream ss;
string str;
map<string, string>::iterator it; m_bPost = IsPost();
m_bUploadFile = m_mapFiles.size()>0; // request line
ss << (m_bPost?"POST":"GET") << " " << m_szUrl; if(m_mapParamUrl.size()>0)
{
if(m_szUrl.find("?")==string::npos)
ss << "?";
else
ss << "&"; for(it=m_mapParamUrl.begin(); it!=m_mapParamUrl.end(); it++)
{
ss << UriEncode(it->first) << "=" << UriEncode(it->second) << "&";
}
}
ss << " HTTP/1.1"; // header
if(m_bPost)
{
AddHeader("Content-Length", GetPostLength());
m_szBoundary = "-------------------------"+GetRandomString(13); if(m_bUploadFile)
{
str = "multipart/form-data; boundary="+m_szBoundary;
m_szBoundary.insert(0, "--");
AddHeader("Content-Type", str);
}
} for(it=m_mapHeaderOut.begin(); it!=m_mapHeaderOut.end(); it++)
{
ss << CRLF;
ss << it->first << ": " << it->second;
}
ss << CRLFCRLF; return Send(ss.str());
} void CHttpClient::DeleteTmpData()
{
DeleteFile(m_szFilePath.c_str());
m_mapHeaderIn.clear();
m_szFilePath = "";
m_szData = "";
} void CHttpClient::Clearup()
{
DeleteTmpData(); m_mapHeaderOut.clear();
m_mapParamUrl.clear();
m_mapParamPost.clear();
m_szFileName = "";
m_mapFiles.clear();
m_dwLenFiles = 0;
}

Connection: Keep-Alive
Connection: Close我都测试了
感觉效果一样

bool CHttpClient::RecvHeader()
{
// state line
if(!RecvLine(m_szStateLine))
return false;

int p, p1; p = m_szStateLine.find(" ");
if(p!=string::npos)
{
if(p<(int)m_szStateLine.length()-3)
m_nStateCode = atoi(m_szStateLine.substr(p+1, 3).c_str());
else
m_nStateCode = -1;
} if(m_bSaveToFile)
{
if(m_nStateCode!=HTTP_STATUS_OK)
m_bSaveToFile = false;
} string szRequest; // headers
while(1)
{
if(!RecvLine(szRequest))
return false; if(szRequest.length()<=0) break; p = szRequest.find(": ", 0);
if(p>0 && p!=string::npos && p!=szRequest.length()-2)
{
string szParam, szValue;
szParam = szRequest.substr(0, p);
szValue = szRequest.substr(p+2, -1);
m_mapHeaderIn[szParam] = szValue;
}
} // file name
p = m_mapHeaderIn["Content-Disposition"].find("filename=\"");
if(p!=string::npos)
{
p1 = m_mapHeaderIn["Content-Disposition"].find("\"", p+10);
if(p1!=string::npos)
m_szFileName = m_mapHeaderIn["Content-Disposition"].substr(p+10, p1-p-10);
} if(GetStateCode()==HTTP_STATUS_CONTINUE)
{
return RecvHeader();
} return true;
} bool CHttpClient::RecvData()
{
int  nLength = atoi(m_mapHeaderIn["Content-Length"].c_str());
bool bChunked = m_mapHeaderIn["Transfer-Encoding"]=="chunked"; HANDLE hFile = INVALID_HANDLE_VALUE;
string str;
int nLen,i; if(m_bSaveToFile)
{
MakeDirectory("temp");
m_szFilePath = "./temp/"+m_szFileName;
hFile = CreateFile(m_szFilePath.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, NULL, NULL);
if(hFile == INVALID_HANDLE_VALUE)
{
m_szFilePath = "./temp/"+GetRandomString(32);
hFile = CreateFile(m_szFilePath.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, NULL, NULL);
} if(hFile==INVALID_HANDLE_VALUE)
{
SetErr(GetWinErr());
return false;
}
} if(bChunked)
{
while(1)
{
if(!RecvLine(str))
{
CloseHandle(hFile);
return false;
}

nLen = strtol(str.c_str(), NULL, 16);
if(nLen==0)
{
RecvLine(str);
break;
}

//---------------------------------
int nNumRecv = nLen/LEN_RECV_ONCE + ((nLen%LEN_RECV_ONCE==0)?0:1); for(i=0; i<nNumRecv; i++)
{
int nRecvOnce = (i<nNumRecv-1) ? LEN_RECV_ONCE:((nLen%LEN_RECV_ONCE==0)?LEN_RECV_ONCE:nLen%LEN_RECV_ONCE); char buf[LEN_RECV_ONCE+1] = {0}; if(!Recv(buf, nRecvOnce))
{
CloseHandle(hFile);
return false;
} if(m_bSaveToFile)
{
DWORD dwWritten;
if(!WriteFile(hFile, buf, nRecvOnce, &dwWritten, NULL))
{
CloseHandle(hFile);
SetErr(GetWinErr());
return false;
}
}
else
{
if(m_szData.length()<LEN_MAX_RECV_DATA)
m_szData += buf;
}
}
//---------------------------------
RecvLine(str);
}
} else
{
nLen = nLength;

//---------------------------------
int nNumRecv = nLen/LEN_RECV_ONCE + (nLen%LEN_RECV_ONCE==0?0:1); for(i=0; i<nNumRecv; i++)
{
int nRecvOnce = (i<nNumRecv-1) ? LEN_RECV_ONCE:((nLen%LEN_RECV_ONCE==0)?LEN_RECV_ONCE:nLen%LEN_RECV_ONCE); char buf[LEN_RECV_ONCE+1] = {0}; if(!Recv(buf, nRecvOnce))
{
CloseHandle(hFile);
return false;
} if(m_bSaveToFile)
{
DWORD dwWritten;
if(!WriteFile(hFile, buf, nRecvOnce, &dwWritten, NULL))
{
CloseHandle(hFile);
SetErr(GetWinErr());
return false;
}
}
else
{
if(m_szData.length()<LEN_MAX_RECV_DATA)
m_szData += buf;
}
}
//---------------------------------
} CloseHandle(hFile); return true;
} void CHttpClient::SetPost(bool bPost)
{
m_bPost = bPost;
} bool CHttpClient::Request(const string &  szUrl, bool bSaveToFile)
{
m_bPost = false; if(szUrl.find("POST ")==0)
{
m_bPost = true;
m_szUrl = szUrl.substr(5, string::npos);
}
else if(szUrl.find("GET ")==0)
{
m_szUrl = szUrl.substr(4, string::npos);
}
else
m_szUrl = szUrl; m_bSaveToFile = bSaveToFile; DeleteTmpData(); bool b = (SendHeader() && SendData() && RecvHeader() && RecvData()); // m_mapHeaderOut.clear();

return b;
} int CHttpClient::GetStateCode()
{
return m_nStateCode;
} string CHttpClient::GetStateLine()
{
return m_szStateLine;
} void CHttpClient::SetAuthBasic(const string &  szUser, const string &  szPass)
{
string szAuth = szUser + ":" + szPass;
string szAuthEnc = string("Basic ") + Base64Encode((const unsigned char *)szAuth.c_str(), szAuth.length()); AddHeader("Authorization", szAuthEnc);
}}在接收超时的情况下 nRecv 也会返回 SOCKET_ERROR ，这个超时时间好像就是30秒。

Connection: close 不知道Close会不会有问题。
你还是用TCP测试工具(TCP&UDP测试工具)测试一下吧。用telnet也行。

只看到组HTTP的REQUEST头部分
其他部分没什么帮忙可以麻烦把CHTTPSocket发送接收的代码贴一下?多谢

#include "stdafx.h"
#include "HttpSocket.h"
#include "HttpDefine.h"#pragma comment(lib, "Ws2_32")
#pragma comment(lib, "mswsock")namespace ZvLibs
{
CHttpSocket::CHttpSocket(bool bOwner, SOCKET s)
{
m_sClient = s;
m_nTmoutConn =
m_nTmoutRecv =
m_nTmoutSend = 10;

m_bOwner = bOwner; if(bOwner)
InitSocket();
} CHttpSocket::~CHttpSocket()
{
if(m_bOwner)
{
Close();
}
} bool CHttpSocket::Connect(const string & szIp, int nPort)
{
Close(); if(!InitSocket())
return false; sockaddr_in addr; addr.sin_family=AF_INET;
addr.sin_port=htons(nPort);
addr.sin_addr.S_un.S_addr=inet_addr(szIp.c_str()); if(connect(m_sClient, (LPSOCKADDR)&addr,sizeof(addr))==0)
return true; if(WSAGetLastError()!=WSAEWOULDBLOCK)
{
SetErr(GetWinErr());
return false;
} FD_SET fd = {1, m_sClient};
TIMEVAL tv = {m_nTmoutConn,  0};   int nSel = select(0, 0, &fd, 0, &tv); if(nSel<= 0)
{
if(nSel==0)
{
WSASetLastError(WSAETIMEDOUT);
}

SetErr(GetWinErr());

return false;
} return true;
} bool CHttpSocket::Recv(char * buf, int nLen)
{
int ret = 0;
int r = 0; FD_SET fd = {1, m_sClient};
TIMEVAL tv = {m_nTmoutRecv,  0};   while(ret<nLen)
{
int nSel = select(0, &fd, NULL, NULL, &tv);
if( nSel > 0)
{
r = recv(m_sClient, buf+ret, nLen-ret, 0);
if(r<0)
{
SetErr(GetWinErr());
return false;
}
else if(r==0)
{
SetErr("connection closed");
return false;
}
ret += r;
continue;
} if(nSel == 0)
{
SetErr("recv data timeout");
}
else
{
SetErr(GetWinErr());
}
return false;
} return true;
} bool CHttpSocket::Recv(string & szStr, int nLen)
{
szStr.resize(nLen); return Recv(&szStr[0], nLen);
} bool CHttpSocket::Send(const char * szData, int nLen)
{
int ret = 0;
int r = 0;
if(nLen<=0)
nLen = strlen(szData); FD_SET fd = {1, m_sClient};
TIMEVAL tv = {m_nTmoutSend, 0};   while(ret < nLen)
{
int nSel = select(0, NULL, &fd, NULL, &tv);
if(nSel>0)
{
r = send(m_sClient, szData+ret, nLen-ret, 0);
if(r<0)
{
SetErr(GetWinErr());
return false;
} ret += r;
continue;
} if(nSel == 0)
{
SetErr("send data timeout");
}
else
{
SetErr(GetWinErr());
}
return false;
} return true;
} bool CHttpSocket::RecvLine(string & szLine, int * pLen)
{
int ret;
char c;
int nLen = 0;
int nExtra = 0;
szLine = ""; while(ret=Recv(&c, 1))
{
switch(c)
{
case CR:
nExtra++;
break;
case LF:
nExtra++;
if(pLen!=NULL)
{
* pLen = nExtra + nLen;
}
return true;
default:
nLen++;
szLine += c;
} if(nLen>LEN_MAX_HEADER_LINE)
{
SetErr("recv line length too long");
return false;
}
} return false;
} bool CHttpSocket::Send(const string & szData)
{
return Send(szData.c_str(), szData.length());
} void CHttpSocket::Close()
{
shutdown(m_sClient, SD_BOTH);
closesocket(m_sClient);
m_sClient = INVALID_SOCKET;
} bool CHttpSocket::InitSocket()
{
if(m_sClient==INVALID_SOCKET)
{
m_sClient = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if(m_sClient==INVALID_SOCKET)
{
SetErr(GetWinErr());
return false;
}
} unsigned long ul;
ul = 1;
ioctlsocket(m_sClient, FIONBIO, (unsigned long*)&ul);
return true;
}}

搞定,是Chunked编码引起的问题
加了个Chunked编码协议栈就行了特别感谢zhengv,也感谢其他给予关注的朋友

chunk实现步骤://while(1){
    1,取chunk头,分析出chunk的body长度(可以参考zhangv代码中RecvLine代码,实现类似)
    2,判断长度是否为0,是则表示已经接收完(当然后面可能还有会footer,一般可以忽略) //  break;
    3,根据上面取得的长度,接收chunk 的body(实现类似zhangv的Recv)
//}PS: 处理中多注意一下CRLF

调试易

为什么WININET 和 SOCKET 实现网页下载速度相差很大

解决方案 »