想做一个爬虫,但是碰到带有框架的网页的就无法获取了,以下是获取网页的代码,请教如何操作 HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(a_strUrl+"&page="+i.ToString ()); myHttpWebRequest.Timeout = 20 * 1000;
myHttpWebRequest.Method = "GET"; myHttpWebRequest.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US ");
myHttpWebRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "utf-8 ");
myHttpWebRequest.UserAgent = "Mozilla/4.0(compatible;MSIE 7.0;Windows NT 5.2;.NET CLR 1.1.4322;.NET CLR 2.0.50727;InfoPath.1;.NET CLR 3.0.04506.30)";
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream responseStream = response.GetResponseStream();
StreamReader sr = new StreamReader(responseStream, Encoding.Default);
content = content + sr.ReadToEnd();
sr.Close();
responseStream.Close();
response.Close();
Thread.Sleep(3000); }
myHttpWebRequest.Method = "GET"; myHttpWebRequest.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US ");
myHttpWebRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "utf-8 ");
myHttpWebRequest.UserAgent = "Mozilla/4.0(compatible;MSIE 7.0;Windows NT 5.2;.NET CLR 1.1.4322;.NET CLR 2.0.50727;InfoPath.1;.NET CLR 3.0.04506.30)";
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream responseStream = response.GetResponseStream();
StreamReader sr = new StreamReader(responseStream, Encoding.Default);
content = content + sr.ReadToEnd();
sr.Close();
responseStream.Close();
response.Close();
Thread.Sleep(3000); }
解决方案 »
免费领取超大流量手机卡,每月29元包185G流量+100分钟通话, 中国电信官方发货