asp.net获取网页源码问题

Question

 protected void Button1_Click(object sender, EventArgs e)
    {
        TextBox1.Text = DownLoadHtml("http://us.my.alibaba.com/product/category_browse_iframe.htm?iframe_delete=true&fromWhere=postStep1&postCategoryVersion=1311636484503&catLang=en_us");
    }
    //WebClient方法获取
    public static string DownLoadHtml(string url)
    {
        string output = "";
        Encoding encode = Encoding.Default;
        WebClient webclient = new WebClient();
        try
        {
            webclient.Headers.Add("Referer", url);
            byte[] buff = webclient.DownloadData(url);
            output = encode.GetString(buff);
        }
        catch
        {
        }
        return output;
    }为什么获取出来的内容和我右键源码看的不一样了

Answer 1

http://blog.csdn.net/kkun/article/details/5835853

Answer 2

编码问题，试下GB2312 或 UTF-8

--reply by CSDN Study V1.0.0.3 (starts_2000)

  

Answer 3

使用HttpWebRequest吧，设置UserAgent属性，可以实现与浏览器完全相同的代码

Answer 4

saveSourceCode这个是什么了，提示错误

Answer 5

UserAgent属性是什么了，或者贴个代码出来可以不

Answer 6

本帖最后由 net_lover 于 2011-07-26 12:46:21 编辑

  

Answer 7

你必须HttpWebRequest进行登录，使用CookieContainer记录登录信息。这样的功能网上有例子http://www.google.com/search?q=C%23+HttpWebRequest+%E7%99%BB%E5%BD%95&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:zh-CN:official&client=firefox-a

Answer 8

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;
using System.Text.RegularExpressions;namespace WikiPageCreater.Common
{
    public class PageHelper
    {
        /// <summary>
        /// 根据 url 获取网页编码
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetEncoding(string url)
        {
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(url);
                request.Timeout = 20000;
                request.AllowAutoRedirect = false;                response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
                {
                    if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                        reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));
                    else
                        reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII);                    string html = reader.ReadToEnd();                    Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)");
                    if (reg_charset.IsMatch(html))
                    {
                        return reg_charset.Match(html).Groups["charset"].Value;
                    }
                    else if (response.CharacterSet != string.Empty)
                    {
                        return response.CharacterSet;
                    }
                    else
                        return Encoding.Default.BodyName;
                }
            }
            catch
            {
            }
            finally
            {                if (response != null)
                {
                    response.Close();
                    response = null;
                }
                if (reader != null)
                    reader.Close();                if (request != null)
                    request = null;            }            return Encoding.Default.BodyName;
        }        /// <summary>
        /// 根据 url 和 encoding 获取当前url页面的 html 源代码        
       /// </summary>
        /// <param name="url"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public static string GetHtml(string url, Encoding encoding)
        {
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(url);
                request.Timeout = 20000;
                request.AllowAutoRedirect = false;                response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
                {
                    if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                        reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding);
                    else
                        reader = new StreamReader(response.GetResponseStream(), encoding);
                    string html = reader.ReadToEnd();                    return html;
                }
            }
            catch
            {
            }
            finally
            {                if (response != null)
                {
                    response.Close();
                    response = null;
                }
                if (reader != null)
                    reader.Close();                if (request != null)
                    request = null;            }            return string.Empty;
        }
    }
}

Answer 9

  WebRequest wr = WebRequest.Create("http://www.baidu.com");//发出对url的请求， 一个抽象类
        WebResponse wp = wr.GetResponse();
        StreamReader sr=new StreamReader (wp.GetResponseStream());
        string data=sr.ReadToEnd();
        sr.Close();
        

Answer 10

自己用fiddler查看下登录过程发送返回的数据
然后用request构造一下，自己琢磨一下早就可以手工了吧