诚心求教关于抓取网页文字内容及图片的问题

C#实现web信息自动抓取
背景

随着Internet的普及，网络信息正以极高的速度增长，在这么多数据中找到自己需要的信息是一件很繁琐的事情，找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。
所谓Internet信息抓取程序，就是程序会按照用户的关键词或关键网站来收集相应的信息，并提供给用户想要的信息格式。

信息量的增加会带来信息网站发布人员工作量的剧增，为实现信息发布系统实现信息自
动发布、减少工作人员工作量、即时跟踪最新信息，就需要自动信息提供程序，因此Internet信息抓取程序应运而生。

目标

实现自定义网站信息分类抓取，存入本地数据库、生成静态页面或其它用户定义的信息结构，并下载与信息相关的多媒体文件。

开发

l         目标站点结构分析
本步骤是准确抓取信息个关键。
首先要选择更新频率高的页面做为抓取地址，然后分析要抓取内容页面url特点。
然后分析要抓取信息页面的元素特性，比如标题位置，内容位置等，得到定位标记点。
将以上信息写成自己的配置文件或存到数据库中。
每个网站都需要分析，写出单独的配置文件，供抓取程序使用。

l         信息提取
根据配置文件取得要抓取页面url，使用HttpWebRequest类获取内容：
//获取http页面函数
        public string Get_Http(string a_strUrl,int timeout)
        {
            string strResult ;

            try
            {
HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;
                myReq.Timeout = timeout;
                HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();

                Stream myStream = HttpWResp.GetResponseStream () ;

                StreamReader sr = new StreamReader(myStream , Encoding.Default);
                StringBuilder strBuilder = new StringBuilder();
                while (-1 != sr.Peek())
                {
                    strBuilder.Append(sr.ReadLine()+"\r\n");
                }

                strResult = strBuilder.ToString();
            }
            catch(Exception exp)
            {
                strResult = "错误：" + exp.Message ;
            }

            return strResult ;

        }

获取页面内容后，分析页面中连接地址取到要抓取的url：
//处理页面标题和链接
        public string SniffWebUrl( string urlStr,string blockB,string blockE )
        {
            string urlch1 = "";
            string urlch2 = "";
            int end_n1 = 0;
            int end_nums = 0;
            int end_nums1 = 0;
            int end_nums2 = 0;
            int end_nums3     = 0;
            string reUTStr = "";
            string reTitle = "";
            string ret = "";
            try
            {
                int pos01 = urlStr.IndexOf( "." );
                int pos02 = urlStr.LastIndexOf( "/" );
                if( pos01 < 0 )
                {
                    return "";
                }
                if( pos02 < 0 )
                {
                    return "";
                }
                int pos03 = urlStr.IndexOf( "/",pos01 );
                if ( pos03 < 0 )
                {
                    urlch1 = urlStr;
                    urlch2 = urlStr;
                }
                else
                {
                    urlch1 = urlStr.Substring( 0,pos03 );
                    urlch2 = urlStr.Substring( 0,pos02 );
                }

                string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);

                int pos1 = tmpAllStr.IndexOf( blockB );
                int pos2 = tmpAllStr.IndexOf( blockE,pos1 + blockB.Length );
                if ( pos1>0 && pos2>0 && pos2>pos1 )
                {
                    ret = tmpAllStr.Substring( pos1 + blockB.Length,pos2 - pos1 - blockB.Length );
                    ret = ret.Substring( ret.IndexOf( "<" ));
                    while( ret.IndexOf( "<A" ) >= 0 )
                    {
                        ret = ret.Substring( 0,ret.IndexOf( "<A" ) ) + "<a" + ret.Substring( ret.IndexOf( "<A" ) + 2 );
                    }
                    while( ret.IndexOf( "</A" ) >=0 )
                    {
                        ret = ret.Substring( 0,ret.IndexOf( "</A" ) ) + "</a" + ret.Substring( ret.IndexOf( "</A" ) + 3 );
                    }
                    while( ret.IndexOf( "Href=" ) >=0 )
                    {
                        ret = ret.Substring( 0,ret.IndexOf( "Href=" )) + "href=" + ret.Substring( ret.IndexOf( "Href=" ) + 5 );
                    }
                    while( ret.IndexOf( "HREF=" ) >=0 )
                    {
                        ret = ret.Substring( 0,ret.IndexOf( "HREF=" )) + "href=" + ret.Substring( ret.IndexOf( "HREF=" ) + 5 );
                    }
                    while( ret.IndexOf( "href='" ) >=0 )
                    {
                        ret = ret.Substring( 0,ret.IndexOf( "href='" )) + "href=\"" + ret.Substring( ret.IndexOf( "href='" ) + 6 );
                    }
                }
                tmpAllStr = ret;
                int begin_nums = tmpAllStr.IndexOf( "href=" );

                while ( begin_nums >= 0 )
                {
                    string tmpStrA = "";
                    string tmpStrB = tmpAllStr.Substring( begin_nums + 5,1 );
                    if ( tmpStrB == "\"" )
                    {
                        end_n1 = begin_nums + 6;
                        if ( ( end_n1 + 1 ) > tmpAllStr.Length )
                        {
                            return "";
                        }
                        tmpStrA = tmpAllStr.Substring( begin_nums+6,1 );
                    }
                    else
                    {
                        end_n1 = begin_nums + 5;
                        tmpStrA = tmpStrB;
                    }

                    if ( tmpStrA == "#" )
                    {
                        tmpAllStr = tmpAllStr.Substring( end_n1 );
                        begin_nums = tmpAllStr.IndexOf( "href=" );
                    }
                    else
                    {
                        end_nums1 = tmpAllStr.IndexOf( " ",end_n1 );
                        end_nums2 = tmpAllStr.IndexOf( ">",end_n1 );
                        end_nums3 = tmpAllStr.IndexOf( "</a",end_nums2 );

                        if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 ) )
                        {
                            reTitle = tmpAllStr.Substring( end_nums2 + 1,end_nums3 - end_nums2 - 1 );

                            if ( end_nums1 > end_nums2 )
                            {
                                end_nums = end_nums2;
                            }
                            else
                            {
                                if ( end_nums1 < 0 )
                                {
                                    end_nums = end_nums2;
                                }
                                else
                                {
                                    end_nums = end_nums1;
                                }
                            }
                            string str4 = tmpAllStr.Substring( end_nums-1, end_nums - end_nums + 1 );

                            if ( str4 =="\"" || str4 == "'" )
                            {
                                end_nums = end_nums - 1;
                            }
                            string sTotalOne = tmpAllStr.Substring( end_n1,end_nums - end_n1 );

                            if ( sTotalOne.IndexOf( "http://" ) <0 )
                            {
                                if ( sTotalOne.IndexOf( "/" ) == 0 )
                                {
                                    sTotalOne = urlch1 + sTotalOne;
                                }
                                else
                                {
                                    int linshiIntNum = 0;
                                    int flags = 0;
                                    string urlChange = urlStr;;
                                    while( sTotalOne.IndexOf( "../" ) >= 0 )
                                    {
                                        sTotalOne = sTotalOne.Substring( sTotalOne.IndexOf( "../" ) + 3 );
                                        linshiIntNum = linshiIntNum + 1;
                                        flags = flags +1;
                                    }
                                    while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )
                                    {
                                        urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );
                                        linshiIntNum = linshiIntNum - 1;
                                    }
                                    if ( flags == 0 )
                                    {
                                        sTotalOne = urlch2 + "/" + sTotalOne;
                                    }
                                    else
                                    {
                                        sTotalOne = urlChange + "/" + sTotalOne;
                                    }
                                }
                            }
                            reUTStr = reUTStr + new PublicFun().RemoveHtmlCode( reTitle ) + sTotalOne;

                            tmpAllStr = tmpAllStr.Substring( end_nums3 + 4 );
                            begin_nums = tmpAllStr.IndexOf( "href=" );
                        }
                        else
                        {
                            begin_nums = -1;
                        }
                    }
                }
                return reUTStr;
            }
            catch( Exception e)
            {
                return "";
            }
        }

得到要抓取内容的url后，处理该页面：
//获取链接内容并分类处理
        public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root )
        {
            string tmpAllStr = "";
            string dfStrB = "";
            string dfStrE = "";
            string rePicStr = "";//图片返回路径
            string reContentStr = "";
            string picHtml = "images"; //本地图片路径

            string urlch1 ="";
            string urlch2 ="";
            int pos1 = gatherUrl.IndexOf( "." );
            int pos2 = gatherUrl.LastIndexOf( "/" );
            if( pos1 < 0 )
            {
                return "";
            }
            if( pos2 < 0 )
            {
                return "";
            }
            int pos3 = gatherUrl.IndexOf( "/",pos1 );
            if ( pos3 < 0 )
            {
                urlch1 = gatherUrl;
                urlch2 = gatherUrl;
            }
            else
            {
                urlch1 = gatherUrl.Substring( 0,pos3 );
                urlch2 = gatherUrl.Substring( 0,pos2 );
            }

            tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );
            //取稿源
            string docFromStr = "";
            if ( dfStrB != "" && dfStrE != "" )
            {
                if ( tmpAllStr != "" )
                {
                    int b_docF = tmpAllStr.IndexOf( dfStrB );
                    if ( b_docF > 0 )
                    {
                        int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF + dfStrB.Length );
                        if ( e_docF > 0 && e_docF > b_docF && e_docF - b_docF < 20 )
                        {
                            docFromStr = tmpAllStr.Substring( b_docF + dfStrB.Length, e_docF - b_docF - dfStrB.Length );
                        }
                    }
                }
            }
            //取内容
            if ( tmpAllStr != "" )
            {
                int begin_strnum = tmpAllStr.IndexOf( b_Content );
                if ( begin_strnum < 0 )
                {
                    return "";
                }
                int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum + b_Content.Length );
                if ( end_strnum < 0 )
                {
                    return "";
                }
                string sTotalSubM = "";
                if ( end_strnum > begin_strnum )
                {
                    sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum - begin_strnum );
                }

                if ( sTotalSubM == "" )
                {
                    return "";
                }
                //过滤无用信息
                int bfnum = sTotalSubM.IndexOf( b_Filter );
                if ( bfnum > -1 )
                {
                    int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );
                    if ( efnum > -1 )
                    {
                        if ( efnum > bfnum )
                        {
                            sTotalSubM = sTotalSubM.Substring( 0,bfnum ) + sTotalSubM.Substring( efnum + e_Filter.Length );
                        }
                    }
                }

接上
//格式化图片标记

                while( sTotalSubM.IndexOf( "Src=" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "Src=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "Src=" ) + 4 );
                }
                while( sTotalSubM.IndexOf( "SRC=" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "SRC=" ) ) + "src=" + sTotalSubM.Substring( sTotalSubM.IndexOf( "SRC=" ) + 4 );
                }
                while( sTotalSubM.IndexOf( "src='" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "src='" ) ) + "src=\"" + sTotalSubM.Substring( sTotalSubM.IndexOf( "src='" ) + 5 );
                }

                //取图片地址
                int end_n12 = 0;
                int end_nums2 = 0;
                int begin_nums2 = sTotalSubM.IndexOf( "src=" );
                while( begin_nums2 >= 0 )
                {
                    String tmpStr = sTotalSubM.Substring( begin_nums2 + 4,1 );
                    if ( tmpStr == "\"" )
                    {
                        end_n12 = begin_nums2 + 5;
                    }
                    else
                    {
                        end_n12 = begin_nums2 + 4;
                    }
                    int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 );
                    int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 );
                    if ( end_nums2b < 0 )
                    {
                        break;
                    }
                    if ( end_nums2a > end_nums2b )
                    {
                        end_nums2 = end_nums2b;
                    }
                    else
                    {
                        if (end_nums2a<0)
                        {
                            end_nums2 = end_nums2b;
                        }
                        else
                        {
                            end_nums2 = end_nums2a;
                        }
                    }
                    tmpStr = sTotalSubM.Substring( end_nums2-1,1 );
                    if ( tmpStr == "\"" || tmpStr == "'" )
                    {
                        end_nums2 = end_nums2 - 1;
                    }
                    string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 - end_n12 );

                    if ( tmpPicStr.IndexOf( "http://" ) < 0 )
                    {
                        if ( tmpPicStr.IndexOf( "/" ) == 0 )
                        {
                            tmpPicStr = urlch1 + tmpPicStr;
                        }
                        else
                        {
                            int linshiIntNum = 0;
                            int flags = 0;
                            string urlChange = subUrl;
                            while( tmpPicStr.IndexOf( "../" ) >= 0 )
                            {
                                tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf("../") + 3 );
                                linshiIntNum = linshiIntNum + 1;
                                flags = flags + 1;
                            }
                            while( ( urlChange.LastIndexOf( "/" ) >= 0 ) && ( linshiIntNum >= 0 ) )
                            {
                                urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );
                                linshiIntNum = linshiIntNum - 1;
                            }
                            if ( flags == 0 )
                            {
                                tmpPicStr = urlch2 + "/" + tmpPicStr;
                            }
                            else
                            {
                                tmpPicStr = urlChange + "/" + tmpPicStr;
                            }
                        }
                    }
                    //tmpPicStr = tmpPicStr.ToLower();
                    string tmpPicStrTmp = tmpPicStr.ToLower();
                    //if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )
                    if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) > 0 )
                    {
                        rePicStr = rePicStr + "||" + tmpPicStr ;

                        int flagN2 = tmpPicStr.LastIndexOf( "/" );
                        string fileN2 = picHtml + tmpPicStr.Substring( flagN2 );
                        sTotalSubM = sTotalSubM.Substring( 0,end_nums2 ) + ">******" + fileN2 + "******<" + sTotalSubM.Substring( end_nums2 );

                        begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + fileN2.Length + 22 );
                    }
                    else
                    {
                        begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2 + 4 );
                    }
                }
                if ( rePicStr.Length > 2 )
                    rePicStr = rePicStr.Substring(2);

接上：//内容处理格式化关键标记
                while( sTotalSubM.IndexOf( "<P" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<P" ) + 2 );
                }
                while( sTotalSubM.IndexOf( "<p" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<p" ) + 2 );
                }
                while( sTotalSubM.IndexOf( "</P" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</P" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</P" ) + 3 );
                }
                while( sTotalSubM.IndexOf( "</p" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</p" ) ) + "|****|<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "</p" ) + 3 );
                }
                while( sTotalSubM.IndexOf( "<br" ) >=0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<br" ) + 3 );
                }
                while( sTotalSubM.IndexOf( "<BR" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<BR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<BR" ) + 3 );
                }
                while( sTotalSubM.IndexOf( "<Br" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<Br" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<Br" ) + 3 );
                }
                while( sTotalSubM.IndexOf( "<bR" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<bR" ) ) + "+****+<" + sTotalSubM.Substring( sTotalSubM.IndexOf( "<bR" ) + 3 );
                }

                //去除html标记
                int linshiInt1 = sTotalSubM.IndexOf( "<" );
                int linshiInt2 = sTotalSubM.IndexOf( ">" );

                if ( linshiInt2 < linshiInt1 )
                {
                    sTotalSubM = sTotalSubM.Substring( linshiInt2 + 1 );
                }
                int linshiInt11 = sTotalSubM.LastIndexOf( "<" );
                int linshiInt12 = sTotalSubM.LastIndexOf( ">" );
                if ( linshiInt12 < linshiInt11 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,linshiInt12 + 1 );
                }
                linshiInt1 = sTotalSubM.IndexOf( "<" );
                while ( linshiInt1 >= 0 )
                {
                    linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1 );
                    if ( linshiInt2 >= 0 )
                    {
                        sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 ) + sTotalSubM.Substring( linshiInt2 + 1 );
                    }
                    else
                    {
                        sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );
                    }
                    linshiInt1 = sTotalSubM.IndexOf("<");
                }

                //还原关键标记
                int linshiInt3 = 0;
                int linshiInt4 = 0;

                while( sTotalSubM.IndexOf( "+****+" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "+****+" ) ) + "<br>\n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "+****+" ) + 9 );
                }
                while( sTotalSubM.IndexOf( "|****|" ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "|****|" ) ) + "<br>\n" + sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" ) + 9 );
                }
                while( sTotalSubM.IndexOf( "******" ) >= 0 )
                {
                    linshiInt3 = sTotalSubM.IndexOf( "******" ) + 9;
                    linshiInt4 = sTotalSubM.IndexOf( "******",linshiInt3 );
                    if ( linshiInt4 >= 0 )
                    {
                        int tmpPos = sTotalSubM.IndexOf( "******" );
                        string tmpStr1 = sTotalSubM.Substring( 0,tmpPos );
                        string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 - linshiInt3 );
                        string tmpStr3 = sTotalSubM.Substring( linshiInt4 + 9 );
                        sTotalSubM = tmpStr1 + "<img src=" + tmpStr2 + ">" + tmpStr3;
                    }
                    else
                    {
                        break;
                    }
                }
                //去除内容中的标题
                if ( sTotalSubM.IndexOf( subTitle ) >= 0 )
                {
                    sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) ) + sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle ) + subTitle.Length );
                }
                reContentStr = sTotalSubM;
                //调用下载图片功能
                //下载图片到指定目录
                string[] img_Url = new PublicFun().split( rePicStr,"||" );
                for ( int i=0;i<img_Url.Length;i++ )
                {
                    if ( img_Url[i] != "" )
                    {
                        new PublicFun().Get_Img( img_Url[i],10000,root + "\\images\\" + img_Url[i].Substring( img_Url[i].LastIndexOf("/")+1 ) );
                    }
                }
            }
            return reContentStr;

        }

以上方法返回要取得的信息，包括标题内容，图片地址等。

下载页面中图片：
//下载图片
        public void Get_Img(string a_strUrl,int timeout,string filepath)
        {
            try
            {
HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;
                myReq.Timeout = timeout;
                HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();
                Stream myStream = HttpWResp.GetResponseStream () ;
                Bitmap map = new Bitmap( myStream );
                PictureBox picB = new PictureBox();
                picB.Image = (Image)map;
                string path = filepath.Substring( 0,filepath.LastIndexOf( "\\" ) );
                if (!Directory.Exists(path))
                {
                    CreateDir( path );
                }
                picB.Image.Save(filepath);
            }
            catch(Exception exp)
            {
                string ss = exp.Message;
    WriteLog( filepath.Substring(0,filepath.LastIndexOf("\\")) + "\\error.log",a_strUrl + "--" + ss + "\r\n");
            }
        }
         保存文件或入库
上面取得的信息可以按自己的要求保存。

****设计的时候没有使用url按层次循环抓取，这样定义抓取url效率更高，速度更快。

各位高手：我这个问题比较棘手
我想抓取的图片是由servlet生成的
因此无法获取图片URL
<img url="servlet?index=0" />
直接在地址栏输入 http"//xxxx/servlet?index=0 在网页能显示图片这种情况，我是不是先保存HTML到本地，再去解析图片地址？
还是HTML里能直接拿到图片呢？或者我直接访问http"//xxxx/servlet?index=0 它会返回图片流吗？

gordonzzl 呵呵在这看见你了
你说返回的数据流是指网页的数据流还是访问 http"//xxxx/servlet?index=0 返回的数据流呢因为访问http"//xxxx/servlet?index=0 会报错，估计在网页上还隐藏了别的参数

能看懂用正则处理 HTML 的代码是外星人用正则处理分析 HTML本生就是错的 W3C 为什么不用正则处理网页应该使用XML来处理分析网页

想问问。那PublicFun（）是个神马东西

调试易

诚心求教关于抓取网页文字内容及图片的问题

解决方案 »