爬虫可以登陆下一个文件,但是不能连续下文件,大家来看看什么原因?

Question

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;using System.Configuration;
using System.IO;
using System.Net;using System.Threading;namespace WindowsApplication1
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }        private const String Session_ID = "ASPSESSIONIDQACSSDTT=PEJGMNEABEPMFKJDFPDMAOIK";
        protected HttpWebRequest request;
        protected String newCookies;
        protected WebClient client = new WebClient();
        protected long T1 = System.DateTime.Now.Ticks;
        protected System.TimeSpan TS;         private void Form1_Load(object sender, EventArgs e)
        {
            this.progressBar1.Maximum = 7;
            this.progressBar1.Minimum = 0;            request = CreateRequest("http://club.joyes.com/User/Login.asp", "http://club.joyes.com/User/Login.asp", "POST");            request.Headers.Add(HttpRequestHeader.Cookie, Session_ID);
            request.Timeout = request.ReadWriteTimeout = 99999999;
            request.AddRange(0);
            request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
            Stream stream = request.GetRequestStream();
            Byte[] bytes = TranslateByte("user=dullwolf&pass=123456&CkiExp=-1&submitflag=ddddls-%2B%2B%2B&regbtn.x=50&regbtn.y=31");
            stream.Write(bytes, 0, bytes.Length);
            stream.Flush();
            stream.Close();
            WebResponse response = request.GetResponse();            //build new Cookis use for Request           newCookies = BuildCookies(response);
            newCookies = String.IsNullOrEmpty(newCookies) ? Session_ID : newCookies + ";" + Session_ID;
                    }              private void button1_Click(object sender, EventArgs e)
        {
            //"http://sou.joyes.com/game_down.aspx?type=gam&id=55712"
            //@"E:\"+ fileName +".jar"
            List<int> idList = new List<int>();
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            //idList.Add(6379);
            //idList.Add(51859);
            //idList.Add(6381);
            //idList.Add(6380);
            //idList.Add(6382);
            //idList.Add(6383);
            //idList.Add(6384);
        
             for (int i = 0; i < 7; i++)
            {
                Application.DoEvents();
                 
                
                this.richTextBox1.Text += "正在进行：" + i.ToString() + "\n";
                try
                {
                    this.Login(@"http://sou.joyes.com/game_down.aspx?type=gam&id=" + idList[i].ToString(), @"E:\" + i.ToString() + ".jar");
                    Application.DoEvents();
                }
                catch(System.Exception Ex)
                {
                                     this.richTextBox1.Text += i.ToString() + ":" + Ex.Message + "\n" ;
                }
                
                this.progressBar1.Value = i+1;
                this.Refresh();
                Application.DoEvents();
            }
            MessageBox.Show("完成！");
                
        } 
        private void Login(string url, string fileName)
        {
                        HttpWebRequest request1 = CreateRequest(url, "http://club.joyes.com/User/login_success.asp", "GET");
            //request1.Timeout = request1.ReadWriteTimeout = 99999999;
            request1.AddRange(0);
            request1.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
            request1.Headers.Add(HttpRequestHeader.Cookie, newCookies);
            WebResponse response1;
            try
            {
                 response1 = request1.GetResponse();
                 
            }
            catch(System.Exception ex)
            {
                this.richTextBox1.Text += ex + "\n";
                return;
            }            try
            {
                client.DownloadFile(response1.Headers[HttpResponseHeader.Location], fileName);
            }
            catch(System.Exception ex)
            {
                    return;
            }                    }        private byte[] TranslateByte(String head)
        {
            return System.Text.Encoding.UTF8.GetBytes(head);
        }        private String BuildCookies(WebResponse response)
        {
            String newCookies = "";
            if (response != null)
            {
                String cookies = response.Headers[HttpResponseHeader.SetCookie];
                String[] buffer = cookies.Split(',');
                foreach (String var in buffer)
                {
                    String[] _buffer = var.Split(';');
                    newCookies = String.IsNullOrEmpty(newCookies) ? _buffer[0] : newCookies + ";" + _buffer[0];
                }
            }
            return newCookies;
        }        private HttpWebRequest CreateRequest(String url, String referer, String method)
        {
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
            request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-ms-application, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-ms-xbap, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/ag-plugin, */*";
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-us");
            request.Referer = referer;
            request.ContentType = "application/x-www-form-urlencoded";
            request.AllowAutoRedirect = false;
            request.Method = method;
            return request;
        }
      
    }
}

Answer 1

czhenq(庄周梦蝶(今天你相亲了没?)) ( ) 信誉：97    Blog   加为好友  2007-6-27 18:15:28  得分: 0  
   
人品问题，我这运行良好 
－－－－－－－－－－－－－－－哈哈，严重同意

Answer 2

先狂晕一下先 居然你下我工作过的joyes.com............马上 测试来看看:)

Answer 3

//测试了下 没有问题.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using System.Data.SqlClient;
using System.Collections;
using System.IO;
using System.Net;using System.Threading;namespace WindowsApplication2
{
    //delegate void myDelegate();
    delegate void SetTextCallback(string text);    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }        private const String Session_ID = "ASPSESSIONIDQACSSDTT=PEJGMNEABEPMFKJDFPDMAOIK";
        protected HttpWebRequest request;
        protected String newCookies;
        protected WebClient client = new WebClient();
        protected long T1 = System.DateTime.Now.Ticks;
        protected System.TimeSpan TS;        //private event myDelegate myEvent;
        //private Thread thd;        private string url = string.Empty;
        private string fileName = string.Empty;
        private void SetText(string text)
        {            
            // InvokeRequired required compares the thread ID of the
            // calling thread to the thread ID of the creating thread.
            // If these threads are different, it returns true.
            if (this.richTextBox1.InvokeRequired)
            {
                SetTextCallback d = new SetTextCallback(SetText);
                this.Invoke(d, new object[] { text });
            }
            else
            {
                this.richTextBox1.Text += text;
            }
        }        private void Form1_Load(object sender, EventArgs e)
        {
            this.progressBar1.Maximum = 7;
            this.progressBar1.Minimum = 0;            request = CreateRequest("http://club.joyes.com/User/Login.asp", "http://club.joyes.com/User/Login.asp", "POST");            request.Headers.Add(HttpRequestHeader.Cookie, Session_ID);
            request.Timeout = request.ReadWriteTimeout = 99999999;
            request.AddRange(0);
            request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
            Stream stream = request.GetRequestStream();
            Byte[] bytes = TranslateByte("user=dullwolf&pass=123456&CkiExp=-1&submitflag=ddddls-%2B%2B%2B&regbtn.x=50&regbtn.y=31");
            stream.Write(bytes, 0, bytes.Length);
            stream.Flush();
            stream.Close();
            WebResponse response = request.GetResponse();            //build new Cookis use for Request            newCookies = BuildCookies(response);
            newCookies = String.IsNullOrEmpty(newCookies) ? Session_ID : newCookies + ";" + Session_ID;
        }        private void button1_Click(object sender, EventArgs e)
        {
            //"http://sou.joyes.com/game_down.aspx?type=gam&id=55712"
            //@"E:\"+ fileName +".jar"
            List<int> idList = new List<int>();
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            idList.Add(55712);
            //idList.Add(6379);
            //idList.Add(51859);
            //idList.Add(6381);
            //idList.Add(6380);
            //idList.Add(6382);
            //idList.Add(6383);
            //idList.Add(6384);            for (int i = 0; i < 7; i++)
            {
                //Application.DoEvents();
                SetText("正在进行：" + i.ToString() + "\n");
                try
                {
                    //url = @"http://sou.joyes.com/game_down.aspx?type=gam&id=" + idList[i].ToString();
                    //fileName = @"E:\TestDown\" + i.ToString() + ".jar";
                    //thd = new Thread(new ThreadStart(Login));
                   // thd.IsBackground = true;                    
                    //thd.Start();
                    //this.Login();
                    this.Login(@"http://sou.joyes.com/game_down.aspx?type=gam&id=" + idList[i].ToString(), @"E:\" + i.ToString() + ".jar");
                    Application.DoEvents();
                }
                catch (System.Exception Ex)
                {
                    SetText(i.ToString() + ":" + Ex.Message + "\n");
                }                this.progressBar1.Value = i + 1;
                this.Refresh();
                //Application.DoEvents();
            }
            MessageBox.Show("完成！");        }
        private void Login(string url, string fileName)
        {
            HttpWebRequest request1 = CreateRequest(url, "http://club.joyes.com/User/login_success.asp", "GET");
            //request1.Timeout = request1.ReadWriteTimeout = 99999999;
            request1.AddRange(0);
            request1.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
            request1.Headers.Add(HttpRequestHeader.Cookie, newCookies);
            WebResponse response1;
            try
            {
                response1 = request1.GetResponse();                            }
            catch (System.Exception ex)
            {
                SetText(ex.Message + "\n");
                return;
            }            try
            {
                client.DownloadFile(response1.Headers[HttpResponseHeader.Location], fileName);                
            }
            catch (System.Exception ex)
            {
                SetText(ex.Message + "\n");
                return;
            }            response1.Close();        }        private byte[] TranslateByte(String head)
        {
            return System.Text.Encoding.UTF8.GetBytes(head);
        }        private String BuildCookies(WebResponse response)
        {
            String newCookies = "";
            if (response != null)
            {
                String cookies = response.Headers[HttpResponseHeader.SetCookie];
                String[] buffer = cookies.Split(',');
                foreach (String var in buffer)
                {
                    String[] _buffer = var.Split(';');
                    newCookies = String.IsNullOrEmpty(newCookies) ? _buffer[0] : newCookies + ";" + _buffer[0];
                }
            }
            return newCookies;
        }        private HttpWebRequest CreateRequest(String url, String referer, String method)
        {
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
            request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-ms-application, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-ms-xbap, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/ag-plugin, */*";
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-us");
            request.Referer = referer;
            request.ContentType = "application/x-www-form-urlencoded";
            request.AllowAutoRedirect = false;
            request.Method = method;
            return request;
        }    
      }
}

Answer 4

楼主可以 看看 关于Invoke的MSDN帮助
http://blog.csdn.net/hertcloud/archive/2007/04/07/1556112.aspx
还有login的最后个人 觉得应该 response1.Close(); 多线程的方法 试验中.

Answer 5

俗话说: 杀鸡焉用牛刀.PHP花一支烟的功夫能搞定的问题何必要 C# 一烛香的时间来调试呢.PHP代码强势出击:<?// 这段代码是运行在命令行模式的.
$ua='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)';
$downlod_path="F:\\";     // 下载文件保存目录
$cookie_file="cookie.dat";
$loginpost='user=dullwolf&pass=123456&CkiExp=-1&submitflag=ddddls-%2B%2B%2B&regbtn.x=50&regbtn.y=31';
$login_url='http://club.joyes.com/User/Login.asp';
$jar_url="http://sou.joyes.com/game_down.aspx?type=gam&id=";
$idlist=array(55397,55409,55387);   //要下多少文件这里自己加function curl_gorush($id=''){
global $cookie_file,$ua,$downlod_path,$loginpost,$login_url,$jar_url;
echo   $id==='' ? "正在登陆...\n" : "正在下载文件\tid=$id\n";
$ch = curl_init();
$url= $id===''?$login_url:$jar_url.$id;
curl_setopt($ch, CURLOPT_URL,$url );
curl_setopt($ch, CURLOPT_REFERER, $url); 
curl_setopt($ch, CURLOPT_USERAGENT,$ua);
curl_setopt($ch, CURLOPT_COOKIEFILE,$cookie_file);    
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file);    
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);  if ($id===''){
curl_setopt($ch, CURLOPT_POST, 1);      
curl_setopt($ch, CURLOPT_POSTFIELDS,$loginpost);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);   
$content=curl_exec($ch);
curl_close ($ch);
return !empty($content) && strpos($content,"登陆成功")!==false;
}else{
$fp=fopen($downlod_path.$id.'.jar','wb');
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_exec($ch);
curl_close ($ch);
fclose($fp);
}
}
if (!extension_loaded('curl')) dl("php_curl." . PHP_SHLIB_SUFFIX);
curl_gorush()?array_walk($idlist,'curl_gorush'):die("登陆失败\n");
?>
复制上面的代码保存到 x:\xxx\xxxxxx.php然后建立一个批处理文件 run.bat (run.bat放哪个目录都可以)
内容如下: 假设你的php安装在D盘d:\your_php.exe_path\php.exe x:\xxx\xxxxxx.php
pause
双击 run.bat 运行, 哈哈,下载的那三个 jar文件乖乖的躺在 F:\了

Answer 6

上面的代码共38行,其实 referer和 useragent欺骗是完全不要的,对方服务器根本没有检查这个只需要在代码中加多1行,就可以支持通过代理服务器抓取.
只需要在代码中加多不超过10行,就可以支持通过代理服务器抓取,并每次抓取都更换不同的代理
再花一泡尿的功夫就可以支持多线程下载. 不过我看这里每个文件才几百kb,也没有必要搞多线程
去下了,线程间调度起来的那些时间都可能下了好几个了.做这种事情还是 3p 的强项啊 (perl,php,python), 估计拿perl写十来行代码都可以搞定.
用c#那就是高射炮打蚊子啦对了,要想搞代理服务器列表的话,来这里,包你满载而归
http://www.cnblogs.com/ashun/archive/2007/06/01/python_proxy_checker.html太热,实在睡不着啊. 又停水.

Answer 7

Go_Rush(我的技术博客http://ashun.cnblogs.com/) ( ) 
如果PHP的话,我想楼主 也不用到这儿 问了  支持狼哥~~~

Answer 8

Go_Rush的代码可以用，不错，短，平，快。

Answer 9

下面这个解决方案需要三个文件1. 批处理文件。 run.bat 代码如下, 注意，只有两行，千万保存的时候不要折行c:\curl.exe --silent --output nul --data "user=dullwolf&pass=123456&CkiExp=-1" --location --cookie-jar %temp%\cookie.dat http://club.joyes.com/User/Login.asp
for /f %%i in (idlist.txt) do c:\curl.exe --output e:\%%i.jar --location --cookie %temp%\cookie.dat "http://sou.joyes.com/game_down.aspx?type=gam&id=%%i"2. curl.exe 文件 
这个文件在苹果机系统里面是操作系统内置的， windows平台 只能自己去下载 
下载地址 http://curl.haxx.se/download/curl-7.16.3-win32-nossl-sspi.zip   
文件大小只有 256kb， 下载完后加解缩，里面只有一个 curl.exe和一些文档
把curl.exe保存到c:\ (如果你保存到其他目录，请修改上面批处理文件的路径为你保存的路径)3. 自己用记事本建立一个文件 idlist.txt
里面存放你要抓取的 id号码,一行一个
我调试的时候 idlist.txt 的内容如下:
55397
55409
55387
ok 搞定后，双击run.bat 自动登录，并下载

Answer 10

友情提示:idlist.txt要和run.bat放在通一个目录.另外，如果你要下载的id是一个区段的话，就没有必要用idlist.txt一个个去加了比如，要下载 id 为 1000,1001,1002,1003,1004..........4000
可以不用idlist.txt
只需要把 run.bat第二行改为c:\curl.exe --output e:\#1.jar --location --cookie %temp%\cookie.dat "http://sou.joyes.com/game_down.aspx?type=gam&id=[1000-4000]"有些网站需要检查 user-agent 和 referer 的那就在 curl.exe 添加他们， 如：
c:\curl.exe --user-agent "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT;)" --referer "http://www.abc.com/123.htm"
把 run.bat 升级了一下
@echo off
set savepath=e:\
set username=dullwolf
set pass=123456
set runcmd=c:\curl.exe
set idfile=idlist.txt%runcmd% --silent --data "user=%username%&pass=%pass%&CkiExp=-1" --location --cookie-jar %temp%\cookie.dat http://club.joyes.com/User/Login.asp | find "登陆成功" if %errorlevel%==1 (goto loginerror) else goto download:download
for /f %%i in (%idfile%) do %runcmd% --output %savepath%%%i.jar --location --cookie %temp%\cookie.dat "http://sou.joyes.com/game_down.aspx?type=gam&id=%%i"
exit:loginerror
echo 登陆失败
pause