多线程httpwebrequest异步获取网页代码，采了几页就没响应了

Question

本帖最后由 zhangsansecond 于 2013-07-21 21:45:37 编辑

Answer 1

//分派工作
        private void DispatchWork()
        {
            if (_stop) //判断是否中止下载
            {
                return;
            }
            for (int i = 0; i < _reqCount; i++)
            {
                lock (dlocker)
                {
                    if (!_reqsBusy[i]) //判断此编号的工作实例是否空闲
                    {
                        RequestResource(i); //让此工作实例请求资源
                    }
                }
            }
        }        //发送请求
        private void RequestResource(int index)
        {
            string record_id;
            string url = "";
            try
            {
                lock (_locker)
                {
                    if (unload.Count <= 0) //判断是否还有未下载的URL
                    {
                        return;
                    }
                    _reqsBusy[index] = true;
                    record_id = unload.First().Key; //取出第一个未下载的URL
                    url = unload.First().Value;
                    //_urlsLoaded.Add(record_id,url); //把该URL加入到已下载里
                    unload.Remove(record_id); //把该URL从未下载中移除
                }                httprequest[index] = (HttpWebRequest)WebRequest.Create(url);
                httprequest[index].Method = _method; //请求方法
                //httprequest[index].Accept = _accept; //接受的内容
                //httprequest[index].UserAgent = _userAgent; //用户代理
                httprequest[index].ReadWriteTimeout = 3000;
                httprequest[index].KeepAlive = true;
                RequestState rs = new RequestState(httprequest[index], url, record_id, index); //回调方法的参数
                var result = httprequest[index].BeginGetResponse(new AsyncCallback(ReceivedResource), rs); //异步请求
                ThreadPool.RegisterWaitForSingleObject(result.AsyncWaitHandle, //注册超时处理方法
                        TimeoutCallback, rs, _maxTime, true);
            }
            catch (WebException we)
            {
                MessageBox.Show("RequestResource " + we.Message + url + we.Status);
            }
        }        //
        private void TimeoutCallback(object state, bool timedOut)
        {
            if (timedOut) //判断是否是超时
            {
                RequestState rs = state as RequestState;
                if (rs != null)
                {
                    //if (!unload.Keys.Contains<string>(rs.Record_id))
                        //unload.Add(rs.Record_id, rs.Url);
                    if (rs.ResStream != null)
                        rs.ResStream.Close();
                    rs.Req.Abort(); //撤销请求
                }
                _reqsBusy[rs.Index] = false; //重置工作状态
                DispatchWork(); //分配新任务
            }
        }        //
        private void ReceivedResource(IAsyncResult ar)
        {
            RequestState rs = (RequestState)ar.AsyncState; //得到请求时传入的参数
            HttpWebRequest req = rs.Req;
            string url = rs.Url;
            try
            {
                HttpWebResponse res = (HttpWebResponse)req.EndGetResponse(ar); //获取响应
                if (_stop) //判断是否中止下载
                {
                    res.Close();
                    req.Abort();
                    return;
                }
                if (res != null && res.StatusCode == HttpStatusCode.OK) //判断是否成功获取响应
                {
                    Stream resStream = res.GetResponseStream(); //得到资源流
                    rs.ResStream = resStream;
                    var result = resStream.BeginRead(rs.Data, 0, rs.BufferSize, //异步请求读取数据
                        new AsyncCallback(ReceivedData), rs);
                }
                else //响应失败
                {
                    if (!unload.Keys.Contains<string>(rs.Record_id))
                        unload.Add(rs.Record_id, rs.Url);
                    rs.Req.Abort();
                    res.Close();
                    _reqsBusy[rs.Index] = false; //重置工作状态
                    DispatchWork(); //分配新任务
                }
            }
            catch (WebException we)
            {
                if(!unload.Keys.Contains<string>(rs.Record_id))
                    unload.Add(rs.Record_id, rs.Url);
                if (rs != null)
                {
                    rs.Req.Abort();
                    if (rs.ResStream != null)
                        rs.ResStream.Close();
                    
                }
                _reqsBusy[rs.Index] = false; //重置工作状态
                DispatchWork(); //分配新任务
                //MessageBox.Show("ReceivedResource " + we.Message + url + we.Status);
            }
        }        //
        private void ReceivedData(IAsyncResult ar)
        {
            RequestState rs = (RequestState)ar.AsyncState; //获取参数
            HttpWebRequest req = rs.Req;
            Stream resStream = rs.ResStream;
            string url = rs.Url;
            string record_id = rs.Record_id;
            string insertSQl;
            string html = null;
            string r = "";
            int index = rs.Index;
            int read = 0;            try
            {                read = resStream.EndRead(ar); //获得数据读取结果
                if (_stop)//判断是否中止下载
                {
                    rs.ResStream.Close();
                    req.Abort();
                    return;
                }
                if (read > 0)
                {
                    MemoryStream ms = new MemoryStream(rs.Data, 0, read); //利用获得的数据创建内存流
                    StreamReader reader = new StreamReader(ms, _encoding);
                    string str = reader.ReadToEnd(); //读取所有字符
                    rs.Html.Append(str); // 添加到之前的末尾
                    var result = resStream.BeginRead(rs.Data, 0, rs.BufferSize, //再次异步请求读取数据
                        new AsyncCallback(ReceivedData), rs);
                    return;
                }
                html = rs.Html.ToString();
                //SaveContents(html, url); //保存到本地
                r = matchField(fieldExpDetail, html);
                DelListBoxDelegate dlbd = new DelListBoxDelegate(DelListBoxDelegateMethod);
                this.Invoke(dlbd, "record_id = " + record_id + " url=" + url);
                ListBoxDelegate lbd = new ListBoxDelegate(ListBoxDelegateMethod);
                this.Invoke(lbd," record_id = " + record_id + " " + r);
                //matchResult.Add(record_id, r);
                lock (dataLocker)
                {                    insertSQl = String.Format("update webdata set {0} where record_id={1}", r, record_id);
                    OleDbConnection conn = DB_helper.GetConnection(strCon_access + newfile);
                    if (conn.State != ConnectionState.Open)
                        conn.Open();                    DB_helper.ChangeData(conn, insertSQl);
                
                _reqsBusy[index] = false; //重置工作状态
                req.Abort();
                rs.ResStream.Close();
                DispatchWork(); //分配新任务
                }
            }
            catch (WebException we)
            {
                MessageBox.Show("ReceivedData Web " + we.Message + url + we.Status);
            }        }

Answer 2

我觉得这个要去问你写的线程管理了。ThreadPool  还有线程池，检查代码吧

Answer 3

应该是像你说的“某个请求没有响应的原因导致后面的请求都终止了”，但是我的异步请求都设了超时，ThreadPool.RegisterWaitForSingleObject(result.AsyncWaitHandle, //注册超时处理方法
                        TimeoutCallback, rs, _maxTime, true);，为什么超时了却无反应呢，还有我怎么记录程序运行的日志啊？

Answer 4

                ServicePointManager.DefaultConnectionLimit = 500;
                ServicePointManager.SetTcpKeepAlive(true, _keepLiveTime, _intervalTime);这个方法好像可以关闭无响应的连接，但是有时在stream.EndRead(ar)时会报这个错：Unable to read data from the transport connection: An existing connection was forcibly closed by the remote host。

Answer 5

一边读一边删
for (int i = 0; i < _reqCount; i++) 比较典型的一个误区，这是遍历不完，你拿个数组试一下就知道了
应该是for (int i = _reqCount - 1; i>=0; i--)
你直接用Queue<T>或者LinkedList<T>就行了

Answer 6

不是用的for，而是while判断list是否为空，然后取first（），再remove