我做了个网页数据提取代码，在html页面能提取出来，但aspx页面就提取不了数据，现在想把如下网页的内容提取出来：http://video.shishicai.cn/haoma/cqssc/list/120.aspx提取内容如下：
20100625-061 1 5 3 3 6 2010-06-25 16:10
20100625-060 2 5 3 5 0 2010-06-25 16:00
20100625-059 4 7 8 0 2 2010-06-25 15:50 请问要怎样实现呢？
20100625-061 1 5 3 3 6 2010-06-25 16:10
20100625-060 2 5 3 5 0 2010-06-25 16:00
20100625-059 4 7 8 0 2 2010-06-25 15:50 请问要怎样实现呢？
");
web.downloadfile("");就可以将文件现在到本地
web.DownloadData("");返回的是网页的数据 匹配正则表达式
还有一种是获得网页内容文本 然后把它装载成dom树,看要查找的信息是在那个节点下面 但是这种应该是会耗资源的
var listIssue = [{"BonusNumberString":"4,3,2,3,8|2|5","BonusTime":"2010-06-25 19:10","IssueNumber":"20100625-079"},{"BonusNumberString":"6,8,1,4,5|2|5","BonusTime":"2010-06-25 19:00","IssueNumber":"20100625-078"},{"BonusNumberString":"2,0,3,0,3|2|5","BonusTime":"2010-06-25 18:50","IssueNumber":"20100625-077"},{"BonusNumberString":"5,3,3,6,5|2|5","BonusTime":"2010-06-25 18:40","IssueNumber":"20100625-076"},{"BonusNumberString":"8,0,3,6,1|2|5","BonusTime":"2010-06-25 18:30","IssueNumber":"20100625-075"},{"BonusNumberString":"7,6,1,2,1|2|5","BonusTime":"2010-06-25 18:20","IssueNumber":"20100625-074"},{"BonusNumberString":"5,0,8,1,1|2|5","BonusTime":"2010-06-25 18:10","IssueNumber":"20100625-073"},{"BonusNumberString":"3,6,0,9,4|2|5","BonusTime":"2010-06-25 18:00","IssueNumber":"20100625-072"},{"BonusNumberString":"9,1,3,8,1|2|5","BonusTime":"2010-06-25 17:50","IssueNumber":"20100625-071"},{"BonusNumberString":"0,6,5,1,5|2|5","BonusTime":"2010-06-25 17:40","IssueNumber":"20100625-070"},{"BonusNumberString":"6,8,9,0,5|2|5","BonusTime":"2010-06-25 17:30","IssueNumber":"20100625-069"},{"BonusNumberString":"6,2,2,7,5|2|5","BonusTime":"2010-06-25 17:20","IssueNumber":"20100625-068"},{"BonusNumberString":"4,3,7,8,5|2|5","BonusTime":"2010-06-25 17:10","IssueNumber":"20100625-067"},{"BonusNumberString":"4,3,5,0,8|2|5","BonusTime":"2010-06-25 17:00","IssueNumber":"20100625-066"},{"BonusNumberString":"8,2,3,0,7|2|5","BonusTime":"2010-06-25 16:50","IssueNumber":"20100625-065"},{"BonusNumberString":"6,3,0,7,9|2|5","BonusTime":"2010-06-25 16:40","IssueNumber":"20100625-064"},{"BonusNumberString":"1,7,1,7,8|2|5","BonusTime":"2010-06-25 16:30","IssueNumber":"20100625-063"},{"BonusNumberString":"2,9,6,2,3|2|5","BonusTime":"2010-06-25 16:20","IssueNumber":"20100625-062"},{"BonusNumberString":"1,5,3,3,6|2|5","BonusTime":"2010-06-25 16:10","IssueNumber":"20100625-061"},{"BonusNumberString":"2,5,3,5,0|2|5","BonusTime":"2010-06-25 16:00","IssueNumber":"20100625-060"},{"BonusNumberString":"4,7,8,0,2|2|5","BonusTime":"2010-06-25 15:50","IssueNumber":"20100625-059"},{"BonusNumberString":"0,2,2,2,1|2|5","BonusTime":"2010-06-25 15:40","IssueNumber":"20100625-058"},{"BonusNumberString":"5,7,8,8,8|2|5","BonusTime":"2010-06-25 15:30","IssueNumber":"20100625-057"},{"BonusNumberString":"4,1,0,1,9|2|5","BonusTime":"2010-06-25 15:20","IssueNumber":"20100625-056"},{"BonusNumberString":"0,0,5,0,0|2|5","BonusTime":"2010-06-25 15:10","IssueNumber":"20100625-055"},{"BonusNumberString":"0,9,5,6,2|2|5","BonusTime":"2010-06-25 15:00","IssueNumber":"20100625-054"},{"BonusNumberString":"0,0,3,3,7|2|5","BonusTime":"2010-06-25 14:50","IssueNumber":"20100625-053"},{"BonusNumberString":"6,9,4,3,5|2|5","BonusTime":"2010-06-25 14:40","IssueNumber":"20100625-052"},{"BonusNumberString":"4,5,3,1,4|2|5","BonusTime":"2010-06-25 14:30","IssueNumber":"20100625-051"},{"BonusNumberString":"4,5,0,8,7|2|5","BonusTime":"2010-06-25 14:20","IssueNumber":"20100625-050"},{"BonusNumberString":"9,3,7,4,3|2|5","BonusTime":"2010-06-25 14:10","IssueNumber":"20100625-049"},{"BonusNumberString":"5,6,2,8,0|2|5","BonusTime":"2010-06-25 14:00","IssueNumber":"20100625-048"},{"BonusNumberString":"1,1,2,0,7|2|5","BonusTime":"2010-06-25 13:50","IssueNumber":"20100625-047"},{"BonusNumberString":"5,5,6,4,2|2|5","BonusTime":"2010-06-25 13:40","IssueNumber":"20100625-046"},{"BonusNumberString":"3,2,0,7,0|2|5","BonusTime":"2010-06-25 13:30","IssueNumber":"20100625-045"},{"BonusNumberString":"6,7,6,2,4|2|5","BonusTime":"2010-06-25 13:20","IssueNumber":"20100625-044"},{"BonusNumberString":"7,4,2,2,7|2|5","BonusTime":"2010-06-25 13:10","IssueNumber":"20100625-043"},{"BonusNumberString":"5,0,8,0,8|2|5","BonusTime":"2010-06-25 13:00","IssueNumber":"20100625-042"},{"BonusNumberString":"6,3,6,5,7|2|5","BonusTime":"2010-06-25 12:50","IssueNumber":"20100625-041"},{"BonusNumberString":"0,1,9,6,4|2|5","BonusTime":"2010-06-25 12:40","IssueNumber":"20100625-040"},{"BonusNumberString":"8,7,3,1,5|2|5","BonusTime":"2010-06-25 12:30","IssueNumber":"20100625-039"}];应该用正则表达式比较方便
System.Net.WebClient wc = new System.Net.WebClient();
Byte[] pageData = wc.DownloadData("网页地址");
string s= System.Text.Encoding.Default.GetString(pageData);
正则获取
http://topic.csdn.net/u/20100615/22/a391fc9d-d219-4724-afb4-e40566d9ab42.html
using System;
using System.Text.RegularExpressions;
using System.IO;
using System.Text;using System.Net;
using System.IO.Compression;
using System.Web;
using System.Collections;namespace 控制台测试
{
class Program
{
static void Main(string[] args)
{
byte[] buffer = getBytes("http://video.shishicai.cn/haoma/cqssc/list/120.aspx", null, null);
string html = Encoding.UTF8.GetString(buffer);
MatchCollection mc= Regex.Matches(html, @"{""BonusNumberString"":""(?<num>[^|]*)\|\d\|\d"",""BonusTime"":""(?<kai>[^""]*)"",""IssueNumber"":""(?<qi>[^""]*)""}");
foreach (Match m in mc)
{
Console.WriteLine(m.Groups["qi"] + " " + m.Groups["num"].Value.Replace(",", " ") + " " + m.Groups["kai"]);
}
Console.WriteLine("程序运行结束，按任意键关闭窗口！");
Console.ReadKey();
} // 读取网络资源，返回字节数组
private static byte[] getBytes(string url, CookieContainer cookie, byte[] postData)
{
int c = url.IndexOf("/", 10);
byte[] data = null;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.AllowAutoRedirect = true;
if (cookie != null) request.CookieContainer = cookie;
request.Referer = (c > 0 ? url.Substring(0, c) : url);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
request.Headers[HttpRequestHeader.AcceptEncoding] = "gzip, deflate"; if (postData != null) // 需要 Post 数据
{
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = postData.Length;
Stream requestStream = request.GetRequestStream();
requestStream.Write(postData, 0, postData.Length);
requestStream.Close();
} HttpWebResponse response = (HttpWebResponse)request.GetResponse();
string ce = response.Headers[HttpResponseHeader.ContentEncoding];
int ContentLength = (int)response.ContentLength;
Stream s = response.GetResponseStream();
c = 1024 * 10;
if (ContentLength < 0) // 不能获取数据的长度
{
data = new byte[c];
MemoryStream ms = new MemoryStream();
int l = s.Read(data, 0, c);
while (l > 0)
{
ms.Write(data, 0, l);
l = s.Read(data, 0, c);
}
data = ms.ToArray();
ms.Close();
}
else // 数据长度已知
{
data = new byte[ContentLength];
int pos = 0;
while (ContentLength > 0)
{
int l = s.Read(data, pos, ContentLength);
pos += l;
ContentLength -= l;
}
}
s.Close();
response.Close(); if (ce == "gzip") // 若数据是压缩格式，则要进行解压
{
MemoryStream js = new MemoryStream(); // 解压后的流
MemoryStream ms = new MemoryStream(data); // 用于解压的流
GZipStream g = new GZipStream(ms, CompressionMode.Decompress);
byte[] buffer = new byte[c]; // 读数据缓冲区
int l = g.Read(buffer, 0, c); // 一次读 10K
while (l > 0)
{
js.Write(buffer, 0, l);
l = g.Read(buffer, 0, c);
}
g.Close();
ms.Close();
data = js.ToArray();
js.Close();
}
return data; // 返回字节数组
}
}
}
运行结果：20100625-083 9 0 7 2 3 2010-06-25 19:50
20100625-082 6 1 3 7 1 2010-06-25 19:40
20100625-081 3 0 9 3 8 2010-06-25 19:30
20100625-080 1 3 8 6 0 2010-06-25 19:20
20100625-079 4 3 2 3 8 2010-06-25 19:10
20100625-078 6 8 1 4 5 2010-06-25 19:00
20100625-077 2 0 3 0 3 2010-06-25 18:50
20100625-076 5 3 3 6 5 2010-06-25 18:40
20100625-075 8 0 3 6 1 2010-06-25 18:30
20100625-074 7 6 1 2 1 2010-06-25 18:20
20100625-073 5 0 8 1 1 2010-06-25 18:10
20100625-072 3 6 0 9 4 2010-06-25 18:00
20100625-071 9 1 3 8 1 2010-06-25 17:50
20100625-070 0 6 5 1 5 2010-06-25 17:40
20100625-069 6 8 9 0 5 2010-06-25 17:30
20100625-068 6 2 2 7 5 2010-06-25 17:20
20100625-067 4 3 7 8 5 2010-06-25 17:10
20100625-066 4 3 5 0 8 2010-06-25 17:00
20100625-065 8 2 3 0 7 2010-06-25 16:50
20100625-064 6 3 0 7 9 2010-06-25 16:40
20100625-063 1 7 1 7 8 2010-06-25 16:30
20100625-062 2 9 6 2 3 2010-06-25 16:20
20100625-061 1 5 3 3 6 2010-06-25 16:10
20100625-060 2 5 3 5 0 2010-06-25 16:00
20100625-059 4 7 8 0 2 2010-06-25 15:50
20100625-058 0 2 2 2 1 2010-06-25 15:40
20100625-057 5 7 8 8 8 2010-06-25 15:30
20100625-056 4 1 0 1 9 2010-06-25 15:20
20100625-055 0 0 5 0 0 2010-06-25 15:10
20100625-054 0 9 5 6 2 2010-06-25 15:00
20100625-053 0 0 3 3 7 2010-06-25 14:50
20100625-052 6 9 4 3 5 2010-06-25 14:40
20100625-051 4 5 3 1 4 2010-06-25 14:30
20100625-050 4 5 0 8 7 2010-06-25 14:20
20100625-049 9 3 7 4 3 2010-06-25 14:10
20100625-048 5 6 2 8 0 2010-06-25 14:00
20100625-047 1 1 2 0 7 2010-06-25 13:50
20100625-046 5 5 6 4 2 2010-06-25 13:40
20100625-045 3 2 0 7 0 2010-06-25 13:30
20100625-044 6 7 6 2 4 2010-06-25 13:20
20100625-043 7 4 2 2 7 2010-06-25 13:10
20100625-042 5 0 8 0 8 2010-06-25 13:00
20100625-041 6 3 6 5 7 2010-06-25 12:50
20100625-040 0 1 9 6 4 2010-06-25 12:40
20100625-039 8 7 3 1 5 2010-06-25 12:30
20100625-038 8 6 7 8 3 2010-06-25 12:20
20100625-037 5 5 2 6 3 2010-06-25 12:10
20100625-036 5 9 5 7 8 2010-06-25 12:00
20100625-035 1 8 3 9 6 2010-06-25 11:50
20100625-034 6 7 8 2 6 2010-06-25 11:40
20100625-033 0 6 4 4 6 2010-06-25 11:30
20100625-032 4 3 9 1 6 2010-06-25 11:20
20100625-031 4 2 9 4 4 2010-06-25 11:10
20100625-030 6 3 1 4 9 2010-06-25 11:00
20100625-029 0 4 7 5 6 2010-06-25 10:50
20100625-028 5 6 5 8 7 2010-06-25 10:40
20100625-027 2 2 1 1 0 2010-06-25 10:30
20100625-026 5 2 2 9 3 2010-06-25 10:20
20100625-025 0 6 0 4 0 2010-06-25 10:10
20100625-024 9 7 9 9 2 2010-06-25 10:00
20100625-023 0 4 9 1 5 2010-06-25 01:55
20100625-022 8 1 2 2 0 2010-06-25 01:50
20100625-021 3 3 6 8 3 2010-06-25 01:45
20100625-020 4 1 1 6 4 2010-06-25 01:40
20100625-019 9 8 0 9 7 2010-06-25 01:35
20100625-018 0 5 6 2 7 2010-06-25 01:30
20100625-017 1 2 6 3 6 2010-06-25 01:25
20100625-016 0 4 6 6 7 2010-06-25 01:20
20100625-015 8 8 7 1 8 2010-06-25 01:15
20100625-014 5 9 0 3 8 2010-06-25 01:10
20100625-013 6 6 9 4 4 2010-06-25 01:05
20100625-012 0 7 6 6 3 2010-06-25 01:00
20100625-011 1 8 1 9 9 2010-06-25 00:55
20100625-010 3 7 9 6 3 2010-06-25 00:50
20100625-009 1 6 8 4 3 2010-06-25 00:45
20100625-008 3 1 0 3 7 2010-06-25 00:40
20100625-007 6 9 3 5 0 2010-06-25 00:35
20100625-006 5 9 7 6 4 2010-06-25 00:30
20100625-005 7 9 3 7 5 2010-06-25 00:25
20100625-004 4 1 4 3 9 2010-06-25 00:20
20100625-003 8 0 1 3 8 2010-06-25 00:15
20100625-002 4 2 7 1 7 2010-06-25 00:10
20100625-001 8 3 5 5 4 2010-06-25 00:05
20100624-120 6 1 2 8 0 2010-06-25 00:00
20100624-119 0 0 7 4 7 2010-06-24 23:55
20100624-118 6 1 4 3 5 2010-06-24 23:50
20100624-117 9 1 8 1 6 2010-06-24 23:45
20100624-116 2 9 4 7 3 2010-06-24 23:40
20100624-115 2 8 3 8 8 2010-06-24 23:35
20100624-114 8 9 0 1 5 2010-06-24 23:30
20100624-113 5 6 5 8 6 2010-06-24 23:25
20100624-112 9 2 0 1 2 2010-06-24 23:20
20100624-111 0 4 3 3 3 2010-06-24 23:15
20100624-110 6 1 3 6 6 2010-06-24 23:10
20100624-109 3 0 2 3 1 2010-06-24 23:05
20100624-108 3 2 5 4 8 2010-06-24 23:00
20100624-107 3 1 5 2 2 2010-06-24 22:55
20100624-106 3 4 7 6 7 2010-06-24 22:50
20100624-105 8 7 0 7 7 2010-06-24 22:45
20100624-104 9 6 5 9 3 2010-06-24 22:40
20100624-103 0 7 3 9 9 2010-06-24 22:35
20100624-102 6 1 7 7 3 2010-06-24 22:30
20100624-101 0 4 6 2 2 2010-06-24 22:25
20100624-100 7 5 3 5 2 2010-06-24 22:20
20100624-099 3 0 3 3 5 2010-06-24 22:15
20100624-098 3 9 4 5 9 2010-06-24 22:10
20100624-097 2 8 1 5 4 2010-06-24 22:05
20100624-096 2 0 0 1 1 2010-06-24 22:00
20100624-095 3 2 7 7 8 2010-06-24 21:50
20100624-094 3 8 2 4 4 2010-06-24 21:40
20100624-093 7 5 5 9 0 2010-06-24 21:30
20100624-092 8 1 3 2 2 2010-06-24 21:20
20100624-091 6 8 5 4 1 2010-06-24 21:10
20100624-090 8 1 8 5 5 2010-06-24 21:00
20100624-089 8 8 3 2 1 2010-06-24 20:50
20100624-088 9 5 2 3 0 2010-06-24 20:40
20100624-087 7 9 3 0 2 2010-06-24 20:30
20100624-086 0 8 6 2 4 2010-06-24 20:20
20100624-085 8 7 0 4 8 2010-06-24 20:10
20100624-084 8 2 7 1 0 2010-06-24 20:00
程序运行结束，按任意键关闭窗口！
{
static void Main(string[] args)
{
string html = WebFunc.GetHtmlEx("http://video.shishicai.cn/haoma/cqssc/list/120.aspx", Encoding.GetEncoding("utf-8"));
string pattern = @"(?is)(?<text1>\d,\d,\d,\d,\d\|\d\|\d).*?(?<text2>\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2}?).*?(?<text3>\d{8}-\d{3})";//
Regex reg = new Regex(pattern); MatchCollection mc = reg.Matches(html);
foreach (Match m in mc)
{
if (m.Success)
{
Console.Write(m.Groups["text1"].Value + " " + m.Groups["text2"].Value + " " + m.Groups["text3"].Value);
Console.WriteLine();
}
}
}
} static class WebFunc
{
private static CookieContainer cookie = new CookieContainer();
private static string contentType = "application/x-www-form-urlencoded";
private static string accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
private static string userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)"; public static string GetHtmlEx(string url, Encoding encoding)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = userAgent;
request.ContentType = contentType;
request.CookieContainer = cookie;
request.Accept = accept;
request.Method = "get"; WebResponse response = request.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream, encoding);
String html = reader.ReadToEnd();
response.Close(); return html;
}
}
{
if (File.Exists("log.txt")) File.Delete("log.txt");
Regex tempRegExp = new Regex(@"[,\|]", RegexOptions.Compiled);//编译以便提高速度
string html = WebFunc.GetHtmlEx("http://video.shishicai.cn/haoma/cqssc/list/120.aspx", Encoding.GetEncoding("GB2312"));
MatchCollection mc = Regex.Matches(html, @"""BonusNumberString\D+(?<ball>[\d,\|]+)\D+(?<time>[- :\d]+)\D+(?<no>[\d-]+)");
foreach (Match m in mc)
{
string line = m.Groups["no"].Value + " " + tempRegExp.Replace(m.Groups["ball"].Value, "") + " " + m.Groups["time"].Value+Environment.NewLine;
Console.Write(line);
File.AppendAllText("log.txt", line);
}
Console.ReadKey();
}