用C#写一个程序,运行在一台10G内存的服务器上,
作用是24小时不停的 ping 10万个ip,以获取每个ip的主机是否正常连接,
每隔2秒钟这10万个ip就全部ping一次,
并记录返回的指定信息,部分信息需要存储在硬盘或者数据库上.
与此同时,返回的信息需要一些运算和加工,展现给使用者.如果ip只有少量几个,而且不是不停的运行,那我可以轻松实现,
但是大量的数据和高并发性出现后,C#的程序是不是应该大换血了?
究竟应该如何处理代码?如何做这个程序?高并发大数据c#ping
作用是24小时不停的 ping 10万个ip,以获取每个ip的主机是否正常连接,
每隔2秒钟这10万个ip就全部ping一次,
并记录返回的指定信息,部分信息需要存储在硬盘或者数据库上.
与此同时,返回的信息需要一些运算和加工,展现给使用者.如果ip只有少量几个,而且不是不停的运行,那我可以轻松实现,
但是大量的数据和高并发性出现后,C#的程序是不是应该大换血了?
究竟应该如何处理代码?如何做这个程序?高并发大数据c#ping
更不要说数据库了
要不分块实施,一块 一千个IP。
在说了既然是ping 肯定得ping 最少1s以上才有返回值 你怎么可能做到?
或者说下1万个ip,每隔60秒ping一次怎样优化性能
ping一个远程IP(发送32字节的包)平均要3ms,1千个IP要3s,1万个ip要30s,理论是可行,如果包大些,就不够了。你用下面代码测试一下,ping一个IP要多少时间?
public string PingT(string ip)
{
System.Net.NetworkInformation.Ping ping = new System.Net.NetworkInformation.Ping();
using (ping)
{
PingOptions options = new PingOptions
{
DontFragment = true
};
string s = "Test Data!";
byte[] bytes = Encoding.ASCII.GetBytes(s);
int timeout = 1000;
PingReply reply = ping.Send(ip, timeout, bytes, options);
if (reply.Status == IPStatus.Success)
{
success = "1";
ipreply = reply.RoundtripTime.ToString()+"ms"; }
else
{
success = "0";
ipreply = "网络异常!";
}
//ping.Dispose(); }
return (success + " " + ipreply );
}
数据库可以采用SqlBulkCopy批量导入。
private static extern int SendARP(Int32 destinationIp, Int32 sourceIp, ref Int64 mac, ref Int32 length);
有的时候返回time out, reply.Address是0.0.0.0
{
private System.Net.NetworkInformation.Ping ping;
private func<System.Net.IPAddress> getIP;
private action<System.Net.IPAddress, bool> onCompleted;
private System.Net.IPAddress currentIP;
PingIP(func<System.Net.IPAddress> getIP, action<System.Net.IPAddress, bool> onCompleted)
{
this.getIP = getIP;
this.onCompleted = onCompleted;
ping = new System.Net.NetworkInformation.Ping();
ping.PingCompleted += pingCompleted;
next();
}
private void pingCompleted(object sender, System.Net.NetworkInformation.PingCompletedEventArgs e)
{
this.onCompleted(currentIP, e.Error == null);
next();
}
private void next()
{
while (ping != null)
{
if ((currentIP = getIP()) == null) Dispose();
else
{
try
{
ping.SendAsync(currentIP, 1000, this);
break;
}
catch
{
this.onCompleted(currentIP, false);
}
}
}
}
public void Dispose()
{
if (ping != null)
{
ping.PingCompleted -= pingCompleted;
ping.Dispose();
ping = null;
}
}
}
{
private class pinger : IDisposable
{
private pingInterval pingInterval;
private System.Net.NetworkInformation.Ping ping;
private System.Net.IPAddress ip;
public pinger(pingInterval pingInterval)
{
this.pingInterval = pingInterval;
ping = new System.Net.NetworkInformation.Ping();
ping.PingCompleted += pingCompleted;
}
internal void Next()
{
while (ping != null)
{
if ((ip = pingInterval.getIP()) == null)
{
pingInterval.free(this);
break;
}
else
{
try
{
ping.SendAsync(ip, 100, this);
break;
}
catch { pingInterval.onCompleted(ip, false); }
}
}
}
private void pingCompleted(object sender, System.Net.NetworkInformation.PingCompletedEventArgs e)
{
pingInterval.onCompleted(ip, e.Error == null);
Next();
}
public void Dispose()
{
if (ping != null)
{
ping.PingCompleted -= pingCompleted;
ping.Dispose();
ping = null;
}
}
}
private readonly System.Net.IPAddress[] ips;
private int ipIndex;
private readonly object ipLock = new object();
private readonly int intervalSeconds;
private action<System.Net.IPAddress, bool> onCompleted;
private pinger[] pings;
private readonly pinger[] freePings;
private readonly pinger[] nextPings;
private int freePingIndex;
private DateTime pingTime;
private readonly System.Timers.Timer timer;
private readonly object pingLock = new object();
public pingInterval(System.Net.IPAddress[] ips, int intervalSeconds, action<System.Net.IPAddress, bool> onCompleted)
{
this.ips = ips;
this.intervalSeconds = intervalSeconds;
this.onCompleted = onCompleted;
freePingIndex = ips.Length / (intervalSeconds * 5) + 1;
pings = new pinger[freePingIndex];
freePings = new pinger[freePingIndex];
nextPings = new pinger[freePingIndex];
for (int index = freePingIndex; index != 0; pings[index] = freePings[index] = new pinger(this)) --index;
timer = new System.Timers.Timer(intervalSeconds * 1000);
timer.Elapsed += next;
pingTime = DateTime.Now;
next(null, null);
}
private void next(object sender, System.Timers.ElapsedEventArgs e)
{
if (sender != null) timer.Stop();
while (wait())
{
System.Threading.Monitor.Enter(pingLock);
int count = freePingIndex;
try
{
Array.Copy(freePings, nextPings, freePingIndex);
freePingIndex = 0;
}
finally { System.Threading.Monitor.Exit(pingLock); }
while (count != 0) nextPings[--count].Next();
DateTime now = DateTime.Now;
if ((pingTime = pingTime.AddSeconds(this.intervalSeconds)) > now)
{
timer.Interval = (pingTime - now).TotalMilliseconds;
if (pings != null) timer.Start();
break;
}
}
}
private bool wait()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (pings != null)
{
if (ipIndex != 0) System.Threading.Monitor.Wait(ipLock);
ipIndex = ips.Length;
return true;
}
}
finally { System.Threading.Monitor.Exit(ipLock); }
return false;
}
private System.Net.IPAddress getIP()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (ipIndex != 0)
{
System.Net.IPAddress ip = ips[--ipIndex];
if (ipIndex == 0) System.Threading.Monitor.Pulse(ipLock);
return ip;
}
}
finally { System.Threading.Monitor.Exit(ipLock); }
return null;
}
private void free(pinger ping)
{
System.Threading.Monitor.Enter(pingLock);
try
{
freePings[freePingIndex++] = ping;
}
finally { System.Threading.Monitor.Exit(pingLock); }
}
public void Dispose()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (pings != null)
{
timer.Stop();
timer.Elapsed -= next;
foreach (pinger ping in pings) ping.Dispose();
pings = null;
}
System.Threading.Monitor.Pulse(ipLock);
}
finally { System.Threading.Monitor.Exit(ipLock); }
}
}注意,onCompleted不要长时间阻塞,最好是仅仅lock操作一下某个队列。
起码2秒误报率比较高。Windows默认的ping timeout是5秒。
把ping的时间间隔扩大,是不是应该不会被试做网络攻击?比如我每隔10分钟才ping一遍.
LZ的测试需要忽略路由器防火墙才可行,否则没什么意义
做网络连通测试的超时时间一般都略长,因为网络本身就是一个参差不齐的“不稳定环境”
{
private sealed class pinger : IDisposable
{
private readonly pingInterval pingInterval;
private System.Net.NetworkInformation.Ping ping;
private System.Net.IPAddress ip;
public pinger(pingInterval pingInterval)
{
this.pingInterval = pingInterval;
ping = new System.Net.NetworkInformation.Ping();
ping.PingCompleted += pingCompleted;
}
internal void Next()
{
while (ping != null)
{
if ((ip = pingInterval.getIP()) == null)
{
pingInterval.free(this);
break;
}
else
{
try
{
ping.SendAsync(ip, pingInterval.timeoutMilliseconds, this);
break;
}
catch { pingInterval.onCompleted(ip, false); }
}
}
}
private void pingCompleted(object sender, System.Net.NetworkInformation.PingCompletedEventArgs e)
{
pingInterval.onCompleted(ip, e.Error == null);
Next();
}
public void Dispose()
{
if (ping != null)
{
ping.PingCompleted -= pingCompleted;
ping.Dispose();
ping = null;
}
}
}
private const int maxPingCount = 1 << 15;
private readonly System.Net.IPAddress[] ips;
private int ipIndex;
private readonly object ipLock = new object();
private readonly int intervalSeconds;
private readonly int timeoutMilliseconds;
private action<System.Net.IPAddress, bool> onCompleted;
private pinger[] pings;
private readonly pinger[] freePings;
private readonly pinger[] nextPings;
private int freePingIndex;
private DateTime pingTime;
private readonly System.Timers.Timer timer;
private readonly object pingLock = new object();
public pingInterval(System.Net.IPAddress[] ips, int intervalSeconds, int timeoutMilliseconds, action<System.Net.IPAddress, bool> onCompleted)
{
this.ips = ips;
this.intervalSeconds = intervalSeconds;
this.timeoutMilliseconds = timeoutMilliseconds;
this.onCompleted = onCompleted;
freePingIndex = (intervalSeconds * 1000) / (timeoutMilliseconds + 100);
if (freePingIndex <= 1) freePingIndex = ips.Length;
else freePingIndex = ips.Length / freePingIndex + 1;
if (freePingIndex > maxPingCount) freePingIndex = maxPingCount;
pings = new pinger[freePingIndex];
freePings = new pinger[freePingIndex];
nextPings = new pinger[freePingIndex];
for (int index = freePingIndex; index != 0; pings[index] = freePings[index] = new pinger(this)) --index;
timer = new System.Timers.Timer(intervalSeconds * 1000);
timer.AutoReset = false;
timer.Elapsed += next;
pingTime = DateTime.Now;
next(null, null);
}
private void next(object sender, System.Timers.ElapsedEventArgs e)
{
while (wait())
{
System.Threading.Monitor.Enter(pingLock);
int count = freePingIndex;
try
{
Array.Copy(freePings, nextPings, freePingIndex);
freePingIndex = 0;
}
finally { System.Threading.Monitor.Exit(pingLock); }
while (count != 0) nextPings[--count].Next();
DateTime now = DateTime.Now;
if ((pingTime = pingTime.AddSeconds(this.intervalSeconds)) > now)
{
timer.Interval = (pingTime - now).TotalMilliseconds;
if (pings != null) timer.Start();
break;
}
}
}
private bool wait()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (pings != null)
{
if (ipIndex != 0) System.Threading.Monitor.Wait(ipLock);
ipIndex = ips.Length;
return true;
}
}
finally { System.Threading.Monitor.Exit(ipLock); }
return false;
}
private System.Net.IPAddress getIP()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (ipIndex != 0)
{
System.Net.IPAddress ip = ips[--ipIndex];
if (ipIndex == 0) System.Threading.Monitor.Pulse(ipLock);
return ip;
}
}
finally { System.Threading.Monitor.Exit(ipLock); }
return null;
}
private void free(pinger ping)
{
System.Threading.Monitor.Enter(pingLock);
try
{
freePings[freePingIndex++] = ping;
}
finally { System.Threading.Monitor.Exit(pingLock); }
}
public void Dispose()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (pings != null)
{
timer.Stop();
timer.Elapsed -= next;
foreach (pinger ping in pings) ping.Dispose();
pings = null;
}
System.Threading.Monitor.Pulse(ipLock);
}
finally { System.Threading.Monitor.Exit(ipLock); }
}
}
至于ISP会如何处理,还真不知道,不过我现在倒是有想法试一试。
关于PING的测试结果非常糟糕。以2秒为一次循环,超时设置为2秒,最大PING实例数量为512,IP数量为2^n。下面是本地测试结果:
IP数量 <= 8 ,完全正常。
16 <= IP数量 <= 32 ,第一次循环正常,从第二次循环开始每轮1个IP失败。
64 <= IP数量 <= 256 ,第一次循环正常,从第二次循环开始每轮10%左右IP失败。
IP数量 = 512 ,第一次循环正常,从第二次循环开始每轮20%左右IP失败。成功 > 200IP/s,测试中的单位时间成功IP最多的情况。
IP数量 = 1024 ,第一次循环50%左右IP失败,从第二次循环开始每轮70%左右IP失败,而且由于超时太多循环周期在3-4s左右。成功 < 100IP/s。
IP数量 = 2048 ,每轮循环75%以上IP失败,循环周期在8s左右,到这里为止最大网络流量不到30KB/s。成功 < 64IP/s。
继续测试,结果会越来越糟糕。接下来不管成功率如果,测试一个“高并发”的情况。
PING实例 = 1024 ,10KB/s <= 网络流量 60KB/s,10% <= CPU <= 30%。
PING实例 = 2048 ,20KB/s <= 网络流量 100KB/s,15% <= CPU <= 50%。
PING实例 = 4096 ,40KB/s <= 网络流量 100KB/s,CPU = 100%。
好了,我这台烂机器没法再测试下去了。由于10W IP空转(仅仅不执行PING操作)测试CPU最多2%,可见PING是非常消耗CPU的操作。
现在有一点可以确定,我本地环境PING的并发量与网络带宽关系不大。至于是不是网络环境(路由节点)的限制,无法确定,但是很可能现实环境无法实现高并发。
如果要解决这个问题,需要自己使用rawSocket发ICMP包,并且管理他们。
新的测试结果表明,失败率高也是由System.Net.NetworkInformation.Ping或者windows操作系统造成的。至于为什么会这样,还不清楚。测试还是以2秒为一次循环,超时设置为2秒,最大PING实例数量为256(本地测试中比较稳定的最大并发数,也许128能更稳定),IP数量为2^n。下面是本地测试结果:
IP数量 <= 8192 ,完全正常,循环周期 < 1s。100KB/s <= 网络流量 300KB/s,CPU <= 50%。
IP数量 = 16384 ,基本正常,1s < 循环周期 < 2s。200KB/s <= 网络流量 600KB/s,70% <= CPU <= 90%。
我本地的带宽是4Mb/s = 512KB/s (上面的600KB/s是任务管理器中的6%),测试只能到这里了。有一点要注意,超时或者出错对于CPU占用率的增大影响严重。基于以上测试,我认为只要带宽达到2MB/s,6个CPU核心就差不多可以达到楼主需求了(我的CPU是单核心Inter Celeron M CPU 520 1.6GHz)。当然重要的是,PING的失败率不能太高,否则4个CPU承受不了,而且循环周期会被延长。
下面是基于fastCSharp的测试程序,由于原来是异步操作,后来改为同步只是简单修改了一下,所以并不逻辑有些怪异。
sealed class pingInterval : IDisposable
{
private sealed class pinger : IDisposable
{
private readonly pingInterval pingInterval;
private fastCSharp.net.ping ping;
private System.Net.IPEndPoint ip;
public pinger(pingInterval pingInterval)
{
this.pingInterval = pingInterval;
ping = new fastCSharp.net.ping(1000, pingInterval.timeoutMilliseconds, null);
}
internal void Next()
{
fastCSharp.threading.task.TinyTask.Add(next);
}
private void next()
{
for (ip = pingInterval.getIP(); ip != null; ip = pingInterval.getIP()) pingInterval.onCompleted(ip, ping.Ping(ip));
pingInterval.free(this);
}
public void Dispose()
{
fastCSharp.net.ping ping = this.ping;
this.ping = null;
if (ping != null) ping.Dispose();
}
}
private const int maxPingCount = 256;
private readonly System.Net.IPEndPoint[] ips;
private int ipIndex;
private readonly object ipLock = new object();
private readonly int intervalSeconds;
private readonly int timeoutMilliseconds;
private action<System.Net.IPEndPoint, bool> onCompleted;
private pinger[] pings;
private readonly pinger[] freePings;
private readonly pinger[] nextPings;
private int freePingIndex;
private DateTime pingTime;
private readonly System.Timers.Timer timer;
private readonly object pingLock = new object();
public pingInterval(System.Net.IPEndPoint[] ips, int intervalSeconds, int timeoutMilliseconds, action<System.Net.IPEndPoint, bool> onCompleted)
{
this.ips = ips;
this.intervalSeconds = intervalSeconds;
this.timeoutMilliseconds = timeoutMilliseconds;
this.onCompleted = onCompleted;
freePingIndex = (intervalSeconds * 1000) / (timeoutMilliseconds + 1000);
if (freePingIndex <= 1) freePingIndex = ips.Length;
else freePingIndex = ips.Length / freePingIndex + 1;
if (freePingIndex > maxPingCount) freePingIndex = maxPingCount;
pings = new pinger[freePingIndex];
freePings = new pinger[freePingIndex];
nextPings = new pinger[freePingIndex];
for (int index = freePingIndex; index != 0; pings[index] = freePings[index] = new pinger(this)) --index;
timer = new System.Timers.Timer(intervalSeconds * 1000);
timer.AutoReset = false;
timer.Elapsed += next;
pingTime = DateTime.Now;
next(null, null);
}
private void next(object sender, System.Timers.ElapsedEventArgs e)
{
while (wait())
{
Console.WriteLine("Start " + DateTime.Now.toString());
System.Threading.Monitor.Enter(pingLock);
int count = freePingIndex;
try
{
Array.Copy(freePings, nextPings, freePingIndex);
freePingIndex = 0;
}
finally { System.Threading.Monitor.Exit(pingLock); }
while (count != 0) nextPings[--count].Next();
DateTime now = DateTime.Now;
if ((pingTime = pingTime.AddSeconds(this.intervalSeconds)) > now)
{
timer.Interval = (pingTime - now).TotalMilliseconds;
if (pings != null) timer.Start();
break;
}
}
}
private bool wait()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (pings != null)
{
if (ipIndex != 0) System.Threading.Monitor.Wait(ipLock);
ipIndex = ips.Length;
return true;
}
}
finally { System.Threading.Monitor.Exit(ipLock); }
return false;
}
private System.Net.IPEndPoint getIP()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (ipIndex != 0)
{
System.Net.IPEndPoint ip = ips[--ipIndex];
if (ipIndex == 0) System.Threading.Monitor.Pulse(ipLock);
return ip;
}
}
finally { System.Threading.Monitor.Exit(ipLock); }
return null;
}
private void free(pinger ping)
{
System.Threading.Monitor.Enter(pingLock);
try
{
freePings[freePingIndex++] = ping;
}
finally { System.Threading.Monitor.Exit(pingLock); }
}
public void Dispose()
{
System.Threading.Monitor.Enter(ipLock);
try
{
if (pings != null)
{
timer.Stop();
timer.Elapsed -= next;
foreach (pinger ping in pings) ping.Dispose();
pings = null;
}
System.Threading.Monitor.Pulse(ipLock);
}
finally { System.Threading.Monitor.Exit(ipLock); }
}
}
static int pingCount;
static int pingErrorCount;
static int loopCount = -1;
static System.Net.IPEndPoint[] ips;
static int ipCount;
static void onPing(System.Net.IPEndPoint ip, bool isPing)
{
if (System.Threading.Interlocked.Decrement(ref ipCount) == -1)
{
System.Threading.Interlocked.Add(ref ipCount, ips.Length);
if (++loopCount != 0) Console.WriteLine(DateTime.Now.toString() + " LOOP[" + loopCount.toString() + "] Count[" + (pingCount / loopCount).toString() + "] ErrorCount[" + pingErrorCount.toString() + "]");
}
if (isPing) ++pingCount;
else ++pingErrorCount;
//Console.WriteLine(DateTime.Now.ToString("HH:mm:ss") + " " + ip.ToString() + " : " + (reply != null && reply.Status == System.Net.NetworkInformation.IPStatus.Success).ToString());
}
static void Main(string[] args)
{
ips = System.IO.File.ReadAllText(@"d:\ip.txt").Split(',').left(16384).getArray(value => new System.Net.IPEndPoint(System.Net.IPAddress.Parse(value), 0));
using (pingInterval pingInterval = new pingInterval(ips, 2, 2000, onPing))
{
Console.ReadKey();
}
Console.WriteLine("End"); Console.ReadKey();
}
我现在这样做,我把1万个IP分成10组,每组对应一个线程,每个线程里边用异步Ping.SendAsync()的方法进行ping;因为要对ping后的结果做些处理,所以异步完成处理事件里要调用一个方法Summary(),但是每次测试都在这个方法里报错..
是不是既用了异步又用多线程,是会发生冲突?
还有我想问下你测试的结果,如果能做到10万个ip两秒一次不停的ping, 7x24不间断运行,需要什么样的硬件配置和程序优化?你直接说个大概结论.
再次感谢各位的关注.
2、我想Summary至少需要操作一个共享队列(最好是仅仅操作一下共享队列,不要阻塞线程池中的线程,数据处理的事应该交给其它线程同步处理),需要lock操作。
3、根据我的测试结果推论,需要2MB/s的上行带宽、3MB/s的下行带宽、8核心的CPU。至于7x24不间断一般都是受实际网络环境影响的。硬件方面可能需要一台备用机器,程序优化要根据你的其他实际业务需求而定。
因为Socket.ReceiveFromAsync会丢包,也许System.Net.NetworkInformation.Ping也是这个原因。
如果IP质量不行,2秒就不能强求了。N个并发处理量基本相当于2*N秒单线程的处理量。
所以如果超时IP数量超过并发数,循环周期就肯定超过2秒。
最糟糕的情况是所有IP都不正常,这时候的循环周期时间接近于 IP数量*超时单位/并发数。