主调用过程:
……
for i:= 0 to ThreadCount - 1 do //分发数据
begin
aFrom:= i * BlockSize;
if i = ADO_List.Count - 1 then
aTo:= ADOQ.RecordCount
else
aTo:= (i + 1) * BlockSize; with TAnalyzeThread.Create(aFrom, aTo, TADOQuery(ADO_List.Items[i]), PIntArr(Count_List.Items[i])^, AnalyzeMethod, pb, i) do
begin
Priority:= tpNormal;
OnTerminate:= ThreadDone;
end;
inc(ThreadRuning);
end;
……
……
for i:= 0 to ThreadCount - 1 do //分发数据
begin
aFrom:= i * BlockSize;
if i = ADO_List.Count - 1 then
aTo:= ADOQ.RecordCount
else
aTo:= (i + 1) * BlockSize; with TAnalyzeThread.Create(aFrom, aTo, TADOQuery(ADO_List.Items[i]), PIntArr(Count_List.Items[i])^, AnalyzeMethod, pb, i) do
begin
Priority:= tpNormal;
OnTerminate:= ThreadDone;
end;
inc(ThreadRuning);
end;
……
unit UAnalyzeThread;interfaceuses
Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
Dialogs, ToolWin, ComCtrls, ExtCtrls, StdCtrls, Tabs, Grids, TeeProcs,
TeEngine, Chart, DBGrids, ADODB, DB, ActnList, Series, math, ImgList, Unit1,
Activex;type
PCountArray = ^TCountArray;
TCountArray = array[0..1000] of Int64; //最大支持3位 TAnalyzeThread = class(TThread)
Private
FSQL: String;
FConn: TADOConnection;
ADOQ: TADOQuery;
FSize: Integer;
AnalyzeArray: PCountArray;
AnalyzeMethod: TAnalyzeMethod;
StartPos: Integer;
EndPos: Integer;
PB: TProgressBar;
Index: Integer;
function PutValueInBarrier(Value: Double; var par: array of Int64): Boolean;
Procedure UpdatePB;
Protected
procedure Execute; override;
Procedure DoAnalyze(var A: array of Int64);
public
constructor Create(aFrom: Integer; aTo: Integer;fADOQ: TADOQuery;
var anyArray: array of int64; am: TAnalyzeMethod; aPB: TProgressBar; I: Integer);
destructor Destroy; override;
end;implementation{ TAnalyzeThread }constructor TAnalyzeThread.Create(aFrom: Integer; aTo: Integer;
fADOQ: TADOQuery; var anyArray: array of int64;
am: TAnalyzeMethod; aPB: TProgressBar; I: Integer);
begin
StartPos:= aFrom;
EndPos:= aTo;
FSize:= High(anyArray) - Low(anyArray) + 1;
ADOQ:= fADOQ; AnalyzeArray:= @anyArray;
FreeOnTerminate:= true;
AnalyzeMethod:= am;
PB:= aPB;
Index:= i; CoInitialize(nil);
inherited Create(False);
end;destructor TAnalyzeThread.Destroy;
begin
CoUnInitialize;
inherited;
end;procedure TAnalyzeThread.DoAnalyze(var A: array of Int64);
var
Value: Double;
i: integer;
begin
try
if i <> 0 then
ADOQ.Last
else
ADOQ.First;
for i:= StartPos to EndPos - 1 do
begin
Value:= ADOQ.Fields.Fields[0].AsFloat; PutValueInBarrier(Value, a); if i = 0 then
ADOQ.Next
else
ADOQ.Prior;
//PB.Position:= Pb.Position + 1;
Synchronize(UpdatePB);
end;
except
On e: Exception do
self.Terminate;
end;
end;procedure TAnalyzeThread.Execute;
begin
DoAnalyze(Slice(AnalyzeArray^, FSize));
end;function TAnalyzeThread.PutValueInBarrier(Value: Double;
var par: array of Int64): Boolean;
var
TmpValue: Double;
i: integer;
begin
TmpValue:= ABS(Value);
while TmpValue < 1000 do
TmpValue:= TmpValue * 1000000;
case AnalyzeMethod of
amFirst: begin
i:= StrToInt(Copy(FloatToStr(TmpValue), 1, 1));
Par[i]:= Par[i] + 1;
end;
amSecond: begin
i:= StrToInt(Copy(FloatToStr(TmpValue), 1, 2));
Par[i]:= Par[i] + 1;
end;
amThird: begin
i:= StrToInt(Copy(FloatToStr(TmpValue), 1, 3));
Par[i]:= Par[i] + 1;
end;
end;
result:= true;
end;procedure TAnalyzeThread.UpdatePB;
begin
PB.Position:= PB.Position + 1;
end;end.
△T:时间的增长
△Inc:数据量的增长
1、将几百万数据从服务器取到客户端做遍历
2、试图“一个从前向后,一个从后向前来遍历一个数据表”以加快速度
是错误的。
1、遍历的目的是什么,要做什么统计么?那直接在数据库用SQL语句就搞定了,特别是建立了适当的索引后,效率很高。
2、“从后向前”的意思是首先要ADOQ.Last,这就造成首先要从数据苦将全部记录取到客户端,放到内存里。
var anyArray: array of int64; am: TAnalyzeMethod; aPB:对象的传递因该是传递地址吧。那么fadoq传递的是什么?是外部adoquery的地址。结果是,你创建若干给线程,每个线程都是使用了同一个adoquery。也就是说搞的不好指针recno会来回抖动。效率很低。比如你分配第一个线程是
1--500000,第二个线程是以后的。那么在第一个查到n条的时候,第二个线程获得了执行。在第二个线程查到了k条的时候,第一条或的了线程。就这样。来来回回的。做了很多重复的工作。除非记录指针的滑动不需要消耗很的资源哦!
但是还是有点不明白就是ADOQuery的CursorLocation = clUseClient是不是就已经把数据取道客户端了。还有我再计算时间的时间的时侯没有把打开数据库及下载数据到客户端的计算在内。如果在线程中这样遍历效率就非常高
线程1:
SQL: 'Select Field form Table order by index'
ADOQuery.first;
ADOQuery.next
线程2:
SQL: 'Select Field form Table order by index desc'
ADOQuery.first;
ADOQuery.next;