import java.util.Hashtable;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.logging.Logger;public class Download implements Runnable { static public Queue<DownloadPage> taskQueue = new ConcurrentLinkedQueue<DownloadPage>();
static public Map<String, CacheStorage> dict = new Hashtable<String, CacheStorage>(); public static void main(String[] args) {
Download d1 = new Download();
Download d2 = new Download();
Download d3 = new Download();
new Thread(d1).start();
new Thread(d2).start();
new Thread(d2).start(); } static Boolean DoTask(DownloadPage dp) throws Exception {
System.out.println(Thread.currentThread().getName()+"进入进程");
if (dp == null) {
return true;
}
String DownloadHtmlUrl = dp.RemoteUrl;
CacheStorage cs = null;
cs = dict.get(DownloadHtmlUrl);
if (cs == null) {
CacheStorage ct = null;
ct = new CacheStorage();
ct.Local = "";
ct.status = "";
ct.Url = DownloadHtmlUrl;
dict.put(DownloadHtmlUrl, ct);
System.out.println("ERR1");
return false;
}
if (cs.status.equals("done")) {
return true;
}
if (cs.status == "downloading") {
return false;
}
if (cs.status == "pending") {
return false;
}
for (String u : dp.DependencyUrls) { CacheStorage csDependency = dict.get(u);
if (csDependency == null) { csDependency = new CacheStorage();
csDependency.Local = "";
csDependency.status = "";
csDependency.Url = u;
dict.put(u, csDependency);
if (csDependency.status != "downloadng") {
csDependency.status = "downloading";
Thread.sleep(2000);
System.out.println("Downloading" + csDependency.Url);
csDependency.status = "done";
csDependency.Local = "Download local file folder"; return false;
} return false;
}
if (csDependency.status != "done") { cs.status = "pending";
DownloadPage down = new DownloadPage();
down.RemoteUrl = csDependency.Url;
taskQueue.offer(down);
return false;
}
}
cs.status = "downloading";
System.out.println("Downloading" + cs.Url);
Thread.sleep(2000);
cs.status = "done";
cs.Local = "Download local file folder";
// System.out.println(cs.Url+"下载完成");
System.out.println(Thread.currentThread().getName()+"离开进程");
return true; } @Override
public void run() {
// TODO Auto-generated method stub
DownloadPage downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "a.html";
downloadPage.DependencyUrls = new String[] { "1,jpg", "2.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "b.html";
downloadPage.DependencyUrls = new String[] { "21,jpg", "22.jpg",
"44.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "c.html";
downloadPage.DependencyUrls = new String[] { "31,jpg", "32.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "d.html";
downloadPage.DependencyUrls = new String[] { "41,jpg", "42.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "e.html";
downloadPage.DependencyUrls = new String[] { "51,jpg", "52.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "f.html";
downloadPage.DependencyUrls = new String[] { "61,jpg", "62.jpg" };
taskQueue.offer(downloadPage);
while (taskQueue.size() != 0) {
try {
DownloadPage dp = taskQueue.poll();
if (dp == null)
break;
if (DoTask(dp) == false) {
/*
* System.out.println("Cannot download" + dp.RemoteUrl +
* "now,will try later");
*/
taskQueue.offer(dp);
} else {
/*
* synchronized (this) { synMethod(dp.RemoteUrl); }
*/ }
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.logging.Logger;public class Download implements Runnable { static public Queue<DownloadPage> taskQueue = new ConcurrentLinkedQueue<DownloadPage>();
static public Map<String, CacheStorage> dict = new Hashtable<String, CacheStorage>(); public static void main(String[] args) {
Download d1 = new Download();
Download d2 = new Download();
Download d3 = new Download();
new Thread(d1).start();
new Thread(d2).start();
new Thread(d2).start(); } static Boolean DoTask(DownloadPage dp) throws Exception {
System.out.println(Thread.currentThread().getName()+"进入进程");
if (dp == null) {
return true;
}
String DownloadHtmlUrl = dp.RemoteUrl;
CacheStorage cs = null;
cs = dict.get(DownloadHtmlUrl);
if (cs == null) {
CacheStorage ct = null;
ct = new CacheStorage();
ct.Local = "";
ct.status = "";
ct.Url = DownloadHtmlUrl;
dict.put(DownloadHtmlUrl, ct);
System.out.println("ERR1");
return false;
}
if (cs.status.equals("done")) {
return true;
}
if (cs.status == "downloading") {
return false;
}
if (cs.status == "pending") {
return false;
}
for (String u : dp.DependencyUrls) { CacheStorage csDependency = dict.get(u);
if (csDependency == null) { csDependency = new CacheStorage();
csDependency.Local = "";
csDependency.status = "";
csDependency.Url = u;
dict.put(u, csDependency);
if (csDependency.status != "downloadng") {
csDependency.status = "downloading";
Thread.sleep(2000);
System.out.println("Downloading" + csDependency.Url);
csDependency.status = "done";
csDependency.Local = "Download local file folder"; return false;
} return false;
}
if (csDependency.status != "done") { cs.status = "pending";
DownloadPage down = new DownloadPage();
down.RemoteUrl = csDependency.Url;
taskQueue.offer(down);
return false;
}
}
cs.status = "downloading";
System.out.println("Downloading" + cs.Url);
Thread.sleep(2000);
cs.status = "done";
cs.Local = "Download local file folder";
// System.out.println(cs.Url+"下载完成");
System.out.println(Thread.currentThread().getName()+"离开进程");
return true; } @Override
public void run() {
// TODO Auto-generated method stub
DownloadPage downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "a.html";
downloadPage.DependencyUrls = new String[] { "1,jpg", "2.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "b.html";
downloadPage.DependencyUrls = new String[] { "21,jpg", "22.jpg",
"44.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "c.html";
downloadPage.DependencyUrls = new String[] { "31,jpg", "32.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "d.html";
downloadPage.DependencyUrls = new String[] { "41,jpg", "42.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "e.html";
downloadPage.DependencyUrls = new String[] { "51,jpg", "52.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "f.html";
downloadPage.DependencyUrls = new String[] { "61,jpg", "62.jpg" };
taskQueue.offer(downloadPage);
while (taskQueue.size() != 0) {
try {
DownloadPage dp = taskQueue.poll();
if (dp == null)
break;
if (DoTask(dp) == false) {
/*
* System.out.println("Cannot download" + dp.RemoteUrl +
* "now,will try later");
*/
taskQueue.offer(dp);
} else {
/*
* synchronized (this) { synMethod(dp.RemoteUrl); }
*/ }
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
还有就是:麻烦楼主以后贴代码时把一些注释也写上,要不别人怎么理解你的意思啊。
public String Url;
public String Local;
public String status;//pending, done, downloading, fail
}
这个类做为保持的下载网页的地址和本地的存储地址和状态
public String RemoteUrl;
public String[] DependencyUrls;
}
这个保持单张页面和页面内的所有图片
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.Logger;public class Download implements Runnable { static public Queue<DownloadPage> taskQueue = new ConcurrentLinkedQueue<DownloadPage>();
static public Map<String, CacheStorage> dict = new Hashtable<String, CacheStorage>();
static final ReentrantLock lock = new ReentrantLock(); static Boolean DoTask(DownloadPage dp) throws Exception {
// System.out.println(Thread.currentThread().getName()+"进入进程");
// System.out.println(Thread.currentThread().getName());
if (dp == null) {
return true;
} String DownloadHtmlUrl = dp.RemoteUrl; CacheStorage cs = null;
cs = dict.get(DownloadHtmlUrl);
if (cs == null) { CacheStorage ct = null;
ct = new CacheStorage();
ct.Local = "";
ct.status = "";
ct.Url = DownloadHtmlUrl;
dict.put(DownloadHtmlUrl, ct);
System.out.println("ERR1");
return false;
}
if (cs.status.equals("done")) {
return true;
}
if (cs.status == "downloading") {
return false;
}
if (cs.status == "pending") {
return false;
}
System.out.println("for");
for (String u : dp.DependencyUrls) {
// System.out.println(u);
CacheStorage csDependency = dict.get(u);
if (csDependency == null) {
CacheStorage ct = null;
ct = new CacheStorage();
ct.status="downloading";
ct.Url=u;
System.out.println("Downloading" + ct.Url);
dict.put(u, ct);
;
//System.out.println(ct.Url + "下载完成");
return false;
}
if (csDependency.status != "done") { cs.status = "pending"; DownloadPage down = new DownloadPage();
down.RemoteUrl = csDependency.Url;
taskQueue.offer(down);
return false;
}
} cs.status = "downloading"; System.out.println("Downloading" + cs.Url);
try {
Thread.sleep(2000); } catch (Exception e) {
// TODO: handle exception
}
System.out.println("for");
cs.status = "done";
cs.Local = "Download local file folder";
System.out.println(cs.Url + "下载完成"); // System.out.println(Thread.currentThread().getName()+"离开进程"); return true; } @Override
public void run() {
// TODO Auto-generated method stub DownloadPage downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "a.html";
downloadPage.DependencyUrls = new String[] { "1,jpg", "2.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "b.html";
downloadPage.DependencyUrls = new String[] { "21,jpg", "22.jpg",
"44.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "c.html";
downloadPage.DependencyUrls = new String[] { "31,jpg", "32.jpg" };
taskQueue.offer(downloadPage);
downloadPage = new DownloadPage();
downloadPage.RemoteUrl = "d.html";
downloadPage.DependencyUrls = new String[] { "41,jpg", "42.jpg" };
taskQueue.offer(downloadPage); while (taskQueue.size() != 0) {
try { DownloadPage dp = taskQueue.poll(); if (dp == null)
break;
if (DoTask(dp) == false) {
/*
* System.out.println("Cannot download" + dp.RemoteUrl +
* "now,will try later");
*/ taskQueue.offer(dp); } else {
/*
* synchronized (this) { synMethod(dp.RemoteUrl); }
*/ }
} catch (Exception e) {
e.printStackTrace();
} }
}
}
这个是线程的处理public class Threadhtml {
public static void main(String[] args) {
Download d1 = new Download();
Download d2 = new Download();
Download d3 = new Download();
new Thread(d1).start();
new Thread(d2).start();
new Thread(d3).start();
new Thread(d3).start();
}
}
在你的while(taskQueue.size() != 0){
if (DoTask(dp) == false) {
/*
* System.out.println("Cannot download" + dp.RemoteUrl +
* "now,will try later");
*/taskQueue.offer(dp);}}
语句中,假设我们取了队列里的最后一个元素,然后这个元素DoTask()返回false,然后又把它加入队列,然后取出来DoTask,返回false,加入队列……
而最终造成DoTask()返回false的语句,几乎都是
if (cs.status.equals("downloading")) {
return false;
}
和
if (cs.status.equals("pending")) {
return false;
}
具体的逻辑其实我还不是很清楚,但你可以参照以上原因再想想~
我想了想,感觉应该是正在downloading的那条线程sleep后一直没有获得CPU使用权,其他线程则把任务看做“pending”或“downloading”,也无法处理,这样才造成的死锁。