avoid ConcurrentModificationException in plasmaCrawlerQueues

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4579 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent 9d0af17c5b
commit 7008a218b3

@ -31,11 +31,10 @@ import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import de.anomic.data.robotsParser; import de.anomic.data.robotsParser;
import de.anomic.kelondro.kelondroFlexWidthArray; import de.anomic.kelondro.kelondroFlexWidthArray;
@ -68,7 +67,7 @@ public class plasmaCrawlQueues {
public plasmaCrawlQueues(plasmaSwitchboard sb, File plasmaPath) { public plasmaCrawlQueues(plasmaSwitchboard sb, File plasmaPath) {
this.sb = sb; this.sb = sb;
this.log = new serverLog("CRAWLER"); this.log = new serverLog("CRAWLER");
this.workers = Collections.synchronizedMap(new HashMap<Integer, crawlWorker>()); this.workers = new ConcurrentHashMap<Integer, crawlWorker>();
this.loader = new plasmaProtocolLoader(sb, log); this.loader = new plasmaProtocolLoader(sb, log);
this.remoteCrawlProviderHashes = new ArrayList<String>(); this.remoteCrawlProviderHashes = new ArrayList<String>();
@ -93,8 +92,9 @@ public class plasmaCrawlQueues {
if (noticeURL.existsInStack(hash)) return "crawler"; if (noticeURL.existsInStack(hash)) return "crawler";
if (delegatedURL.exists(hash)) return "delegated"; if (delegatedURL.exists(hash)) return "delegated";
if (errorURL.exists(hash)) return "errors"; if (errorURL.exists(hash)) return "errors";
Iterator<crawlWorker> i = workers.values().iterator(); for (crawlWorker worker: workers.values()) {
while (i.hasNext()) if (i.next().entry.url().hash().equals(hash)) return "worker"; if (worker.entry.url().hash().equals(hash)) return "worker";
}
return null; return null;
} }
@ -112,10 +112,7 @@ public class plasmaCrawlQueues {
if (ee != null) return ee.url(); if (ee != null) return ee.url();
ee = errorURL.getEntry(urlhash); ee = errorURL.getEntry(urlhash);
if (ee != null) return ee.url(); if (ee != null) return ee.url();
Iterator<crawlWorker> i = workers.values().iterator(); for (crawlWorker w: workers.values()) {
crawlWorker w;
while (i.hasNext()) {
w = i.next();
if (w.entry.url().hash().equals(urlhash)) return w.entry.url(); if (w.entry.url().hash().equals(urlhash)) return w.entry.url();
} }
return null; return null;
@ -123,8 +120,9 @@ public class plasmaCrawlQueues {
public void close() { public void close() {
// wait for all workers to finish // wait for all workers to finish
Iterator<crawlWorker> i = workers.values().iterator(); for (crawlWorker w: workers.values()) {
while (i.hasNext()) i.next().interrupt(); w.interrupt();
}
// TODO: wait some more time until all threads are finished // TODO: wait some more time until all threads are finished
noticeURL.close(); noticeURL.close();
errorURL.close(); errorURL.close();
@ -133,13 +131,7 @@ public class plasmaCrawlQueues {
public plasmaCrawlEntry[] activeWorker() { public plasmaCrawlEntry[] activeWorker() {
synchronized (workers) { synchronized (workers) {
plasmaCrawlEntry[] w = new plasmaCrawlEntry[workers.size()]; return workers.values().toArray(new plasmaCrawlEntry[0]);
int i = 0;
Iterator<crawlWorker> j = workers.values().iterator();
while (j.hasNext()) {
w[i++] = j.next().entry;
}
return w;
} }
} }

Loading…
Cancel
Save