- adding interface class (plasma/crawler/plasmaCrawlWorker.java) for protocol specific crawl-worker threads
- moving reusable code into abstract crawl-worker class AbstractCrawlWorker.java
- the load method of the worker threads should not be called directly anymore (e.g. by the snippet fetcher)
to crawl a page and wait for the result use function plasmaCrawlLoader.loadSync([...])
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2474 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli
19 years ago
parent
eb9b138986
commit
09b106eb04
@ -0,0 +1,214 @@
|
|||||||
|
package de.anomic.plasma.crawler;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import de.anomic.index.indexURL;
|
||||||
|
import de.anomic.net.URL;
|
||||||
|
import de.anomic.plasma.plasmaCrawlEURL;
|
||||||
|
import de.anomic.plasma.plasmaCrawlLoaderMessage;
|
||||||
|
import de.anomic.plasma.plasmaCrawlProfile;
|
||||||
|
import de.anomic.plasma.plasmaHTCache;
|
||||||
|
import de.anomic.plasma.plasmaSwitchboard;
|
||||||
|
import de.anomic.server.logging.serverLog;
|
||||||
|
import de.anomic.tools.bitfield;
|
||||||
|
import de.anomic.yacy.yacyCore;
|
||||||
|
|
||||||
|
public abstract class AbstractCrawlWorker extends Thread implements plasmaCrawlWorker {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The protocol that is supported by this crawler
|
||||||
|
* e.g. <code>http</code>, <code>ftp</code>, etc.
|
||||||
|
*/
|
||||||
|
protected String protocol;
|
||||||
|
|
||||||
|
/* ============================================================
|
||||||
|
* Variables for thread pool management
|
||||||
|
* ============================================================ */
|
||||||
|
public boolean destroyed = false;
|
||||||
|
protected boolean running = false;
|
||||||
|
protected boolean stopped = false;
|
||||||
|
protected boolean done = false;
|
||||||
|
|
||||||
|
/* ============================================================
|
||||||
|
* Crawl job specific variables
|
||||||
|
* ============================================================ */
|
||||||
|
public plasmaCrawlLoaderMessage theMsg;
|
||||||
|
protected URL url;
|
||||||
|
protected String name;
|
||||||
|
protected String refererURLString;
|
||||||
|
protected String initiator;
|
||||||
|
protected int depth;
|
||||||
|
protected long startdate;
|
||||||
|
protected plasmaCrawlProfile.entry profile;
|
||||||
|
protected boolean acceptAllContent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The crawler thread pool
|
||||||
|
*/
|
||||||
|
protected final plasmaCrawlerPool myPool;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* reference to the plasma switchboard
|
||||||
|
*/
|
||||||
|
protected final plasmaSwitchboard sb;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* reference to the cache manager
|
||||||
|
*/
|
||||||
|
protected final plasmaHTCache cacheManager;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Logging class
|
||||||
|
*/
|
||||||
|
protected final serverLog log;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor of this class
|
||||||
|
* @param theTG the crawl worker thread group
|
||||||
|
* @param thePool the crawl worker thread pool
|
||||||
|
* @param theSb plasma switchboard
|
||||||
|
* @param theCacheManager cache manager
|
||||||
|
* @param theLog server log
|
||||||
|
*/
|
||||||
|
public AbstractCrawlWorker(
|
||||||
|
ThreadGroup theTG,
|
||||||
|
plasmaCrawlerPool thePool,
|
||||||
|
plasmaSwitchboard theSb,
|
||||||
|
plasmaHTCache theCacheManager,
|
||||||
|
serverLog theLog
|
||||||
|
) {
|
||||||
|
super(theTG,plasmaCrawlWorker.threadBaseName + "_created");
|
||||||
|
|
||||||
|
this.myPool = thePool;
|
||||||
|
this.sb = theSb;
|
||||||
|
this.cacheManager = theCacheManager;
|
||||||
|
this.log = theLog;
|
||||||
|
}
|
||||||
|
|
||||||
|
public abstract void close();
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
this.running = true;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// The thread keeps running.
|
||||||
|
while (!this.stopped && !this.isInterrupted() && !this.myPool.isClosed) {
|
||||||
|
if (this.done) {
|
||||||
|
synchronized (this) {
|
||||||
|
// return thread back into pool
|
||||||
|
this.myPool.returnObject(this.protocol,this);
|
||||||
|
|
||||||
|
// We are waiting for a new task now.
|
||||||
|
if (!this.stopped && !this.destroyed && !this.isInterrupted()) {
|
||||||
|
this.wait();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
// executing the new task
|
||||||
|
execute();
|
||||||
|
} finally {
|
||||||
|
reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException ex) {
|
||||||
|
serverLog.logFiner("CRAWLER-POOL","Interruption of thread '" + this.getName() + "' detected.");
|
||||||
|
} finally {
|
||||||
|
if (this.myPool != null && !this.destroyed)
|
||||||
|
this.myPool.invalidateObject(this.protocol,this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void execute() {
|
||||||
|
try {
|
||||||
|
// setting threadname
|
||||||
|
this.setName(plasmaCrawlWorker.threadBaseName + "_" + this.url);
|
||||||
|
|
||||||
|
// load some configuration variables
|
||||||
|
init();
|
||||||
|
|
||||||
|
// loading resource
|
||||||
|
plasmaHTCache.Entry resource = load();
|
||||||
|
|
||||||
|
// store a reference to the result in the message object
|
||||||
|
// this is e.g. needed by the snippet fetcher
|
||||||
|
this.theMsg.setResult(resource);
|
||||||
|
|
||||||
|
} catch (IOException e) {
|
||||||
|
//throw e;
|
||||||
|
} finally {
|
||||||
|
this.done = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void execute(plasmaCrawlLoaderMessage theNewMsg) {
|
||||||
|
synchronized (this) {
|
||||||
|
|
||||||
|
this.theMsg = theNewMsg;
|
||||||
|
|
||||||
|
this.url = theNewMsg.url;
|
||||||
|
this.name = theNewMsg.name;
|
||||||
|
this.refererURLString = theNewMsg.referer;
|
||||||
|
this.initiator = theNewMsg.initiator;
|
||||||
|
this.depth = theNewMsg.depth;
|
||||||
|
this.profile = theNewMsg.profile;
|
||||||
|
this.acceptAllContent = theNewMsg.acceptAllContent;
|
||||||
|
|
||||||
|
this.startdate = System.currentTimeMillis();
|
||||||
|
|
||||||
|
this.done = false;
|
||||||
|
|
||||||
|
if (!this.running) {
|
||||||
|
// if the thread is not running until yet, we need to start it now
|
||||||
|
this.start();
|
||||||
|
} else {
|
||||||
|
// inform the thread about the new crawl job
|
||||||
|
this.notifyAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setStopped(boolean isStopped) {
|
||||||
|
this.stopped = isStopped;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isRunning() {
|
||||||
|
return this.running;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reset() {
|
||||||
|
this.theMsg = null;
|
||||||
|
this.url = null;
|
||||||
|
this.name = null;
|
||||||
|
this.refererURLString = null;
|
||||||
|
this.initiator = null;
|
||||||
|
this.depth = 0;
|
||||||
|
this.startdate = 0;
|
||||||
|
this.profile = null;
|
||||||
|
this.acceptAllContent = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void addURLtoErrorDB(String failreason) {
|
||||||
|
// convert the referrer URL into a hash value
|
||||||
|
String referrerHash = (this.refererURLString==null)?null:indexURL.urlHash(this.refererURLString);
|
||||||
|
|
||||||
|
// create a new errorURL DB entry
|
||||||
|
plasmaCrawlEURL.Entry ee = this.sb.urlPool.errorURL.newEntry(
|
||||||
|
this.url,
|
||||||
|
referrerHash,
|
||||||
|
this.initiator,
|
||||||
|
yacyCore.seedDB.mySeed.hash,
|
||||||
|
this.name,
|
||||||
|
(failreason==null)?"Unknown reason":failreason,
|
||||||
|
new bitfield(indexURL.urlFlagLength)
|
||||||
|
);
|
||||||
|
|
||||||
|
// store the entry
|
||||||
|
ee.store();
|
||||||
|
|
||||||
|
// push it onto the stack
|
||||||
|
this.sb.urlPool.errorURL.stackPushEntry(ee);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
package de.anomic.plasma.crawler;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import de.anomic.plasma.plasmaCrawlLoaderMessage;
|
||||||
|
import de.anomic.plasma.plasmaHTCache;
|
||||||
|
|
||||||
|
|
||||||
|
public interface plasmaCrawlWorker {
|
||||||
|
|
||||||
|
public static final String threadBaseName = "CrawlerWorker";
|
||||||
|
|
||||||
|
public void reset();
|
||||||
|
public void execute();
|
||||||
|
public void execute(plasmaCrawlLoaderMessage theNewMsg);
|
||||||
|
public void init();
|
||||||
|
|
||||||
|
public void close();
|
||||||
|
public plasmaHTCache.Entry load() throws IOException;
|
||||||
|
}
|
||||||
Loading…
Reference in new issue