- can be used to import the crawling queue (noticeUrlDB + stacks) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1518 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
theli
19 years ago
parent
4fa2be73c3
commit
50d85657b8
@ -0,0 +1,212 @@
|
||||
package de.anomic.plasma.dbImport;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import de.anomic.plasma.plasmaCrawlNURL;
|
||||
import de.anomic.plasma.plasmaCrawlProfile;
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.plasma.plasmaCrawlNURL.Entry;
|
||||
|
||||
public class plasmaCrawlNURLImporter extends AbstractImporter implements
|
||||
dbImporter {
|
||||
|
||||
private HashSet importProfileHandleCache = new HashSet();
|
||||
private plasmaCrawlProfile importProfileDB;
|
||||
private plasmaCrawlNURL importNurlDB;
|
||||
private int importStartSize;
|
||||
private int urlCount = 0;
|
||||
private int profileCount = 0;
|
||||
|
||||
public plasmaCrawlNURLImporter(plasmaSwitchboard theSb) {
|
||||
super(theSb);
|
||||
this.jobType="NURL";
|
||||
}
|
||||
|
||||
public long getEstimatedTime() {
|
||||
return (this.urlCount==0)?0:((this.importStartSize*getElapsedTime())/(this.urlCount))-getElapsedTime();
|
||||
}
|
||||
|
||||
public String getJobName() {
|
||||
return this.importPath.toString();
|
||||
}
|
||||
|
||||
public int getProcessingStatusPercent() {
|
||||
return (this.urlCount)/((this.importStartSize<100)?1:(this.importStartSize)/100);
|
||||
}
|
||||
|
||||
public String getStatus() {
|
||||
StringBuffer theStatus = new StringBuffer();
|
||||
|
||||
theStatus.append("#URLs=").append(this.urlCount).append("\n");
|
||||
theStatus.append("#Profiles=").append(this.profileCount);
|
||||
|
||||
return theStatus.toString();
|
||||
}
|
||||
|
||||
public void init(File theImportPath, int theCacheSize) {
|
||||
super.init(theImportPath);
|
||||
this.cacheSize = theCacheSize;
|
||||
|
||||
File noticeUrlDbFile = new File(this.importPath,"urlNotice1.db");
|
||||
File profileDbFile = new File(this.importPath, "crawlProfiles0.db");
|
||||
|
||||
String errorMsg = null;
|
||||
if (!this.importPath.exists())
|
||||
errorMsg = "The import path '" + this.importPath + "' does not exist.";
|
||||
else if (!this.importPath.isDirectory())
|
||||
errorMsg = "The import path '" + this.importPath + "' is not a directory.";
|
||||
else if (!this.importPath.canRead())
|
||||
errorMsg = "The import path '" + this.importPath + "' is not readable.";
|
||||
else if (!this.importPath.canWrite())
|
||||
errorMsg = "The import path '" + this.importPath + "' is not writeable.";
|
||||
|
||||
else if (!noticeUrlDbFile.exists())
|
||||
errorMsg = "The noticeUrlDB file '" + noticeUrlDbFile + "' does not exist.";
|
||||
else if (noticeUrlDbFile.isDirectory())
|
||||
errorMsg = "The noticeUrlDB file '" + noticeUrlDbFile + "' is not a file.";
|
||||
else if (!noticeUrlDbFile.canRead())
|
||||
errorMsg = "The noticeUrlDB file '" + noticeUrlDbFile + "' is not readable.";
|
||||
else if (!noticeUrlDbFile.canWrite())
|
||||
errorMsg = "The noticeUrlDB file '" + noticeUrlDbFile + "' is not writeable.";
|
||||
|
||||
else if (!profileDbFile.exists())
|
||||
errorMsg = "The profileDB file '" + profileDbFile + "' does not exist.";
|
||||
else if (profileDbFile.isDirectory())
|
||||
errorMsg = "The profileDB file '" + profileDbFile + "' is not a file.";
|
||||
else if (!profileDbFile.canRead())
|
||||
errorMsg = "The profileDB file '" + profileDbFile + "' is not readable.";
|
||||
// else if (!profileDbFile.canWrite())
|
||||
// errorMsg = "The profileDB file '" + profileDbFile + "' is not writeable.";
|
||||
|
||||
if (errorMsg != null) {
|
||||
this.log.logSevere(errorMsg);
|
||||
throw new IllegalArgumentException(errorMsg);
|
||||
}
|
||||
|
||||
// init noticeUrlDB
|
||||
this.log.logInfo("Initializing the source noticeUrlDB");
|
||||
this.importNurlDB = new plasmaCrawlNURL(this.importPath, this.cacheSize*(3/4));
|
||||
this.importStartSize = this.importNurlDB.size();
|
||||
int stackSize = this.importNurlDB.stackSize();
|
||||
|
||||
// init profile DB
|
||||
this.log.logInfo("Initializing the source profileDB");
|
||||
this.importProfileDB = new plasmaCrawlProfile(profileDbFile,this.cacheSize*(1/3));
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
// waiting on init thread to finish
|
||||
this.importNurlDB.waitOnInitThread();
|
||||
|
||||
// the stack types we want to import
|
||||
int[] stackTypes = new int[] {plasmaCrawlNURL.STACK_TYPE_CORE,
|
||||
plasmaCrawlNURL.STACK_TYPE_LIMIT,
|
||||
plasmaCrawlNURL.STACK_TYPE_REMOTE,
|
||||
-1};
|
||||
|
||||
// looping through the various stacks
|
||||
for (int i=0; i< stackTypes.length; i++) {
|
||||
if (stackTypes[i] != -1) {
|
||||
this.log.logInfo("Starting to import stacktype '" + stackTypes[i] + "' containing '" + this.importNurlDB.stackSize(stackTypes[i]) + "' entries.");
|
||||
} else {
|
||||
this.log.logInfo("Starting to import '" + this.importNurlDB.size() + "' entries not available in any stack.");
|
||||
}
|
||||
|
||||
// getting an interator and loop through the URL entries
|
||||
Iterator iter = (stackTypes[i] == -1)?this.importNurlDB.urlHashes("------------", true):null;
|
||||
while (true) {
|
||||
|
||||
String nextHash = null;
|
||||
Entry urlEntry = null;
|
||||
|
||||
try {
|
||||
if (stackTypes[i] != -1) {
|
||||
if (this.importNurlDB.stackSize(stackTypes[i]) == 0) break;
|
||||
|
||||
this.urlCount++;
|
||||
urlEntry = this.importNurlDB.pop(stackTypes[i]);
|
||||
nextHash = urlEntry.hash();
|
||||
} else {
|
||||
if (!iter.hasNext()) break;
|
||||
|
||||
this.urlCount++;
|
||||
nextHash = (String)iter.next();
|
||||
urlEntry = this.importNurlDB.getEntry(nextHash);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
this.log.logWarning("Unable to import entry: " + e.toString());
|
||||
|
||||
if ((stackTypes[i] != -1) &&(this.importNurlDB.stackSize(stackTypes[i]) == 0)) break;
|
||||
continue;
|
||||
}
|
||||
|
||||
// getting a handler to the crawling profile the url belongs to
|
||||
try {
|
||||
String profileHandle = urlEntry.profileHandle();
|
||||
if (profileHandle == null) {
|
||||
this.log.logWarning("Profile handle of url entry '" + nextHash + "' unknown.");
|
||||
continue;
|
||||
}
|
||||
|
||||
// if we havn't imported the profile until yet we need to do it now
|
||||
if (!this.importProfileHandleCache.contains(profileHandle)) {
|
||||
|
||||
// testing if the profile is already known
|
||||
plasmaCrawlProfile.entry profileEntry = this.sb.profiles.getEntry(profileHandle);
|
||||
|
||||
// if not we need to import it
|
||||
if (profileEntry == null) {
|
||||
// copy and store the source profile entry into the destination db
|
||||
plasmaCrawlProfile.entry sourceEntry = this.importProfileDB.getEntry(profileHandle);
|
||||
if (sourceEntry != null) {
|
||||
this.profileCount++;
|
||||
this.importProfileHandleCache.add(profileHandle);
|
||||
this.sb.profiles.newEntry((TreeMap)((TreeMap)sourceEntry.map()).clone());
|
||||
} else {
|
||||
this.log.logWarning("Profile '" + profileHandle + "' of url entry '" + nextHash + "' unknown.");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if the url does not alredy exists in the destination stack we insert it now
|
||||
if (!this.sb.urlPool.noticeURL.existsInStack(nextHash)) {
|
||||
this.sb.urlPool.noticeURL.newEntry(urlEntry,(stackTypes[i] != -1)?stackTypes[i]:plasmaCrawlNURL.STACK_TYPE_CORE);
|
||||
}
|
||||
|
||||
// removing hash from the import db
|
||||
} finally {
|
||||
this.importNurlDB.remove(nextHash);
|
||||
}
|
||||
|
||||
if (this.urlCount % 100 == 0) {
|
||||
this.log.logFine(this.urlCount + " URLs and '" + this.profileCount + "' profile entries processed so far.");
|
||||
}
|
||||
if (this.isAborted()) break;
|
||||
}
|
||||
this.log.logInfo("Finished to import stacktype '" + stackTypes[i] + "'");
|
||||
}
|
||||
|
||||
int size = this.importNurlDB.size();
|
||||
int stackSize = this.importNurlDB.stackSize();
|
||||
|
||||
// TODO: what todo with nurlDB entries that do not exist in any stack?
|
||||
|
||||
} catch (Exception e) {
|
||||
this.error = e.toString();
|
||||
this.log.logSevere("Import process had detected an error",e);
|
||||
} finally {
|
||||
this.log.logInfo("Import process finished.");
|
||||
this.globalEnd = System.currentTimeMillis();
|
||||
this.sb.dbImportManager.finishedJobs.add(this);
|
||||
this.importNurlDB.close();
|
||||
this.importProfileDB.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue