boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) plasmaSwitchboard.java

Browse Source 
adding more verbose logging output for db initialization
*) httpdFileHandler.java
   adding cache for servlet response methods


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@897 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 20 years ago
parent e3a586d7bd
commit 1688be8590
3 changed files with 78 additions and 21 deletions
Show Stats Download Patch File Download Diff File

4
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -92,6 +92,10 @@ public class wikiBoard {
return datbase.size() + bkpbase.size();
}
public int size() {
return datbase.size();
}
public int[] dbCacheChunkSize() {
int[] db = datbase.cacheChunkSize();
int[] bk = bkpbase.cacheChunkSize();

32
source/de/anomic/http/httpdFileHandler.java
Unescape View File

@ -130,12 +130,16 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
* Template Cache
* @param switchboard
*/
private static final HashMap templateCache = new HashMap();
private static final HashMap templateCache;
private static final HashMap templateMethodCache;
public static boolean useTemplateCache = false;
static {
useTemplateCache = plasmaSwitchboard.getSwitchboard().getConfig("enableTemplateCache","true").equalsIgnoreCase("true");
templateCache = (useTemplateCache)? new HashMap() : new HashMap(0);
templateMethodCache = (useTemplateCache) ? new HashMap() : new HashMap(0);
// create a class loader
provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
@ -709,10 +713,26 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
}
}
private Method rewriteMethod(File classFile) {
private final Method rewriteMethod(File classFile) {
Method m = null;
long start = System.currentTimeMillis();
// now make a class out of the stream
try {
if (useTemplateCache) {
SoftReference ref = (SoftReference) templateMethodCache.get(classFile);
if (ref != null) {
m = (Method) ref.get();
if (m == null) {
templateMethodCache.remove(classFile);
} else {
this.theLogger.logFine("Cache HIT for file " + classFile);
return m;
}
}
}
//System.out.println("**DEBUG** loading class file " + classFile);
Class c = provider.loadClass(classFile);
Class[] params = new Class[] {
@ -720,6 +740,14 @@ public final class httpdFileHandler extends httpdAbstractHandler implements http
Class.forName("de.anomic.server.serverObjects"),
Class.forName("de.anomic.server.serverSwitch")};
m = c.getMethod("respond", params);
if (useTemplateCache) {
// storing the method into the cache
SoftReference ref = new SoftReference(m);
templateMethodCache.put(classFile,ref);
this.theLogger.logFine("Cache MISS for file " + classFile);
}
} catch (ClassNotFoundException e) {
System.out.println("INTERNAL ERROR: class " + classFile + " is missing:" + e.getMessage());
} catch (NoSuchMethodException e) {

61
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -205,8 +205,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
setLog(new serverLog("PLASMA"));
// load values from configs
plasmaPath = new File(rootPath, getConfig("dbPath", "PLASMADB"));
listsPath = new File(rootPath, getConfig("listsPath", "LISTS"));
this.plasmaPath = new File(rootPath, getConfig("dbPath", "PLASMADB"));
this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString());
this.listsPath = new File(rootPath, getConfig("listsPath", "LISTS"));
this.log.logConfig("Lists Path: " + this.listsPath.toString());
// remote proxy configuration
remoteProxyHost = getConfig("remoteProxyHost", "");
@ -233,23 +235,31 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (blueList == null) {
// read only once upon first instantiation of this class
String f = getConfig("plasmaBlueList", null);
if (f != null) blueList = kelondroMSetTools.loadList(new File(f)); else blueList= new TreeSet();
this.log.logConfig("loaded blue-list from file " + f + ", " + blueList.size() + " entries");
File plasmaBlueListFile = new File(f);
if (f != null) blueList = kelondroMSetTools.loadList(plasmaBlueListFile); else blueList= new TreeSet();
this.log.logConfig("loaded blue-list from file " + plasmaBlueListFile.getName() + ", " +
blueList.size() + " entries, " +
ppRamString(plasmaBlueListFile.length()/1024));
}
// load the black-list / inspired by [AS]
urlBlacklist = new plasmaURLPattern(new File(getRootPath(), getConfig("listsPath", "DATA/LISTS")));
File ulrBlackListFile = new File(getRootPath(), getConfig("listsPath", "DATA/LISTS"));
urlBlacklist = new plasmaURLPattern(ulrBlackListFile);
String f = getConfig("proxyBlackListsActive", null);
if (f != null) {
urlBlacklist.loadLists("black", f, "/");
this.log.logConfig("loaded black-list from file " + f + ", " + urlBlacklist.size() + " entries");
this.log.logConfig("loaded black-list from file " + ulrBlackListFile.getName() + ", " +
urlBlacklist.size() + " entries, " +
ppRamString(ulrBlackListFile.length()/1024));
}
// load stopwords
if (stopwords == null) {
File stopwordsFile = new File(rootPath, "yacy.stopwords");
stopwords = kelondroMSetTools.loadList(stopwordsFile);
this.log.logConfig("loaded stopwords from file " + stopwordsFile + ", " + stopwords.size() + " entries");
this.log.logConfig("loaded stopwords from file " + stopwordsFile.getName() + ", " +
stopwords.size() + " entries, " +
ppRamString(stopwordsFile.length()/1024));
}
// read memory amount
@ -279,7 +289,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
File profilesFile = new File(this.plasmaPath, "crawlProfiles0.db");
this.profiles = new plasmaCrawlProfile(profilesFile, ramProfiles);
initProfiles();
log.logConfig("Loaded profiles from file " + profilesFile +
log.logConfig("Loaded profiles from file " + profilesFile.getName() +
", " + this.profiles.size() + " entries" +
", " + ppRamString(profilesFile.length()/1024));
@ -287,7 +297,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.log.logConfig("Initializing robots.txt DB");
File robotsDBFile = new File(this.plasmaPath, "crawlRobotsTxt.db");
this.robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots);
this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile +
this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() +
", " + this.robots.size() + " entries" +
", " + ppRamString(robotsDBFile.length()/1024));
@ -360,22 +370,37 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.log);
// starting message board
log.logConfig("Starting Message Board");
messageDB = new messageBoard(new File(getRootPath(), "DATA/SETTINGS/message.db"), ramMessage);
this.log.logConfig("Starting Message Board");
File messageDbFile = new File(getRootPath(), "DATA/SETTINGS/message.db");
this.messageDB = new messageBoard(messageDbFile, ramMessage);
this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() +
", " + this.messageDB.size() + " entries" +
", " + ppRamString(messageDbFile.length()/1024));
// starting wiki
log.logConfig("Starting Wiki Board");
wikiDB = new wikiBoard(new File(getRootPath(), "DATA/SETTINGS/wiki.db"),
this.log.logConfig("Starting Wiki Board");
File wikiDbFile = new File(getRootPath(), "DATA/SETTINGS/wiki.db");
this.wikiDB = new wikiBoard(wikiDbFile,
new File(getRootPath(), "DATA/SETTINGS/wiki-bkp.db"), ramWiki);
userDB = new userDB(new File(getRootPath(), "DATA/SETTINGS/user.db"), 512);
this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() +
", " + this.wikiDB.size() + " entries" +
", " + ppRamString(wikiDbFile.length()/1024));
// Init User DB
this.log.logConfig("Loading User DB");
File userDbFile = new File(getRootPath(), "DATA/SETTINGS/user.db");
this.userDB = new userDB(userDbFile, 512);
this.log.logConfig("Loaded User DB from file " + userDbFile.getName() +
", " + this.userDB.size() + " entries" +
", " + ppRamString(userDbFile.length()/1024));
// init cookie-Monitor
log.logConfig("Starting Cookie Monitor");
outgoingCookies = new HashMap();
incomingCookies = new HashMap();
this.log.logConfig("Starting Cookie Monitor");
this.outgoingCookies = new HashMap();
this.incomingCookies = new HashMap();
// clean up profiles
log.logConfig("Cleaning Profiles");
this.log.logConfig("Cleaning Profiles");
cleanProfiles();
// init facility DB

Write Preview
Loading…
Cancel
Save