From 1778fb420d8431f0a316150e098c432c0a146101 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 19 Oct 2008 18:10:42 +0000 Subject: [PATCH] - added some performance tweaks to the new BLOB buffer - removed the now superfluous HT storage thread - reduced number of file decompression by shifting the compression moment to the future git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5286 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- build.properties | 2 +- defaults/performance_dht.profile | 3 -- defaults/yacy.init | 3 -- htroot/ProxyIndexingMonitor_p.java | 11 ++-- source/de/anomic/crawler/ProtocolLoader.java | 7 +-- source/de/anomic/crawler/ZURL.java | 1 + source/de/anomic/http/httpdProxyHandler.java | 54 +++++++++---------- source/de/anomic/icap/icapd.java | 16 +++--- .../de/anomic/kelondro/kelondroBLOBArray.java | 13 ++--- .../anomic/kelondro/kelondroBLOBBuffer.java | 42 ++++++--------- source/de/anomic/plasma/plasmaHTCache.java | 32 +++-------- .../de/anomic/plasma/plasmaSnippetCache.java | 11 ++-- .../de/anomic/plasma/plasmaSwitchboard.java | 44 ++++----------- .../plasma/plasmaSwitchboardConstants.java | 14 ----- 14 files changed, 90 insertions(+), 163 deletions(-) diff --git a/build.properties b/build.properties index 37585ee80..baa9dbb8a 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5 # Release Configuration -releaseVersion=0.611 +releaseVersion=0.612 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz diff --git a/defaults/performance_dht.profile b/defaults/performance_dht.profile index d174d27a6..322283f9d 100644 --- a/defaults/performance_dht.profile +++ b/defaults/performance_dht.profile @@ -25,9 +25,6 @@ 62_remotetriggeredcrawl_busysleep=1000 62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_isPaused=false -70_cachemanager_idlesleep=1000 -70_cachemanager_busysleep=1 -70_cachemanager_memprereq=1048576 80_indexing_idlesleep=1000 80_indexing_busysleep=100 80_indexing_memprereq=6291456 diff --git a/defaults/yacy.init b/defaults/yacy.init index aac717f50..972713e2d 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -558,9 +558,6 @@ performanceSpeed=100 62_remotetriggeredcrawl_busysleep=1000 62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_isPaused=false -70_cachemanager_idlesleep=1000 -70_cachemanager_busysleep=0 -70_cachemanager_memprereq=1048576 80_indexing_idlesleep=1000 80_indexing_busysleep=10 80_indexing_memprereq=6291456 diff --git a/htroot/ProxyIndexingMonitor_p.java b/htroot/ProxyIndexingMonitor_p.java index 8702bddf3..f054553f8 100644 --- a/htroot/ProxyIndexingMonitor_p.java +++ b/htroot/ProxyIndexingMonitor_p.java @@ -31,6 +31,7 @@ import java.io.File; import java.io.IOException; import de.anomic.http.httpRequestHeader; +import de.anomic.plasma.plasmaHTCache; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.server.serverObjects; @@ -88,11 +89,11 @@ public class ProxyIndexingMonitor_p { if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs(); // proxyCacheSize - oldProxyCacheSize = getStringLong(env.getConfig("proxyCacheSize", "64")); - newProxyCacheSize = getStringLong(post.get("proxyCacheSize", "64")); + oldProxyCacheSize = getStringLong(env.getConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "64")); + newProxyCacheSize = getStringLong(post.get(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "64")); if (getLong(newProxyCacheSize) < 4) { newProxyCacheSize = "4"; } - env.setConfig("proxyCacheSize", newProxyCacheSize); - sb.setCacheSize(Long.parseLong(newProxyCacheSize)); + env.setConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize); + plasmaHTCache.setCacheSize(Long.parseLong(newProxyCacheSize) * 1024 * 1024); // implant these settings also into the crawling profile for the proxy if (sb.webIndex.defaultProxyProfile == null) { @@ -152,7 +153,7 @@ public class ProxyIndexingMonitor_p { prop.put("proxyIndexingLocalText", env.getConfig("proxyIndexingLocalText", "").equals("true") ? "1" : "0"); prop.put("proxyIndexingLocalMedia", env.getConfig("proxyIndexingLocalMedia", "").equals("true") ? "1" : "0"); prop.put("proxyCache", env.getConfig(plasmaSwitchboardConstants.HTCACHE_PATH, plasmaSwitchboardConstants.HTCACHE_PATH_DEFAULT)); - prop.put("proxyCacheSize", env.getConfigLong("proxyCacheSize", 64)); + prop.put("proxyCacheSize", env.getConfigLong(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, 64)); // return rewrite properties return prop; } diff --git a/source/de/anomic/crawler/ProtocolLoader.java b/source/de/anomic/crawler/ProtocolLoader.java index 88fb464b5..5be6efdfa 100644 --- a/source/de/anomic/crawler/ProtocolLoader.java +++ b/source/de/anomic/crawler/ProtocolLoader.java @@ -123,9 +123,4 @@ public final class ProtocolLoader { } } -} - - - - - +} \ No newline at end of file diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 4dfc97386..6209df0f7 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -133,6 +133,7 @@ public class ZURL { public synchronized Entry getEntry(final String urlhash) { try { + if (urlIndex == null) return null; final kelondroRow.Entry entry = urlIndex.get(urlhash.getBytes()); if (entry == null) return null; return new Entry(entry); diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 0de8e6603..395743abf 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -98,7 +98,7 @@ public final class httpdProxyHandler { // static variables // can only be instantiated upon first instantiation of this class object - private static plasmaSwitchboard switchboard = null; + private static plasmaSwitchboard sb = null; private static final HashSet yellowList; private static int timeout = 30000; private static boolean yacyTrigger = true; @@ -165,16 +165,16 @@ public final class httpdProxyHandler { theLogger.logSevere("Unable to configure proxy access logging.",e); } - switchboard = plasmaSwitchboard.getSwitchboard(); - if (switchboard != null) { + sb = plasmaSwitchboard.getSwitchboard(); + if (sb != null) { - isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue(); + isTransparentProxy = Boolean.valueOf(sb.getConfig("isTransparentProxy","false")).booleanValue(); // set timeout - timeout = Integer.parseInt(switchboard.getConfig("proxy.clientTimeout", "10000")); + timeout = Integer.parseInt(sb.getConfig("proxy.clientTimeout", "10000")); // create a htRootPath: system pages - htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot")); + htRootPath = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot")); if (!(htRootPath.exists())) { if(!htRootPath.mkdir()) serverLog.logSevere("PROXY", "could not create htRoot "+ htRootPath); @@ -182,10 +182,10 @@ public final class httpdProxyHandler { // load a transformer transformer = new htmlFilterContentTransformer(); - transformer.init(new File(switchboard.getRootPath(), switchboard.getConfig(plasmaSwitchboardConstants.LIST_BLUE, "")).toString()); + transformer.init(new File(sb.getRootPath(), sb.getConfig(plasmaSwitchboardConstants.LIST_BLUE, "")).toString()); // load the yellow-list - final String f = switchboard.getConfig("proxyYellowList", null); + final String f = sb.getConfig("proxyYellowList", null); if (f != null) { yellowList = serverFileUtils.loadList(new File(f)); theLogger.logConfig("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries"); @@ -193,7 +193,7 @@ public final class httpdProxyHandler { yellowList = new HashSet(); } - final String redirectorPath = switchboard.getConfig("externalRedirector", ""); + final String redirectorPath = sb.getConfig("externalRedirector", ""); if (redirectorPath.length() > 0 && redirectorEnabled == false){ try { redirectorProcess=Runtime.getRuntime().exec(redirectorPath); @@ -260,11 +260,11 @@ public final class httpdProxyHandler { path = "$Path" "=" value domain = "$Domain" "=" value */ - if (switchboard.getConfigBool("proxy.monitorCookies", false)) { + if (sb.getConfigBool("proxy.monitorCookies", false)) { if (requestHeader.containsKey(httpRequestHeader.COOKIE)) { final Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpRequestHeader.COOKIE)}; - synchronized(switchboard.outgoingCookies) { - switchboard.outgoingCookies.put(targethost, entry); + synchronized(sb.outgoingCookies) { + sb.outgoingCookies.put(targethost, entry); } } } @@ -286,11 +286,11 @@ public final class httpdProxyHandler { | "Secure" | "Version" "=" 1*DIGIT */ - if (switchboard.getConfigBool("proxy.monitorCookies", false)) { + if (sb.getConfigBool("proxy.monitorCookies", false)) { if (respondHeader.containsKey(httpResponseHeader.SET_COOKIE)) { final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpResponseHeader.SET_COOKIE)}; - synchronized(switchboard.incomingCookies) { - switchboard.incomingCookies.put(serverhost, entry); + synchronized(sb.incomingCookies) { + sb.incomingCookies.put(serverhost, entry); } } } @@ -310,7 +310,7 @@ public final class httpdProxyHandler { final Date requestDate = new Date(); // remember the time... conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); - switchboard.proxyLastAccess = System.currentTimeMillis(); + sb.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes (needed for the logfile) countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); @@ -422,7 +422,7 @@ public final class httpdProxyHandler { requestHeader, cachedResponseHeader, null, // initiator - switchboard.webIndex.defaultProxyProfile // profile + sb.webIndex.defaultProxyProfile // profile ); plasmaHTCache.storeMetadata(cachedResponseHeader, cacheEntry); // TODO: check if this storeMetadata is necessary @@ -536,7 +536,7 @@ public final class httpdProxyHandler { requestHeader, responseHeader, null, - switchboard.webIndex.defaultProxyProfile + sb.webIndex.defaultProxyProfile ); plasmaHTCache.storeMetadata(responseHeader, cacheEntry); @@ -622,7 +622,7 @@ public final class httpdProxyHandler { // totally fresh file //cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert cacheEntry.setCacheArray(cacheArray); - plasmaHTCache.push(cacheEntry); + sb.htEntryStoreProcess(cacheEntry); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); } else if (cacheArray != null && sizeBeforeDelete == cacheArray.length) { // before we came here we deleted a cache entry @@ -634,7 +634,7 @@ public final class httpdProxyHandler { // before we came here we deleted a cache entry //cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD; cacheEntry.setCacheArray(cacheArray); - plasmaHTCache.push(cacheEntry); // necessary update, write response header to cache + sb.htEntryStoreProcess(cacheEntry); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); } } else { @@ -858,7 +858,7 @@ public final class httpdProxyHandler { final Date requestDate = new Date(); // remember the time... conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); - switchboard.proxyLastAccess = System.currentTimeMillis(); + sb.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); @@ -958,7 +958,7 @@ public final class httpdProxyHandler { final Date requestDate = new Date(); // remember the time... conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime())); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); - switchboard.proxyLastAccess = System.currentTimeMillis(); + sb.proxyLastAccess = System.currentTimeMillis(); // using an ByteCount OutputStream to count the send bytes countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); @@ -1252,7 +1252,7 @@ public final class httpdProxyHandler { */ private static void addXForwardedForHeader(final Properties conProp, final httpRequestHeader requestHeader) { // setting the X-Forwarded-For Header - if (switchboard.getConfigBool("proxy.sendXForwardedForHeader", true)) { + if (sb.getConfigBool("proxy.sendXForwardedForHeader", true)) { requestHeader.put(httpHeader.X_FORWARDED_FOR, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP)); } } @@ -1294,7 +1294,7 @@ public final class httpdProxyHandler { } private static void setViaHeader(final httpHeader header, final String httpVer) { - if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return; + if (!sb.getConfigBool("proxy.sendViaHeader", true)) return; final String myAddress = (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().myAlternativeAddress(); if (myAddress != null) { @@ -1307,7 +1307,7 @@ public final class httpdProxyHandler { viaValue .append(httpVer).append(" ") .append(myAddress).append(" ") - .append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")"); + .append("(YaCy ").append(sb.getConfig("vString", "0.0")).append(")"); // storing header back header.put(httpHeader.VIA, new String(viaValue)); @@ -1316,7 +1316,7 @@ public final class httpdProxyHandler { public static void doConnect(final Properties conProp, final httpRequestHeader requestHeader, final InputStream clientIn, final OutputStream clientOut) throws IOException { - switchboard.proxyLastAccess = System.currentTimeMillis(); + sb.proxyLastAccess = System.currentTimeMillis(); String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); final String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); @@ -1666,7 +1666,7 @@ public final class httpdProxyHandler { userAgentStr .append(browserUserAgent.substring(0,pos)) .append("; YaCy ") - .append(switchboard.getConfig("vString","0.1")) + .append(sb.getConfig("vString","0.1")) .append("; yacy.net") .append(browserUserAgent.substring(pos)); } else { diff --git a/source/de/anomic/icap/icapd.java b/source/de/anomic/icap/icapd.java index 5859b1cec..d0da3fd87 100644 --- a/source/de/anomic/icap/icapd.java +++ b/source/de/anomic/icap/icapd.java @@ -71,16 +71,16 @@ public class icapd implements serverHandler, Cloneable { // needed for logging private final serverLog log = new serverLog("ICAPD"); - private static plasmaSwitchboard switchboard = null; + private static plasmaSwitchboard sb = null; private static String virtualHost = null; private static boolean keepAliveSupport = true; public icapd() { - if (switchboard == null) { - switchboard = plasmaSwitchboard.getSwitchboard(); - virtualHost = switchboard.getConfig("fileHost","localhost"); + if (sb == null) { + sb = plasmaSwitchboard.getSwitchboard(); + virtualHost = sb.getConfig("fileHost","localhost"); } } @@ -124,9 +124,9 @@ public class icapd implements serverHandler, Cloneable { public icapHeader getDefaultHeaders() { final icapHeader newHeaders = new icapHeader(); - newHeaders.put(icapHeader.SERVER,"YaCy/" + switchboard.getConfig("vString","")); + newHeaders.put(icapHeader.SERVER,"YaCy/" + sb.getConfig("vString","")); newHeaders.put(icapHeader.DATE, HttpClient.dateString(new Date())); - newHeaders.put(icapHeader.ISTAG, "\"" + switchboard.getConfig("vString","") + "\""); + newHeaders.put(icapHeader.ISTAG, "\"" + sb.getConfig("vString","") + "\""); return newHeaders; } @@ -384,7 +384,7 @@ public class icapd implements serverHandler, Cloneable { httpRespStatusLine, httpReqHeader, httpResHeader, null, - switchboard.webIndex.defaultProxyProfile + sb.webIndex.defaultProxyProfile ); // copy the response body into the file @@ -397,7 +397,7 @@ public class icapd implements serverHandler, Cloneable { plasmaHTCache.storeMetadata(httpResHeader, cacheEntry); // indexing the response - plasmaHTCache.push(cacheEntry); + sb.htEntryStoreProcess(cacheEntry); } catch (final Exception e) { e.printStackTrace(); } diff --git a/source/de/anomic/kelondro/kelondroBLOBArray.java b/source/de/anomic/kelondro/kelondroBLOBArray.java index e203e261b..6f36c870e 100755 --- a/source/de/anomic/kelondro/kelondroBLOBArray.java +++ b/source/de/anomic/kelondro/kelondroBLOBArray.java @@ -67,14 +67,13 @@ public class kelondroBLOBArray implements kelondroBLOB { public kelondroBLOBArray( final File heapLocation, - final int keylength, final kelondroByteOrder ordering, - long agelimit, long sizelimit - ) throws IOException { + final int keylength, + final kelondroByteOrder ordering) throws IOException { this.keylength = keylength; this.ordering = ordering; this.heapLocation = heapLocation; - this.fileAgeLimit = agelimit; - this.fileSizeLimit = sizelimit; + this.fileAgeLimit = oneMonth; + this.fileSizeLimit = oneGigabyte; this.repositoryAgeMax = Long.MAX_VALUE; this.repositorySizeMax = Long.MAX_VALUE; @@ -111,10 +110,12 @@ public class kelondroBLOBArray implements kelondroBLOB { public void setMaxAge(long maxAge) { this.repositoryAgeMax = maxAge; + this.fileAgeLimit = Math.min(oneMonth, maxAge / 10); } public void setMaxSize(long maxSize) { this.repositorySizeMax = maxSize; + this.fileSizeLimit = Math.min(oneGigabyte, maxSize / 10); } private void executeLimits() { @@ -311,7 +312,7 @@ public class kelondroBLOBArray implements kelondroBLOB { final File f = new File("/Users/admin/blobarraytest"); try { //f.delete(); - final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder, oneMonth, oneGigabyte); + final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder); heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); diff --git a/source/de/anomic/kelondro/kelondroBLOBBuffer.java b/source/de/anomic/kelondro/kelondroBLOBBuffer.java index 68dac46a1..9a6d79be3 100644 --- a/source/de/anomic/kelondro/kelondroBLOBBuffer.java +++ b/source/de/anomic/kelondro/kelondroBLOBBuffer.java @@ -136,9 +136,10 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { private byte[] compress(byte[] b) { // compressed a byte array and adds a leading magic for the compression try { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.out.print("/"); // DEBUG + final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5); baos.write(gzipMagic); - final OutputStream os = new GZIPOutputStream(baos, 128); + final OutputStream os = new GZIPOutputStream(baos, 512); os.write(b); os.close(); baos.close(); @@ -150,6 +151,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { } private byte[] markWithPlainMagic(byte[] b) { + System.out.print("+"); // DEBUG byte[] r = new byte[b.length + 2]; r[0] = plainMagic[0]; r[1] = plainMagic[1]; @@ -160,6 +162,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { private byte[] decompress(byte[] b) { // use a magic in the head of the bytes to identify compression type if (kelondroByteArray.equals(b, gzipMagic)) { + System.out.print("\\"); // DEBUG ByteArrayInputStream bais = new ByteArrayInputStream(b); // eat up the magic bais.read(); @@ -168,7 +171,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { InputStream gis; try { gis = new GZIPInputStream(bais); - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length); final byte[] buf = new byte[1024]; int n; while ((n = gis.read(buf)) > 0) baos.write(buf, 0, n); @@ -182,6 +185,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { return null; } } else if (kelondroByteArray.equals(b, plainMagic)) { + System.out.print("-"); // DEBUG byte[] r = new byte[b.length - 2]; System.arraycopy(b, 2, r, 0, b.length - 2); return r; @@ -286,20 +290,12 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { // check if the buffer is full or could be full after this write if (this.queueLength + b.length * 2 > this.maxCacheSize) flushAll(); - - // depending on execution cases, write into different queues - if (!this.executing && this.compress) { - // files are compressed when they arrive and written to the compressed queue - byte[] bb = compress(b); - this.compressedQueue.add(new Entry(key, bb)); - this.queueLength += bb.length; - } else { - // files are written uncompressed to the uncompressed-queue - // they are either written uncompressed to the database - // or compressed with the concurrent thread and written later - this.rawQueue.add(new Entry(key, b)); - this.queueLength += b.length; - } + + // files are written uncompressed to the uncompressed-queue + // they are either written uncompressed to the database + // or compressed with the concurrent thread and written later + this.rawQueue.add(new Entry(key, b)); + this.queueLength += b.length; } public synchronized void remove(byte[] key) throws IOException { @@ -322,15 +318,11 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { } private boolean flushOne(boolean block) throws IOException { - if (!this.executing && this.compress) { + if (rawQueue.size() > 0) { // files are compressed when they arrive and written to the compressed queue - return flushOneCompressed(block); + return flushOneRaw(block); } else { - // files are written uncompressed to the uncompressed-queue - // they are either written uncompressed to the database - // or compressed with the concurrent thread and written later - if (flushOneRaw(block)) return true; - else return flushOneCompressed(block); + return flushOneCompressed(block); } } @@ -371,7 +363,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB { } private void flushAll() throws IOException { - while ((this.rawQueue != null && this.rawQueue.size() > 0) || + while (this.rawQueue.size() > 0 || (this.compressedQueue != null && this.compressedQueue.size() > 0)) { if (!flushOne(false)) break; } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 773e219ed..9dfa313db 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentLinkedQueue; import de.anomic.http.httpResponseHeader; import de.anomic.index.indexDocumentMetadata; @@ -59,13 +58,12 @@ public final class plasmaHTCache { public static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap"; public static final String FILE_DB_NAME = "file.array"; - private static final int stackLimit = 150; // if we exceed that limit, we do not check idle public static final long oneday = 1000L * 60L * 60L * 24L; // milliseconds of a day private static kelondroMap responseHeaderDB = null; private static kelondroBLOBBuffer fileDB = null; + private static kelondroBLOBArray fileDBunbuffered = null; - private static final ConcurrentLinkedQueue cacheStack = new ConcurrentLinkedQueue(); public static long maxCacheSize = 0l; public static File cachePath = null; public static final serverLog log = new serverLog("HTCACHE"); @@ -135,9 +133,10 @@ public final class plasmaHTCache { } responseHeaderDB = new kelondroMap(blob, 500); try { - kelondroBLOBArray fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder, kelondroBLOBArray.oneMonth, kelondroBLOBArray.oneGigabyte); - fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 1024 * 1024, true); - fileDB.start(); + fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder); + fileDBunbuffered.setMaxSize(maxCacheSize); + fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 2 * 1024 * 1024, true); + //fileDB.start(); } catch (IOException e) { e.printStackTrace(); } @@ -159,10 +158,6 @@ public final class plasmaHTCache { directory.delete(); } - public static int size() { - return cacheStack.size(); - } - public static int responseHeaderDBSize() { return responseHeaderDB.size(); } @@ -170,14 +165,6 @@ public final class plasmaHTCache { public static long fileDBSize() { return fileDB.length(); } - - public static void push(final indexDocumentMetadata entry) { - cacheStack.add(entry); - } - - public static indexDocumentMetadata pop() { - return cacheStack.poll(); - } /** * This method changes the HTCache size.
@@ -185,6 +172,7 @@ public final class plasmaHTCache { */ public static void setCacheSize(final long newCacheSize) { maxCacheSize = newCacheSize; + fileDBunbuffered.setMaxSize(maxCacheSize); } /** @@ -201,14 +189,6 @@ public final class plasmaHTCache { fileDB.close(); } - public static boolean full() { - return (cacheStack.size() > stackLimit); - } - - public static boolean empty() { - return (cacheStack.size() == 0); - } - public static boolean isPicture(final String mimeType) { if (mimeType == null) return false; return mimeType.toUpperCase().startsWith("IMAGE"); diff --git a/source/de/anomic/plasma/plasmaSnippetCache.java b/source/de/anomic/plasma/plasmaSnippetCache.java index f0668366e..733dbce20 100644 --- a/source/de/anomic/plasma/plasmaSnippetCache.java +++ b/source/de/anomic/plasma/plasmaSnippetCache.java @@ -65,9 +65,9 @@ public class plasmaSnippetCache { public static final int ERROR_PARSER_NO_LINES = 15; public static final int ERROR_NO_MATCH = 16; - private static int snippetsScoreCounter = 0; + private static int snippetsScoreCounter = 0; private static kelondroMScoreCluster snippetsScore = null; - private static final HashMap snippetsCache = new HashMap(); + private static final HashMap snippetsCache = new HashMap(); /** * a cache holding URLs to favicons specified by the page content, e.g. by using the html link-tag. e.g. @@ -78,13 +78,16 @@ public class plasmaSnippetCache { private static final HashMap faviconCache = new HashMap(); private static plasmaParser parser = null; private static serverLog log = null; + private static plasmaSwitchboard sb = null; public static void init( final plasmaParser parserx, - final serverLog logx + final serverLog logx, + final plasmaSwitchboard switchboard ) { parser = parserx; log = logx; + sb = switchboard; snippetsScoreCounter = 0; snippetsScore = new kelondroMScoreCluster(); snippetsCache.clear(); @@ -290,7 +293,7 @@ public class plasmaSnippetCache { // getting resource metadata (e.g. the http headers for http resources) if (entry != null) { // place entry on crawl queue - plasmaHTCache.push(entry); + sb.htEntryStoreProcess(entry); // read resource body (if it is there) final byte []resourceArray = entry.cacheArray(); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index f8b7e7829..3bad52e3a 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -544,7 +544,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch - * @param newCacheSize in MB - */ - public final void setCacheSize(final long newCacheSize) { - plasmaHTCache.setCacheSize(1048576 * newCacheSize); - } - public boolean onlineCaution() { return (System.currentTimeMillis() - this.proxyLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, "30000"))) || @@ -1081,15 +1070,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitchpublic static final String PROXY_CACHE_ENQUEUE = "70_cachemanager"

- *

Name of the proxy cache enqueue thread which fetches a given website and saves the site itself as well as it's - * HTTP-headers in the HTCACHE

- * - * @see plasmaSwitchboard#PROXY_CACHE_PATH - */ - public static final String PROXY_CACHE_ENQUEUE = "70_cachemanager"; - public static final String PROXY_CACHE_ENQUEUE_METHOD_START = "htEntryStoreJob"; - public static final String PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT = "htEntrySize"; - public static final String PROXY_CACHE_ENQUEUE_METHOD_FREEMEM = null; - public static final String PROXY_CACHE_ENQUEUE_IDLESLEEP = "70_cachemanager_idlesleep"; - public static final String PROXY_CACHE_ENQUEUE_BUSYSLEEP = "70_cachemanager_busysleep"; // 74_parsing /** *

public static final String INDEXER = "80_indexing"