boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added some performance tweaks to the new BLOB buffer

Browse Source 
- removed the now superfluous HT storage thread
- reduced number of file decompression by shifting the compression moment to the future


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5286 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 77e41da7d2
commit 1778fb420d
14 changed files with 90 additions and 163 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5 javacTarget=1.5
# Release Configuration # Release Configuration
releaseVersion=0.611 releaseVersion=0.612
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

3
defaults/performance_dht.profile
Unescape View File

@ -25,9 +25,6 @@
62_remotetriggeredcrawl_busysleep=1000 62_remotetriggeredcrawl_busysleep=1000
62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false 62_remotetriggeredcrawl_isPaused=false
70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=1
70_cachemanager_memprereq=1048576
80_indexing_idlesleep=1000 80_indexing_idlesleep=1000
80_indexing_busysleep=100 80_indexing_busysleep=100
80_indexing_memprereq=6291456 80_indexing_memprereq=6291456

3
defaults/yacy.init
Unescape View File

@ -558,9 +558,6 @@ performanceSpeed=100
62_remotetriggeredcrawl_busysleep=1000 62_remotetriggeredcrawl_busysleep=1000
62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false 62_remotetriggeredcrawl_isPaused=false
70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=0
70_cachemanager_memprereq=1048576
80_indexing_idlesleep=1000 80_indexing_idlesleep=1000
80_indexing_busysleep=10 80_indexing_busysleep=10
80_indexing_memprereq=6291456 80_indexing_memprereq=6291456

11
htroot/ProxyIndexingMonitor_p.java
Unescape View File

@ -31,6 +31,7 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import de.anomic.http.httpRequestHeader; import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -88,11 +89,11 @@ public class ProxyIndexingMonitor_p {
if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs(); if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs();
// proxyCacheSize // proxyCacheSize
oldProxyCacheSize = getStringLong(env.getConfig("proxyCacheSize", "64")); oldProxyCacheSize = getStringLong(env.getConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "64"));
newProxyCacheSize = getStringLong(post.get("proxyCacheSize", "64")); newProxyCacheSize = getStringLong(post.get(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "64"));
if (getLong(newProxyCacheSize) < 4) { newProxyCacheSize = "4"; } if (getLong(newProxyCacheSize) < 4) { newProxyCacheSize = "4"; }
env.setConfig("proxyCacheSize", newProxyCacheSize); env.setConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);
sb.setCacheSize(Long.parseLong(newProxyCacheSize)); plasmaHTCache.setCacheSize(Long.parseLong(newProxyCacheSize) * 1024 * 1024);
// implant these settings also into the crawling profile for the proxy // implant these settings also into the crawling profile for the proxy
if (sb.webIndex.defaultProxyProfile == null) { if (sb.webIndex.defaultProxyProfile == null) {
@ -152,7 +153,7 @@ public class ProxyIndexingMonitor_p {
prop.put("proxyIndexingLocalText", env.getConfig("proxyIndexingLocalText", "").equals("true") ? "1" : "0"); prop.put("proxyIndexingLocalText", env.getConfig("proxyIndexingLocalText", "").equals("true") ? "1" : "0");
prop.put("proxyIndexingLocalMedia", env.getConfig("proxyIndexingLocalMedia", "").equals("true") ? "1" : "0"); prop.put("proxyIndexingLocalMedia", env.getConfig("proxyIndexingLocalMedia", "").equals("true") ? "1" : "0");
prop.put("proxyCache", env.getConfig(plasmaSwitchboardConstants.HTCACHE_PATH, plasmaSwitchboardConstants.HTCACHE_PATH_DEFAULT)); prop.put("proxyCache", env.getConfig(plasmaSwitchboardConstants.HTCACHE_PATH, plasmaSwitchboardConstants.HTCACHE_PATH_DEFAULT));
prop.put("proxyCacheSize", env.getConfigLong("proxyCacheSize", 64)); prop.put("proxyCacheSize", env.getConfigLong(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, 64));
// return rewrite properties // return rewrite properties
return prop; return prop;
} }

7
source/de/anomic/crawler/ProtocolLoader.java
Unescape View File

@ -123,9 +123,4 @@ public final class ProtocolLoader {
} }
} }
} }

1
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -133,6 +133,7 @@ public class ZURL {
public synchronized Entry getEntry(final String urlhash) { public synchronized Entry getEntry(final String urlhash) {
try { try {
if (urlIndex == null) return null;
final kelondroRow.Entry entry = urlIndex.get(urlhash.getBytes()); final kelondroRow.Entry entry = urlIndex.get(urlhash.getBytes());
if (entry == null) return null; if (entry == null) return null;
return new Entry(entry); return new Entry(entry);

54
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -98,7 +98,7 @@ public final class httpdProxyHandler {
// static variables // static variables
// can only be instantiated upon first instantiation of this class object // can only be instantiated upon first instantiation of this class object
private static plasmaSwitchboard switchboard = null; private static plasmaSwitchboard sb = null;
private static final HashSet<String> yellowList; private static final HashSet<String> yellowList;
private static int timeout = 30000; private static int timeout = 30000;
private static boolean yacyTrigger = true; private static boolean yacyTrigger = true;
@ -165,16 +165,16 @@ public final class httpdProxyHandler {
theLogger.logSevere("Unable to configure proxy access logging.",e); theLogger.logSevere("Unable to configure proxy access logging.",e);
} }
switchboard = plasmaSwitchboard.getSwitchboard(); sb = plasmaSwitchboard.getSwitchboard();
if (switchboard != null) { if (sb != null) {
isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue(); isTransparentProxy = Boolean.valueOf(sb.getConfig("isTransparentProxy","false")).booleanValue();
// set timeout // set timeout
timeout = Integer.parseInt(switchboard.getConfig("proxy.clientTimeout", "10000")); timeout = Integer.parseInt(sb.getConfig("proxy.clientTimeout", "10000"));
// create a htRootPath: system pages // create a htRootPath: system pages
htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot")); htRootPath = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot"));
if (!(htRootPath.exists())) { if (!(htRootPath.exists())) {
if(!htRootPath.mkdir()) if(!htRootPath.mkdir())
serverLog.logSevere("PROXY", "could not create htRoot "+ htRootPath); serverLog.logSevere("PROXY", "could not create htRoot "+ htRootPath);
@ -182,10 +182,10 @@ public final class httpdProxyHandler {
// load a transformer // load a transformer
transformer = new htmlFilterContentTransformer(); transformer = new htmlFilterContentTransformer();
transformer.init(new File(switchboard.getRootPath(), switchboard.getConfig(plasmaSwitchboardConstants.LIST_BLUE, "")).toString()); transformer.init(new File(sb.getRootPath(), sb.getConfig(plasmaSwitchboardConstants.LIST_BLUE, "")).toString());
// load the yellow-list // load the yellow-list
final String f = switchboard.getConfig("proxyYellowList", null); final String f = sb.getConfig("proxyYellowList", null);
if (f != null) { if (f != null) {
yellowList = serverFileUtils.loadList(new File(f)); yellowList = serverFileUtils.loadList(new File(f));
theLogger.logConfig("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries"); theLogger.logConfig("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
@ -193,7 +193,7 @@ public final class httpdProxyHandler {
yellowList = new HashSet<String>(); yellowList = new HashSet<String>();
} }
final String redirectorPath = switchboard.getConfig("externalRedirector", ""); final String redirectorPath = sb.getConfig("externalRedirector", "");
if (redirectorPath.length() > 0 && redirectorEnabled == false){ if (redirectorPath.length() > 0 && redirectorEnabled == false){
try { try {
redirectorProcess=Runtime.getRuntime().exec(redirectorPath); redirectorProcess=Runtime.getRuntime().exec(redirectorPath);
@ -260,11 +260,11 @@ public final class httpdProxyHandler {
path = "$Path" "=" value path = "$Path" "=" value
domain = "$Domain" "=" value domain = "$Domain" "=" value
*/ */
if (switchboard.getConfigBool("proxy.monitorCookies", false)) { if (sb.getConfigBool("proxy.monitorCookies", false)) {
if (requestHeader.containsKey(httpRequestHeader.COOKIE)) { if (requestHeader.containsKey(httpRequestHeader.COOKIE)) {
final Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpRequestHeader.COOKIE)}; final Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpRequestHeader.COOKIE)};
synchronized(switchboard.outgoingCookies) { synchronized(sb.outgoingCookies) {
switchboard.outgoingCookies.put(targethost, entry); sb.outgoingCookies.put(targethost, entry);
} }
} }
} }
@ -286,11 +286,11 @@ public final class httpdProxyHandler {
| "Secure" | "Secure"
| "Version" "=" 1*DIGIT | "Version" "=" 1*DIGIT
*/ */
if (switchboard.getConfigBool("proxy.monitorCookies", false)) { if (sb.getConfigBool("proxy.monitorCookies", false)) {
if (respondHeader.containsKey(httpResponseHeader.SET_COOKIE)) { if (respondHeader.containsKey(httpResponseHeader.SET_COOKIE)) {
final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpResponseHeader.SET_COOKIE)}; final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpResponseHeader.SET_COOKIE)};
synchronized(switchboard.incomingCookies) { synchronized(sb.incomingCookies) {
switchboard.incomingCookies.put(serverhost, entry); sb.incomingCookies.put(serverhost, entry);
} }
} }
} }
@ -310,7 +310,7 @@ public final class httpdProxyHandler {
final Date requestDate = new Date(); // remember the time... final Date requestDate = new Date(); // remember the time...
conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis(); sb.proxyLastAccess = System.currentTimeMillis();
// using an ByteCount OutputStream to count the send bytes (needed for the logfile) // using an ByteCount OutputStream to count the send bytes (needed for the logfile)
countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY");
@ -422,7 +422,7 @@ public final class httpdProxyHandler {
requestHeader, requestHeader,
cachedResponseHeader, cachedResponseHeader,
null, // initiator null, // initiator
switchboard.webIndex.defaultProxyProfile // profile sb.webIndex.defaultProxyProfile // profile
); );
plasmaHTCache.storeMetadata(cachedResponseHeader, cacheEntry); // TODO: check if this storeMetadata is necessary plasmaHTCache.storeMetadata(cachedResponseHeader, cacheEntry); // TODO: check if this storeMetadata is necessary
@ -536,7 +536,7 @@ public final class httpdProxyHandler {
requestHeader, requestHeader,
responseHeader, responseHeader,
null, null,
switchboard.webIndex.defaultProxyProfile sb.webIndex.defaultProxyProfile
); );
plasmaHTCache.storeMetadata(responseHeader, cacheEntry); plasmaHTCache.storeMetadata(responseHeader, cacheEntry);
@ -622,7 +622,7 @@ public final class httpdProxyHandler {
// totally fresh file // totally fresh file
//cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert //cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert
cacheEntry.setCacheArray(cacheArray); cacheEntry.setCacheArray(cacheArray);
plasmaHTCache.push(cacheEntry); sb.htEntryStoreProcess(cacheEntry);
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
} else if (cacheArray != null && sizeBeforeDelete == cacheArray.length) { } else if (cacheArray != null && sizeBeforeDelete == cacheArray.length) {
// before we came here we deleted a cache entry // before we came here we deleted a cache entry
@ -634,7 +634,7 @@ public final class httpdProxyHandler {
// before we came here we deleted a cache entry // before we came here we deleted a cache entry
//cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD; //cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD;
cacheEntry.setCacheArray(cacheArray); cacheEntry.setCacheArray(cacheArray);
plasmaHTCache.push(cacheEntry); // necessary update, write response header to cache sb.htEntryStoreProcess(cacheEntry);
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS"); conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
} }
} else { } else {
@ -858,7 +858,7 @@ public final class httpdProxyHandler {
final Date requestDate = new Date(); // remember the time... final Date requestDate = new Date(); // remember the time...
conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis(); sb.proxyLastAccess = System.currentTimeMillis();
// using an ByteCount OutputStream to count the send bytes // using an ByteCount OutputStream to count the send bytes
respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY");
@ -958,7 +958,7 @@ public final class httpdProxyHandler {
final Date requestDate = new Date(); // remember the time... final Date requestDate = new Date(); // remember the time...
conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime())); conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction(); if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis(); sb.proxyLastAccess = System.currentTimeMillis();
// using an ByteCount OutputStream to count the send bytes // using an ByteCount OutputStream to count the send bytes
countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY"); countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY");
@ -1252,7 +1252,7 @@ public final class httpdProxyHandler {
*/ */
private static void addXForwardedForHeader(final Properties conProp, final httpRequestHeader requestHeader) { private static void addXForwardedForHeader(final Properties conProp, final httpRequestHeader requestHeader) {
// setting the X-Forwarded-For Header // setting the X-Forwarded-For Header
if (switchboard.getConfigBool("proxy.sendXForwardedForHeader", true)) { if (sb.getConfigBool("proxy.sendXForwardedForHeader", true)) {
requestHeader.put(httpHeader.X_FORWARDED_FOR, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP)); requestHeader.put(httpHeader.X_FORWARDED_FOR, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
} }
} }
@ -1294,7 +1294,7 @@ public final class httpdProxyHandler {
} }
private static void setViaHeader(final httpHeader header, final String httpVer) { private static void setViaHeader(final httpHeader header, final String httpVer) {
if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return; if (!sb.getConfigBool("proxy.sendViaHeader", true)) return;
final String myAddress = (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().myAlternativeAddress(); final String myAddress = (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().myAlternativeAddress();
if (myAddress != null) { if (myAddress != null) {
@ -1307,7 +1307,7 @@ public final class httpdProxyHandler {
viaValue viaValue
.append(httpVer).append(" ") .append(httpVer).append(" ")
.append(myAddress).append(" ") .append(myAddress).append(" ")
.append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")"); .append("(YaCy ").append(sb.getConfig("vString", "0.0")).append(")");
// storing header back // storing header back
header.put(httpHeader.VIA, new String(viaValue)); header.put(httpHeader.VIA, new String(viaValue));
@ -1316,7 +1316,7 @@ public final class httpdProxyHandler {
public static void doConnect(final Properties conProp, final httpRequestHeader requestHeader, final InputStream clientIn, final OutputStream clientOut) throws IOException { public static void doConnect(final Properties conProp, final httpRequestHeader requestHeader, final InputStream clientIn, final OutputStream clientOut) throws IOException {
switchboard.proxyLastAccess = System.currentTimeMillis(); sb.proxyLastAccess = System.currentTimeMillis();
String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST); String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
final String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER); final String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
@ -1666,7 +1666,7 @@ public final class httpdProxyHandler {
userAgentStr userAgentStr
.append(browserUserAgent.substring(0,pos)) .append(browserUserAgent.substring(0,pos))
.append("; YaCy ") .append("; YaCy ")
.append(switchboard.getConfig("vString","0.1")) .append(sb.getConfig("vString","0.1"))
.append("; yacy.net") .append("; yacy.net")
.append(browserUserAgent.substring(pos)); .append(browserUserAgent.substring(pos));
} else { } else {

16
source/de/anomic/icap/icapd.java
Unescape View File

@ -71,16 +71,16 @@ public class icapd implements serverHandler, Cloneable {
// needed for logging // needed for logging
private final serverLog log = new serverLog("ICAPD"); private final serverLog log = new serverLog("ICAPD");
private static plasmaSwitchboard switchboard = null; private static plasmaSwitchboard sb = null;
private static String virtualHost = null; private static String virtualHost = null;
private static boolean keepAliveSupport = true; private static boolean keepAliveSupport = true;
public icapd() { public icapd() {
if (switchboard == null) { if (sb == null) {
switchboard = plasmaSwitchboard.getSwitchboard(); sb = plasmaSwitchboard.getSwitchboard();
virtualHost = switchboard.getConfig("fileHost","localhost"); virtualHost = sb.getConfig("fileHost","localhost");
} }
} }
@ -124,9 +124,9 @@ public class icapd implements serverHandler, Cloneable {
public icapHeader getDefaultHeaders() { public icapHeader getDefaultHeaders() {
final icapHeader newHeaders = new icapHeader(); final icapHeader newHeaders = new icapHeader();
newHeaders.put(icapHeader.SERVER,"YaCy/" + switchboard.getConfig("vString","")); newHeaders.put(icapHeader.SERVER,"YaCy/" + sb.getConfig("vString",""));
newHeaders.put(icapHeader.DATE, HttpClient.dateString(new Date())); newHeaders.put(icapHeader.DATE, HttpClient.dateString(new Date()));
newHeaders.put(icapHeader.ISTAG, "\"" + switchboard.getConfig("vString","") + "\""); newHeaders.put(icapHeader.ISTAG, "\"" + sb.getConfig("vString","") + "\"");
return newHeaders; return newHeaders;
} }
@ -384,7 +384,7 @@ public class icapd implements serverHandler, Cloneable {
httpRespStatusLine, httpRespStatusLine,
httpReqHeader, httpResHeader, httpReqHeader, httpResHeader,
null, null,
switchboard.webIndex.defaultProxyProfile sb.webIndex.defaultProxyProfile
); );
// copy the response body into the file // copy the response body into the file
@ -397,7 +397,7 @@ public class icapd implements serverHandler, Cloneable {
plasmaHTCache.storeMetadata(httpResHeader, cacheEntry); plasmaHTCache.storeMetadata(httpResHeader, cacheEntry);
// indexing the response // indexing the response
plasmaHTCache.push(cacheEntry); sb.htEntryStoreProcess(cacheEntry);
} catch (final Exception e) { } catch (final Exception e) {
e.printStackTrace(); e.printStackTrace();
} }

13
source/de/anomic/kelondro/kelondroBLOBArray.java
Unescape View File

@ -67,14 +67,13 @@ public class kelondroBLOBArray implements kelondroBLOB {
public kelondroBLOBArray( public kelondroBLOBArray(
final File heapLocation, final File heapLocation,
final int keylength, final kelondroByteOrder ordering, final int keylength,
long agelimit, long sizelimit final kelondroByteOrder ordering) throws IOException {
) throws IOException {
this.keylength = keylength; this.keylength = keylength;
this.ordering = ordering; this.ordering = ordering;
this.heapLocation = heapLocation; this.heapLocation = heapLocation;
this.fileAgeLimit = agelimit; this.fileAgeLimit = oneMonth;
this.fileSizeLimit = sizelimit; this.fileSizeLimit = oneGigabyte;
this.repositoryAgeMax = Long.MAX_VALUE; this.repositoryAgeMax = Long.MAX_VALUE;
this.repositorySizeMax = Long.MAX_VALUE; this.repositorySizeMax = Long.MAX_VALUE;
@ -111,10 +110,12 @@ public class kelondroBLOBArray implements kelondroBLOB {
public void setMaxAge(long maxAge) { public void setMaxAge(long maxAge) {
this.repositoryAgeMax = maxAge; this.repositoryAgeMax = maxAge;
this.fileAgeLimit = Math.min(oneMonth, maxAge / 10);
} }
public void setMaxSize(long maxSize) { public void setMaxSize(long maxSize) {
this.repositorySizeMax = maxSize; this.repositorySizeMax = maxSize;
this.fileSizeLimit = Math.min(oneGigabyte, maxSize / 10);
} }
private void executeLimits() { private void executeLimits() {
@ -311,7 +312,7 @@ public class kelondroBLOBArray implements kelondroBLOB {
final File f = new File("/Users/admin/blobarraytest"); final File f = new File("/Users/admin/blobarraytest");
try { try {
//f.delete(); //f.delete();
final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder, oneMonth, oneGigabyte); final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder);
heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());

42
source/de/anomic/kelondro/kelondroBLOBBuffer.java
Unescape View File

@ -136,9 +136,10 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
private byte[] compress(byte[] b) { private byte[] compress(byte[] b) {
// compressed a byte array and adds a leading magic for the compression // compressed a byte array and adds a leading magic for the compression
try { try {
final ByteArrayOutputStream baos = new ByteArrayOutputStream(); System.out.print("/"); // DEBUG
final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5);
baos.write(gzipMagic); baos.write(gzipMagic);
final OutputStream os = new GZIPOutputStream(baos, 128); final OutputStream os = new GZIPOutputStream(baos, 512);
os.write(b); os.write(b);
os.close(); os.close();
baos.close(); baos.close();
@ -150,6 +151,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
} }
private byte[] markWithPlainMagic(byte[] b) { private byte[] markWithPlainMagic(byte[] b) {
System.out.print("+"); // DEBUG
byte[] r = new byte[b.length + 2]; byte[] r = new byte[b.length + 2];
r[0] = plainMagic[0]; r[0] = plainMagic[0];
r[1] = plainMagic[1]; r[1] = plainMagic[1];
@ -160,6 +162,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
private byte[] decompress(byte[] b) { private byte[] decompress(byte[] b) {
// use a magic in the head of the bytes to identify compression type // use a magic in the head of the bytes to identify compression type
if (kelondroByteArray.equals(b, gzipMagic)) { if (kelondroByteArray.equals(b, gzipMagic)) {
System.out.print("\\"); // DEBUG
ByteArrayInputStream bais = new ByteArrayInputStream(b); ByteArrayInputStream bais = new ByteArrayInputStream(b);
// eat up the magic // eat up the magic
bais.read(); bais.read();
@ -168,7 +171,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
InputStream gis; InputStream gis;
try { try {
gis = new GZIPInputStream(bais); gis = new GZIPInputStream(bais);
final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length);
final byte[] buf = new byte[1024]; final byte[] buf = new byte[1024];
int n; int n;
while ((n = gis.read(buf)) > 0) baos.write(buf, 0, n); while ((n = gis.read(buf)) > 0) baos.write(buf, 0, n);
@ -182,6 +185,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
return null; return null;
} }
} else if (kelondroByteArray.equals(b, plainMagic)) { } else if (kelondroByteArray.equals(b, plainMagic)) {
System.out.print("-"); // DEBUG
byte[] r = new byte[b.length - 2]; byte[] r = new byte[b.length - 2];
System.arraycopy(b, 2, r, 0, b.length - 2); System.arraycopy(b, 2, r, 0, b.length - 2);
return r; return r;
@ -286,20 +290,12 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
// check if the buffer is full or could be full after this write // check if the buffer is full or could be full after this write
if (this.queueLength + b.length * 2 > this.maxCacheSize) flushAll(); if (this.queueLength + b.length * 2 > this.maxCacheSize) flushAll();
// depending on execution cases, write into different queues // files are written uncompressed to the uncompressed-queue
if (!this.executing && this.compress) { // they are either written uncompressed to the database
// files are compressed when they arrive and written to the compressed queue // or compressed with the concurrent thread and written later
byte[] bb = compress(b); this.rawQueue.add(new Entry(key, b));
this.compressedQueue.add(new Entry(key, bb)); this.queueLength += b.length;
this.queueLength += bb.length;
} else {
// files are written uncompressed to the uncompressed-queue
// they are either written uncompressed to the database
// or compressed with the concurrent thread and written later
this.rawQueue.add(new Entry(key, b));
this.queueLength += b.length;
}
} }
public synchronized void remove(byte[] key) throws IOException { public synchronized void remove(byte[] key) throws IOException {
@ -322,15 +318,11 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
} }
private boolean flushOne(boolean block) throws IOException { private boolean flushOne(boolean block) throws IOException {
if (!this.executing && this.compress) { if (rawQueue.size() > 0) {
// files are compressed when they arrive and written to the compressed queue // files are compressed when they arrive and written to the compressed queue
return flushOneCompressed(block); return flushOneRaw(block);
} else { } else {
// files are written uncompressed to the uncompressed-queue return flushOneCompressed(block);
// they are either written uncompressed to the database
// or compressed with the concurrent thread and written later
if (flushOneRaw(block)) return true;
else return flushOneCompressed(block);
} }
} }
@ -371,7 +363,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
} }
private void flushAll() throws IOException { private void flushAll() throws IOException {
while ((this.rawQueue != null && this.rawQueue.size() > 0) || while (this.rawQueue.size() > 0 ||
(this.compressedQueue != null && this.compressedQueue.size() > 0)) { (this.compressedQueue != null && this.compressedQueue.size() > 0)) {
if (!flushOne(false)) break; if (!flushOne(false)) break;
} }

32
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -40,7 +40,6 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.http.httpResponseHeader; import de.anomic.http.httpResponseHeader;
import de.anomic.index.indexDocumentMetadata; import de.anomic.index.indexDocumentMetadata;
@ -59,13 +58,12 @@ public final class plasmaHTCache {
public static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap"; public static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap";
public static final String FILE_DB_NAME = "file.array"; public static final String FILE_DB_NAME = "file.array";
private static final int stackLimit = 150; // if we exceed that limit, we do not check idle
public static final long oneday = 1000L * 60L * 60L * 24L; // milliseconds of a day public static final long oneday = 1000L * 60L * 60L * 24L; // milliseconds of a day
private static kelondroMap responseHeaderDB = null; private static kelondroMap responseHeaderDB = null;
private static kelondroBLOBBuffer fileDB = null; private static kelondroBLOBBuffer fileDB = null;
private static kelondroBLOBArray fileDBunbuffered = null;
private static final ConcurrentLinkedQueue<indexDocumentMetadata> cacheStack = new ConcurrentLinkedQueue<indexDocumentMetadata>();
public static long maxCacheSize = 0l; public static long maxCacheSize = 0l;
public static File cachePath = null; public static File cachePath = null;
public static final serverLog log = new serverLog("HTCACHE"); public static final serverLog log = new serverLog("HTCACHE");
@ -135,9 +133,10 @@ public final class plasmaHTCache {
} }
responseHeaderDB = new kelondroMap(blob, 500); responseHeaderDB = new kelondroMap(blob, 500);
try { try {
kelondroBLOBArray fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder, kelondroBLOBArray.oneMonth, kelondroBLOBArray.oneGigabyte); fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder);
fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 1024 * 1024, true); fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB.start(); fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 2 * 1024 * 1024, true);
//fileDB.start();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -159,10 +158,6 @@ public final class plasmaHTCache {
directory.delete(); directory.delete();
} }
public static int size() {
return cacheStack.size();
}
public static int responseHeaderDBSize() { public static int responseHeaderDBSize() {
return responseHeaderDB.size(); return responseHeaderDB.size();
} }
@ -170,14 +165,6 @@ public final class plasmaHTCache {
public static long fileDBSize() { public static long fileDBSize() {
return fileDB.length(); return fileDB.length();
} }
public static void push(final indexDocumentMetadata entry) {
cacheStack.add(entry);
}
public static indexDocumentMetadata pop() {
return cacheStack.poll();
}
/** /**
* This method changes the HTCache size.<br> * This method changes the HTCache size.<br>
@ -185,6 +172,7 @@ public final class plasmaHTCache {
*/ */
public static void setCacheSize(final long newCacheSize) { public static void setCacheSize(final long newCacheSize) {
maxCacheSize = newCacheSize; maxCacheSize = newCacheSize;
fileDBunbuffered.setMaxSize(maxCacheSize);
} }
/** /**
@ -201,14 +189,6 @@ public final class plasmaHTCache {
fileDB.close(); fileDB.close();
} }
public static boolean full() {
return (cacheStack.size() > stackLimit);
}
public static boolean empty() {
return (cacheStack.size() == 0);
}
public static boolean isPicture(final String mimeType) { public static boolean isPicture(final String mimeType) {
if (mimeType == null) return false; if (mimeType == null) return false;
return mimeType.toUpperCase().startsWith("IMAGE"); return mimeType.toUpperCase().startsWith("IMAGE");

11
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -65,9 +65,9 @@ public class plasmaSnippetCache {
public static final int ERROR_PARSER_NO_LINES = 15; public static final int ERROR_PARSER_NO_LINES = 15;
public static final int ERROR_NO_MATCH = 16; public static final int ERROR_NO_MATCH = 16;
private static int snippetsScoreCounter = 0; private static int snippetsScoreCounter = 0;
private static kelondroMScoreCluster<String> snippetsScore = null; private static kelondroMScoreCluster<String> snippetsScore = null;
private static final HashMap<String, String> snippetsCache = new HashMap<String, String>(); private static final HashMap<String, String> snippetsCache = new HashMap<String, String>();
/** /**
* a cache holding URLs to favicons specified by the page content, e.g. by using the html link-tag. e.g. * a cache holding URLs to favicons specified by the page content, e.g. by using the html link-tag. e.g.
@ -78,13 +78,16 @@ public class plasmaSnippetCache {
private static final HashMap<String, yacyURL> faviconCache = new HashMap<String, yacyURL>(); private static final HashMap<String, yacyURL> faviconCache = new HashMap<String, yacyURL>();
private static plasmaParser parser = null; private static plasmaParser parser = null;
private static serverLog log = null; private static serverLog log = null;
private static plasmaSwitchboard sb = null;
public static void init( public static void init(
final plasmaParser parserx, final plasmaParser parserx,
final serverLog logx final serverLog logx,
final plasmaSwitchboard switchboard
) { ) {
parser = parserx; parser = parserx;
log = logx; log = logx;
sb = switchboard;
snippetsScoreCounter = 0; snippetsScoreCounter = 0;
snippetsScore = new kelondroMScoreCluster<String>(); snippetsScore = new kelondroMScoreCluster<String>();
snippetsCache.clear(); snippetsCache.clear();
@ -290,7 +293,7 @@ public class plasmaSnippetCache {
// getting resource metadata (e.g. the http headers for http resources) // getting resource metadata (e.g. the http headers for http resources)
if (entry != null) { if (entry != null) {
// place entry on crawl queue // place entry on crawl queue
plasmaHTCache.push(entry); sb.htEntryStoreProcess(entry);
// read resource body (if it is there) // read resource body (if it is there)
final byte []resourceArray = entry.cacheArray(); final byte []resourceArray = entry.cacheArray();

44
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -544,7 +544,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// generate snippets cache // generate snippets cache
log.logConfig("Initializing Snippet Cache"); log.logConfig("Initializing Snippet Cache");
plasmaSnippetCache.init(parser, log); plasmaSnippetCache.init(parser, log, this);
final String wikiParserClassName = getConfig(plasmaSwitchboardConstants.WIKIPARSER_CLASS, plasmaSwitchboardConstants.WIKIPARSER_CLASS_DEFAULT); final String wikiParserClassName = getConfig(plasmaSwitchboardConstants.WIKIPARSER_CLASS, plasmaSwitchboardConstants.WIKIPARSER_CLASS_DEFAULT);
this.log.logConfig("Loading wiki parser " + wikiParserClassName + " ..."); this.log.logConfig("Loading wiki parser " + wikiParserClassName + " ...");
@ -590,24 +590,21 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
moreMemory.schedule(new MoreMemory(), 300000, 600000); moreMemory.schedule(new MoreMemory(), 300000, 600000);
deployThread(plasmaSwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null, deployThread(plasmaSwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
new serverInstantBusyThread(this, plasmaSwitchboardConstants.CLEANUP_METHOD_START, plasmaSwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run new serverInstantBusyThread(this, plasmaSwitchboardConstants.CLEANUP_METHOD_START, plasmaSwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run
deployThread(plasmaSwitchboardConstants.CRAWLSTACK, "Crawl URL Stacker", "process that checks url for double-occurrences and for allowance/disallowance by robots.txt", null, deployThread(plasmaSwitchboardConstants.CRAWLSTACK, "Crawl URL Stacker", "process that checks url for double-occurrences and for allowance/disallowance by robots.txt", null,
new serverInstantBusyThread(crawlStacker, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_START, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_FREEMEM), 8000); new serverInstantBusyThread(crawlStacker, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_START, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_FREEMEM), 8000);
deployThread(plasmaSwitchboardConstants.INDEXER, "Indexing", "thread that either initiates a parsing/indexing queue, distributes the index into the DHT, stores parsed documents or flushes the index cache", "/IndexCreateIndexingQueue_p.html", deployThread(plasmaSwitchboardConstants.INDEXER, "Indexing", "thread that either initiates a parsing/indexing queue, distributes the index into the DHT, stores parsed documents or flushes the index cache", "/IndexCreateIndexingQueue_p.html",
new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEXER_METHOD_START, plasmaSwitchboardConstants.INDEXER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEXER_METHOD_FREEMEM), 10000); new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEXER_METHOD_START, plasmaSwitchboardConstants.INDEXER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEXER_METHOD_FREEMEM), 10000);
deployThread(plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE, "Proxy Cache Enqueue", "job takes new input files from RAM stack, stores them, and hands over to the Indexing Stack", null,
new serverInstantBusyThread(this, plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_METHOD_START, plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_METHOD_FREEMEM), 10000);
deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null, deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000); new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000);
deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "", deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "",
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here? new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here?
deployThread(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html", deployThread(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html",
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000); new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000);
deployThread(plasmaSwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null, deployThread(plasmaSwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null,
new serverInstantBusyThread(yc, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_START, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000); new serverInstantBusyThread(yc, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_START, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000);
deployThread(plasmaSwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null, deployThread(plasmaSwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null,
new serverInstantBusyThread(yc, plasmaSwitchboardConstants.PEER_PING_METHOD_START, plasmaSwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000); new serverInstantBusyThread(yc, plasmaSwitchboardConstants.PEER_PING_METHOD_START, plasmaSwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000);
deployThread(plasmaSwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null, deployThread(plasmaSwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null,
new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEX_DIST_METHOD_START, plasmaSwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEX_DIST_METHOD_FREEMEM), 60000, new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEX_DIST_METHOD_START, plasmaSwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEX_DIST_METHOD_FREEMEM), 60000,
Long.parseLong(getConfig(plasmaSwitchboardConstants.INDEX_DIST_IDLESLEEP , "5000")), Long.parseLong(getConfig(plasmaSwitchboardConstants.INDEX_DIST_IDLESLEEP , "5000")),
@ -932,14 +929,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
new plasmaSearchRankingProfile("", crypt.simpleDecode(sb.getConfig("rankingProfile", ""), null)); new plasmaSearchRankingProfile("", crypt.simpleDecode(sb.getConfig("rankingProfile", ""), null));
} }
/**
* This method changes the HTCache size.<br>
* @param newCacheSize in MB
*/
public final void setCacheSize(final long newCacheSize) {
plasmaHTCache.setCacheSize(1048576 * newCacheSize);
}
public boolean onlineCaution() { public boolean onlineCaution() {
return return
(System.currentTimeMillis() - this.proxyLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, "30000"))) || (System.currentTimeMillis() - this.proxyLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, "30000"))) ||
@ -1081,15 +1070,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
return true; return true;
} }
public boolean htEntryStoreJob() {
if (plasmaHTCache.empty()) return false;
return htEntryStoreProcess(plasmaHTCache.pop());
}
public int htEntrySize() {
return plasmaHTCache.size();
}
public void close() { public void close() {
log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:"); log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
serverProfiling.stopSystemProfiling(); serverProfiling.stopSystemProfiling();
@ -1884,12 +1864,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
thread.setIdleSleep(2000); thread.setIdleSleep(2000);
} }
thread = getThread(plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE);
if (thread != null) {
setConfig(plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_BUSYSLEEP , thread.setBusySleep(0));
thread.setIdleSleep(2000);
}
thread = getThread(plasmaSwitchboardConstants.INDEXER); thread = getThread(plasmaSwitchboardConstants.INDEXER);
if (thread != null) { if (thread != null) {
setConfig(plasmaSwitchboardConstants.INDEXER_BUSYSLEEP , thread.setBusySleep(newBusySleep / 8)); setConfig(plasmaSwitchboardConstants.INDEXER_BUSYSLEEP , thread.setBusySleep(newBusySleep / 8));

14
source/de/anomic/plasma/plasmaSwitchboardConstants.java
Unescape View File

@ -110,20 +110,6 @@ public final class plasmaSwitchboardConstants {
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM = null; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM = null;
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep"; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep";
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep"; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep";
// 70_cachemanager
/**
* <p><code>public static final String <strong>PROXY_CACHE_ENQUEUE</strong> = "70_cachemanager"</code></p>
* <p>Name of the proxy cache enqueue thread which fetches a given website and saves the site itself as well as it's
* HTTP-headers in the HTCACHE</p>
*
* @see plasmaSwitchboard#PROXY_CACHE_PATH
*/
public static final String PROXY_CACHE_ENQUEUE = "70_cachemanager";
public static final String PROXY_CACHE_ENQUEUE_METHOD_START = "htEntryStoreJob";
public static final String PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT = "htEntrySize";
public static final String PROXY_CACHE_ENQUEUE_METHOD_FREEMEM = null;
public static final String PROXY_CACHE_ENQUEUE_IDLESLEEP = "70_cachemanager_idlesleep";
public static final String PROXY_CACHE_ENQUEUE_BUSYSLEEP = "70_cachemanager_busysleep";
// 74_parsing // 74_parsing
/** /**
* <p><code>public static final String <strong>INDEXER</strong> = "80_indexing"</code></p> * <p><code>public static final String <strong>INDEXER</strong> = "80_indexing"</code></p>

Write Preview
Loading…
Cancel
Save