boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added some performance tweaks to the new BLOB buffer

Browse Source 
- removed the now superfluous HT storage thread
- reduced number of file decompression by shifting the compression moment to the future


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5286 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 77e41da7d2
commit 1778fb420d
14 changed files with 90 additions and 163 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.611
releaseVersion=0.612
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

3
defaults/performance_dht.profile
Unescape View File

@ -25,9 +25,6 @@
62_remotetriggeredcrawl_busysleep=1000
62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false
70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=1
70_cachemanager_memprereq=1048576
80_indexing_idlesleep=1000
80_indexing_busysleep=100
80_indexing_memprereq=6291456

3
defaults/yacy.init
Unescape View File

@ -558,9 +558,6 @@ performanceSpeed=100
62_remotetriggeredcrawl_busysleep=1000
62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false
70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=0
70_cachemanager_memprereq=1048576
80_indexing_idlesleep=1000
80_indexing_busysleep=10
80_indexing_memprereq=6291456

11
htroot/ProxyIndexingMonitor_p.java
Unescape View File

@ -31,6 +31,7 @@ import java.io.File;
import java.io.IOException;
import de.anomic.http.httpRequestHeader;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects;
@ -88,11 +89,11 @@ public class ProxyIndexingMonitor_p {
if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs();
// proxyCacheSize
oldProxyCacheSize = getStringLong(env.getConfig("proxyCacheSize", "64"));
newProxyCacheSize = getStringLong(post.get("proxyCacheSize", "64"));
oldProxyCacheSize = getStringLong(env.getConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "64"));
newProxyCacheSize = getStringLong(post.get(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "64"));
if (getLong(newProxyCacheSize) < 4) { newProxyCacheSize = "4"; }
env.setConfig("proxyCacheSize", newProxyCacheSize);
sb.setCacheSize(Long.parseLong(newProxyCacheSize));
env.setConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);
plasmaHTCache.setCacheSize(Long.parseLong(newProxyCacheSize) * 1024 * 1024);
// implant these settings also into the crawling profile for the proxy
if (sb.webIndex.defaultProxyProfile == null) {
@ -152,7 +153,7 @@ public class ProxyIndexingMonitor_p {
prop.put("proxyIndexingLocalText", env.getConfig("proxyIndexingLocalText", "").equals("true") ? "1" : "0");
prop.put("proxyIndexingLocalMedia", env.getConfig("proxyIndexingLocalMedia", "").equals("true") ? "1" : "0");
prop.put("proxyCache", env.getConfig(plasmaSwitchboardConstants.HTCACHE_PATH, plasmaSwitchboardConstants.HTCACHE_PATH_DEFAULT));
prop.put("proxyCacheSize", env.getConfigLong("proxyCacheSize", 64));
prop.put("proxyCacheSize", env.getConfigLong(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, 64));
// return rewrite properties
return prop;
}

7
source/de/anomic/crawler/ProtocolLoader.java
Unescape View File

@ -123,9 +123,4 @@ public final class ProtocolLoader {
}
}
}
}

1
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -133,6 +133,7 @@ public class ZURL {
public synchronized Entry getEntry(final String urlhash) {
try {
if (urlIndex == null) return null;
final kelondroRow.Entry entry = urlIndex.get(urlhash.getBytes());
if (entry == null) return null;
return new Entry(entry);

54
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -98,7 +98,7 @@ public final class httpdProxyHandler {
// static variables
// can only be instantiated upon first instantiation of this class object
private static plasmaSwitchboard switchboard = null;
private static plasmaSwitchboard sb = null;
private static final HashSet<String> yellowList;
private static int timeout = 30000;
private static boolean yacyTrigger = true;
@ -165,16 +165,16 @@ public final class httpdProxyHandler {
theLogger.logSevere("Unable to configure proxy access logging.",e);
}
switchboard = plasmaSwitchboard.getSwitchboard();
if (switchboard != null) {
sb = plasmaSwitchboard.getSwitchboard();
if (sb != null) {
isTransparentProxy = Boolean.valueOf(switchboard.getConfig("isTransparentProxy","false")).booleanValue();
isTransparentProxy = Boolean.valueOf(sb.getConfig("isTransparentProxy","false")).booleanValue();
// set timeout
timeout = Integer.parseInt(switchboard.getConfig("proxy.clientTimeout", "10000"));
timeout = Integer.parseInt(sb.getConfig("proxy.clientTimeout", "10000"));
// create a htRootPath: system pages
htRootPath = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot"));
htRootPath = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot"));
if (!(htRootPath.exists())) {
if(!htRootPath.mkdir())
serverLog.logSevere("PROXY", "could not create htRoot "+ htRootPath);
@ -182,10 +182,10 @@ public final class httpdProxyHandler {
// load a transformer
transformer = new htmlFilterContentTransformer();
transformer.init(new File(switchboard.getRootPath(), switchboard.getConfig(plasmaSwitchboardConstants.LIST_BLUE, "")).toString());
transformer.init(new File(sb.getRootPath(), sb.getConfig(plasmaSwitchboardConstants.LIST_BLUE, "")).toString());
// load the yellow-list
final String f = switchboard.getConfig("proxyYellowList", null);
final String f = sb.getConfig("proxyYellowList", null);
if (f != null) {
yellowList = serverFileUtils.loadList(new File(f));
theLogger.logConfig("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
@ -193,7 +193,7 @@ public final class httpdProxyHandler {
yellowList = new HashSet<String>();
}
final String redirectorPath = switchboard.getConfig("externalRedirector", "");
final String redirectorPath = sb.getConfig("externalRedirector", "");
if (redirectorPath.length() > 0 && redirectorEnabled == false){
try {
redirectorProcess=Runtime.getRuntime().exec(redirectorPath);
@ -260,11 +260,11 @@ public final class httpdProxyHandler {
path = "$Path" "=" value
domain = "$Domain" "=" value
*/
if (switchboard.getConfigBool("proxy.monitorCookies", false)) {
if (sb.getConfigBool("proxy.monitorCookies", false)) {
if (requestHeader.containsKey(httpRequestHeader.COOKIE)) {
final Object[] entry = new Object[]{new Date(), clienthost, requestHeader.getMultiple(httpRequestHeader.COOKIE)};
synchronized(switchboard.outgoingCookies) {
switchboard.outgoingCookies.put(targethost, entry);
synchronized(sb.outgoingCookies) {
sb.outgoingCookies.put(targethost, entry);
}
}
}
@ -286,11 +286,11 @@ public final class httpdProxyHandler {
| "Secure"
| "Version" "=" 1*DIGIT
*/
if (switchboard.getConfigBool("proxy.monitorCookies", false)) {
if (sb.getConfigBool("proxy.monitorCookies", false)) {
if (respondHeader.containsKey(httpResponseHeader.SET_COOKIE)) {
final Object[] entry = new Object[]{new Date(), targetclient, respondHeader.getMultiple(httpResponseHeader.SET_COOKIE)};
synchronized(switchboard.incomingCookies) {
switchboard.incomingCookies.put(serverhost, entry);
synchronized(sb.incomingCookies) {
sb.incomingCookies.put(serverhost, entry);
}
}
}
@ -310,7 +310,7 @@ public final class httpdProxyHandler {
final Date requestDate = new Date(); // remember the time...
conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis();
sb.proxyLastAccess = System.currentTimeMillis();
// using an ByteCount OutputStream to count the send bytes (needed for the logfile)
countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY");
@ -422,7 +422,7 @@ public final class httpdProxyHandler {
requestHeader,
cachedResponseHeader,
null, // initiator
switchboard.webIndex.defaultProxyProfile // profile
sb.webIndex.defaultProxyProfile // profile
);
plasmaHTCache.storeMetadata(cachedResponseHeader, cacheEntry); // TODO: check if this storeMetadata is necessary
@ -536,7 +536,7 @@ public final class httpdProxyHandler {
requestHeader,
responseHeader,
null,
switchboard.webIndex.defaultProxyProfile
sb.webIndex.defaultProxyProfile
);
plasmaHTCache.storeMetadata(responseHeader, cacheEntry);
@ -622,7 +622,7 @@ public final class httpdProxyHandler {
// totally fresh file
//cacheEntry.status = plasmaHTCache.CACHE_FILL; // it's an insert
cacheEntry.setCacheArray(cacheArray);
plasmaHTCache.push(cacheEntry);
sb.htEntryStoreProcess(cacheEntry);
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_MISS");
} else if (cacheArray != null && sizeBeforeDelete == cacheArray.length) {
// before we came here we deleted a cache entry
@ -634,7 +634,7 @@ public final class httpdProxyHandler {
// before we came here we deleted a cache entry
//cacheEntry.status = plasmaHTCache.CACHE_STALE_RELOAD_GOOD;
cacheEntry.setCacheArray(cacheArray);
plasmaHTCache.push(cacheEntry); // necessary update, write response header to cache
sb.htEntryStoreProcess(cacheEntry);
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
}
} else {
@ -858,7 +858,7 @@ public final class httpdProxyHandler {
final Date requestDate = new Date(); // remember the time...
conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis();
sb.proxyLastAccess = System.currentTimeMillis();
// using an ByteCount OutputStream to count the send bytes
respond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY");
@ -958,7 +958,7 @@ public final class httpdProxyHandler {
final Date requestDate = new Date(); // remember the time...
conProp.put(httpHeader.CONNECTION_PROP_REQUEST_START, Long.valueOf(requestDate.getTime()));
if (yacyTrigger) de.anomic.yacy.yacyCore.triggerOnlineAction();
switchboard.proxyLastAccess = System.currentTimeMillis();
sb.proxyLastAccess = System.currentTimeMillis();
// using an ByteCount OutputStream to count the send bytes
countedRespond = new httpdByteCountOutputStream(respond,conProp.getProperty(httpHeader.CONNECTION_PROP_REQUESTLINE).length() + 2,"PROXY");
@ -1252,7 +1252,7 @@ public final class httpdProxyHandler {
*/
private static void addXForwardedForHeader(final Properties conProp, final httpRequestHeader requestHeader) {
// setting the X-Forwarded-For Header
if (switchboard.getConfigBool("proxy.sendXForwardedForHeader", true)) {
if (sb.getConfigBool("proxy.sendXForwardedForHeader", true)) {
requestHeader.put(httpHeader.X_FORWARDED_FOR, conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP));
}
}
@ -1294,7 +1294,7 @@ public final class httpdProxyHandler {
}
private static void setViaHeader(final httpHeader header, final String httpVer) {
if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return;
if (!sb.getConfigBool("proxy.sendViaHeader", true)) return;
final String myAddress = (httpd.getAlternativeResolver() == null) ? null : httpd.getAlternativeResolver().myAlternativeAddress();
if (myAddress != null) {
@ -1307,7 +1307,7 @@ public final class httpdProxyHandler {
viaValue
.append(httpVer).append(" ")
.append(myAddress).append(" ")
.append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")");
.append("(YaCy ").append(sb.getConfig("vString", "0.0")).append(")");
// storing header back
header.put(httpHeader.VIA, new String(viaValue));
@ -1316,7 +1316,7 @@ public final class httpdProxyHandler {
public static void doConnect(final Properties conProp, final httpRequestHeader requestHeader, final InputStream clientIn, final OutputStream clientOut) throws IOException {
switchboard.proxyLastAccess = System.currentTimeMillis();
sb.proxyLastAccess = System.currentTimeMillis();
String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
final String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
@ -1666,7 +1666,7 @@ public final class httpdProxyHandler {
userAgentStr
.append(browserUserAgent.substring(0,pos))
.append("; YaCy ")
.append(switchboard.getConfig("vString","0.1"))
.append(sb.getConfig("vString","0.1"))
.append("; yacy.net")
.append(browserUserAgent.substring(pos));
} else {

16
source/de/anomic/icap/icapd.java
Unescape View File

@ -71,16 +71,16 @@ public class icapd implements serverHandler, Cloneable {
// needed for logging
private final serverLog log = new serverLog("ICAPD");
private static plasmaSwitchboard switchboard = null;
private static plasmaSwitchboard sb = null;
private static String virtualHost = null;
private static boolean keepAliveSupport = true;
public icapd() {
if (switchboard == null) {
switchboard = plasmaSwitchboard.getSwitchboard();
virtualHost = switchboard.getConfig("fileHost","localhost");
if (sb == null) {
sb = plasmaSwitchboard.getSwitchboard();
virtualHost = sb.getConfig("fileHost","localhost");
}
}
@ -124,9 +124,9 @@ public class icapd implements serverHandler, Cloneable {
public icapHeader getDefaultHeaders() {
final icapHeader newHeaders = new icapHeader();
newHeaders.put(icapHeader.SERVER,"YaCy/" + switchboard.getConfig("vString",""));
newHeaders.put(icapHeader.SERVER,"YaCy/" + sb.getConfig("vString",""));
newHeaders.put(icapHeader.DATE, HttpClient.dateString(new Date()));
newHeaders.put(icapHeader.ISTAG, "\"" + switchboard.getConfig("vString","") + "\"");
newHeaders.put(icapHeader.ISTAG, "\"" + sb.getConfig("vString","") + "\"");
return newHeaders;
}
@ -384,7 +384,7 @@ public class icapd implements serverHandler, Cloneable {
httpRespStatusLine,
httpReqHeader, httpResHeader,
null,
switchboard.webIndex.defaultProxyProfile
sb.webIndex.defaultProxyProfile
);
// copy the response body into the file
@ -397,7 +397,7 @@ public class icapd implements serverHandler, Cloneable {
plasmaHTCache.storeMetadata(httpResHeader, cacheEntry);
// indexing the response
plasmaHTCache.push(cacheEntry);
sb.htEntryStoreProcess(cacheEntry);
} catch (final Exception e) {
e.printStackTrace();
}

13
source/de/anomic/kelondro/kelondroBLOBArray.java
Unescape View File

@ -67,14 +67,13 @@ public class kelondroBLOBArray implements kelondroBLOB {
public kelondroBLOBArray(
final File heapLocation,
final int keylength, final kelondroByteOrder ordering,
long agelimit, long sizelimit
) throws IOException {
final int keylength,
final kelondroByteOrder ordering) throws IOException {
this.keylength = keylength;
this.ordering = ordering;
this.heapLocation = heapLocation;
this.fileAgeLimit = agelimit;
this.fileSizeLimit = sizelimit;
this.fileAgeLimit = oneMonth;
this.fileSizeLimit = oneGigabyte;
this.repositoryAgeMax = Long.MAX_VALUE;
this.repositorySizeMax = Long.MAX_VALUE;
@ -111,10 +110,12 @@ public class kelondroBLOBArray implements kelondroBLOB {
public void setMaxAge(long maxAge) {
this.repositoryAgeMax = maxAge;
this.fileAgeLimit = Math.min(oneMonth, maxAge / 10);
}
public void setMaxSize(long maxSize) {
this.repositorySizeMax = maxSize;
this.fileSizeLimit = Math.min(oneGigabyte, maxSize / 10);
}
private void executeLimits() {
@ -311,7 +312,7 @@ public class kelondroBLOBArray implements kelondroBLOB {
final File f = new File("/Users/admin/blobarraytest");
try {
//f.delete();
final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder, oneMonth, oneGigabyte);
final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder);
heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());

42
source/de/anomic/kelondro/kelondroBLOBBuffer.java
Unescape View File

@ -136,9 +136,10 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
private byte[] compress(byte[] b) {
// compressed a byte array and adds a leading magic for the compression
try {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
System.out.print("/"); // DEBUG
final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5);
baos.write(gzipMagic);
final OutputStream os = new GZIPOutputStream(baos, 128);
final OutputStream os = new GZIPOutputStream(baos, 512);
os.write(b);
os.close();
baos.close();
@ -150,6 +151,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
}
private byte[] markWithPlainMagic(byte[] b) {
System.out.print("+"); // DEBUG
byte[] r = new byte[b.length + 2];
r[0] = plainMagic[0];
r[1] = plainMagic[1];
@ -160,6 +162,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
private byte[] decompress(byte[] b) {
// use a magic in the head of the bytes to identify compression type
if (kelondroByteArray.equals(b, gzipMagic)) {
System.out.print("\\"); // DEBUG
ByteArrayInputStream bais = new ByteArrayInputStream(b);
// eat up the magic
bais.read();
@ -168,7 +171,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
InputStream gis;
try {
gis = new GZIPInputStream(bais);
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length);
final byte[] buf = new byte[1024];
int n;
while ((n = gis.read(buf)) > 0) baos.write(buf, 0, n);
@ -182,6 +185,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
return null;
}
} else if (kelondroByteArray.equals(b, plainMagic)) {
System.out.print("-"); // DEBUG
byte[] r = new byte[b.length - 2];
System.arraycopy(b, 2, r, 0, b.length - 2);
return r;
@ -286,20 +290,12 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
// check if the buffer is full or could be full after this write
if (this.queueLength + b.length * 2 > this.maxCacheSize) flushAll();
// depending on execution cases, write into different queues
if (!this.executing && this.compress) {
// files are compressed when they arrive and written to the compressed queue
byte[] bb = compress(b);
this.compressedQueue.add(new Entry(key, bb));
this.queueLength += bb.length;
} else {
// files are written uncompressed to the uncompressed-queue
// they are either written uncompressed to the database
// or compressed with the concurrent thread and written later
this.rawQueue.add(new Entry(key, b));
this.queueLength += b.length;
}
// files are written uncompressed to the uncompressed-queue
// they are either written uncompressed to the database
// or compressed with the concurrent thread and written later
this.rawQueue.add(new Entry(key, b));
this.queueLength += b.length;
}
public synchronized void remove(byte[] key) throws IOException {
@ -322,15 +318,11 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
}
private boolean flushOne(boolean block) throws IOException {
if (!this.executing && this.compress) {
if (rawQueue.size() > 0) {
// files are compressed when they arrive and written to the compressed queue
return flushOneCompressed(block);
return flushOneRaw(block);
} else {
// files are written uncompressed to the uncompressed-queue
// they are either written uncompressed to the database
// or compressed with the concurrent thread and written later
if (flushOneRaw(block)) return true;
else return flushOneCompressed(block);
return flushOneCompressed(block);
}
}
@ -371,7 +363,7 @@ public class kelondroBLOBBuffer extends Thread implements kelondroBLOB {
}
private void flushAll() throws IOException {
while ((this.rawQueue != null && this.rawQueue.size() > 0) ||
while (this.rawQueue.size() > 0 ||
(this.compressedQueue != null && this.compressedQueue.size() > 0)) {
if (!flushOne(false)) break;
}

32
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -40,7 +40,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.http.httpResponseHeader;
import de.anomic.index.indexDocumentMetadata;
@ -59,13 +58,12 @@ public final class plasmaHTCache {
public static final String RESPONSE_HEADER_DB_NAME = "responseHeader.heap";
public static final String FILE_DB_NAME = "file.array";
private static final int stackLimit = 150; // if we exceed that limit, we do not check idle
public static final long oneday = 1000L * 60L * 60L * 24L; // milliseconds of a day
private static kelondroMap responseHeaderDB = null;
private static kelondroBLOBBuffer fileDB = null;
private static kelondroBLOBArray fileDBunbuffered = null;
private static final ConcurrentLinkedQueue<indexDocumentMetadata> cacheStack = new ConcurrentLinkedQueue<indexDocumentMetadata>();
public static long maxCacheSize = 0l;
public static File cachePath = null;
public static final serverLog log = new serverLog("HTCACHE");
@ -135,9 +133,10 @@ public final class plasmaHTCache {
}
responseHeaderDB = new kelondroMap(blob, 500);
try {
kelondroBLOBArray fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder, kelondroBLOBArray.oneMonth, kelondroBLOBArray.oneGigabyte);
fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 1024 * 1024, true);
fileDB.start();
fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder);
fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 2 * 1024 * 1024, true);
//fileDB.start();
} catch (IOException e) {
e.printStackTrace();
}
@ -159,10 +158,6 @@ public final class plasmaHTCache {
directory.delete();
}
public static int size() {
return cacheStack.size();
}
public static int responseHeaderDBSize() {
return responseHeaderDB.size();
}
@ -170,14 +165,6 @@ public final class plasmaHTCache {
public static long fileDBSize() {
return fileDB.length();
}
public static void push(final indexDocumentMetadata entry) {
cacheStack.add(entry);
}
public static indexDocumentMetadata pop() {
return cacheStack.poll();
}
/**
* This method changes the HTCache size.<br>
@ -185,6 +172,7 @@ public final class plasmaHTCache {
*/
public static void setCacheSize(final long newCacheSize) {
maxCacheSize = newCacheSize;
fileDBunbuffered.setMaxSize(maxCacheSize);
}
/**
@ -201,14 +189,6 @@ public final class plasmaHTCache {
fileDB.close();
}
public static boolean full() {
return (cacheStack.size() > stackLimit);
}
public static boolean empty() {
return (cacheStack.size() == 0);
}
public static boolean isPicture(final String mimeType) {
if (mimeType == null) return false;
return mimeType.toUpperCase().startsWith("IMAGE");

11
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -65,9 +65,9 @@ public class plasmaSnippetCache {
public static final int ERROR_PARSER_NO_LINES = 15;
public static final int ERROR_NO_MATCH = 16;
private static int snippetsScoreCounter = 0;
private static int snippetsScoreCounter = 0;
private static kelondroMScoreCluster<String> snippetsScore = null;
private static final HashMap<String, String> snippetsCache = new HashMap<String, String>();
private static final HashMap<String, String> snippetsCache = new HashMap<String, String>();
/**
* a cache holding URLs to favicons specified by the page content, e.g. by using the html link-tag. e.g.
@ -78,13 +78,16 @@ public class plasmaSnippetCache {
private static final HashMap<String, yacyURL> faviconCache = new HashMap<String, yacyURL>();
private static plasmaParser parser = null;
private static serverLog log = null;
private static plasmaSwitchboard sb = null;
public static void init(
final plasmaParser parserx,
final serverLog logx
final serverLog logx,
final plasmaSwitchboard switchboard
) {
parser = parserx;
log = logx;
sb = switchboard;
snippetsScoreCounter = 0;
snippetsScore = new kelondroMScoreCluster<String>();
snippetsCache.clear();
@ -290,7 +293,7 @@ public class plasmaSnippetCache {
// getting resource metadata (e.g. the http headers for http resources)
if (entry != null) {
// place entry on crawl queue
plasmaHTCache.push(entry);
sb.htEntryStoreProcess(entry);
// read resource body (if it is there)
final byte []resourceArray = entry.cacheArray();

44
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -544,7 +544,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
// generate snippets cache
log.logConfig("Initializing Snippet Cache");
plasmaSnippetCache.init(parser, log);
plasmaSnippetCache.init(parser, log, this);
final String wikiParserClassName = getConfig(plasmaSwitchboardConstants.WIKIPARSER_CLASS, plasmaSwitchboardConstants.WIKIPARSER_CLASS_DEFAULT);
this.log.logConfig("Loading wiki parser " + wikiParserClassName + " ...");
@ -590,24 +590,21 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
moreMemory.schedule(new MoreMemory(), 300000, 600000);
deployThread(plasmaSwitchboardConstants.CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
new serverInstantBusyThread(this, plasmaSwitchboardConstants.CLEANUP_METHOD_START, plasmaSwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run
new serverInstantBusyThread(this, plasmaSwitchboardConstants.CLEANUP_METHOD_START, plasmaSwitchboardConstants.CLEANUP_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CLEANUP_METHOD_FREEMEM), 600000); // all 5 Minutes, wait 10 minutes until first run
deployThread(plasmaSwitchboardConstants.CRAWLSTACK, "Crawl URL Stacker", "process that checks url for double-occurrences and for allowance/disallowance by robots.txt", null,
new serverInstantBusyThread(crawlStacker, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_START, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_FREEMEM), 8000);
new serverInstantBusyThread(crawlStacker, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_START, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLSTACK_METHOD_FREEMEM), 8000);
deployThread(plasmaSwitchboardConstants.INDEXER, "Indexing", "thread that either initiates a parsing/indexing queue, distributes the index into the DHT, stores parsed documents or flushes the index cache", "/IndexCreateIndexingQueue_p.html",
new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEXER_METHOD_START, plasmaSwitchboardConstants.INDEXER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEXER_METHOD_FREEMEM), 10000);
deployThread(plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE, "Proxy Cache Enqueue", "job takes new input files from RAM stack, stores them, and hands over to the Indexing Stack", null,
new serverInstantBusyThread(this, plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_METHOD_START, plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_METHOD_FREEMEM), 10000);
new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEXER_METHOD_START, plasmaSwitchboardConstants.INDEXER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEXER_METHOD_FREEMEM), 10000);
deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000);
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000);
deployThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER, "Remote Crawl URL Loader", "thread that loads remote crawl lists from other peers", "",
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here?
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_METHOD_FREEMEM), 30000); // error here?
deployThread(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html",
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000);
new serverInstantBusyThread(crawlQueues, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_START, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000);
deployThread(plasmaSwitchboardConstants.SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null,
new serverInstantBusyThread(yc, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_START, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000);
new serverInstantBusyThread(yc, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_START, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_JOBCOUNT, plasmaSwitchboardConstants.SEED_UPLOAD_METHOD_FREEMEM), 180000);
deployThread(plasmaSwitchboardConstants.PEER_PING, "YaCy Core", "this is the p2p-control and peer-ping task", null,
new serverInstantBusyThread(yc, plasmaSwitchboardConstants.PEER_PING_METHOD_START, plasmaSwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000);
new serverInstantBusyThread(yc, plasmaSwitchboardConstants.PEER_PING_METHOD_START, plasmaSwitchboardConstants.PEER_PING_METHOD_JOBCOUNT, plasmaSwitchboardConstants.PEER_PING_METHOD_FREEMEM), 2000);
deployThread(plasmaSwitchboardConstants.INDEX_DIST, "DHT Distribution", "selection, transfer and deletion of index entries that are not searched on your peer, but on others", null,
new serverInstantBusyThread(this, plasmaSwitchboardConstants.INDEX_DIST_METHOD_START, plasmaSwitchboardConstants.INDEX_DIST_METHOD_JOBCOUNT, plasmaSwitchboardConstants.INDEX_DIST_METHOD_FREEMEM), 60000,
Long.parseLong(getConfig(plasmaSwitchboardConstants.INDEX_DIST_IDLESLEEP , "5000")),
@ -932,14 +929,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
new plasmaSearchRankingProfile("", crypt.simpleDecode(sb.getConfig("rankingProfile", ""), null));
}
/**
* This method changes the HTCache size.<br>
* @param newCacheSize in MB
*/
public final void setCacheSize(final long newCacheSize) {
plasmaHTCache.setCacheSize(1048576 * newCacheSize);
}
public boolean onlineCaution() {
return
(System.currentTimeMillis() - this.proxyLastAccess < Integer.parseInt(getConfig(plasmaSwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, "30000"))) ||
@ -1081,15 +1070,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
return true;
}
public boolean htEntryStoreJob() {
if (plasmaHTCache.empty()) return false;
return htEntryStoreProcess(plasmaHTCache.pop());
}
public int htEntrySize() {
return plasmaHTCache.size();
}
public void close() {
log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
serverProfiling.stopSystemProfiling();
@ -1884,12 +1864,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
thread.setIdleSleep(2000);
}
thread = getThread(plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE);
if (thread != null) {
setConfig(plasmaSwitchboardConstants.PROXY_CACHE_ENQUEUE_BUSYSLEEP , thread.setBusySleep(0));
thread.setIdleSleep(2000);
}
thread = getThread(plasmaSwitchboardConstants.INDEXER);
if (thread != null) {
setConfig(plasmaSwitchboardConstants.INDEXER_BUSYSLEEP , thread.setBusySleep(newBusySleep / 8));

14
source/de/anomic/plasma/plasmaSwitchboardConstants.java
Unescape View File

@ -110,20 +110,6 @@ public final class plasmaSwitchboardConstants {
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM = null;
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep";
public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep";
// 70_cachemanager
/**
* <p><code>public static final String <strong>PROXY_CACHE_ENQUEUE</strong> = "70_cachemanager"</code></p>
* <p>Name of the proxy cache enqueue thread which fetches a given website and saves the site itself as well as it's
* HTTP-headers in the HTCACHE</p>
*
* @see plasmaSwitchboard#PROXY_CACHE_PATH
*/
public static final String PROXY_CACHE_ENQUEUE = "70_cachemanager";
public static final String PROXY_CACHE_ENQUEUE_METHOD_START = "htEntryStoreJob";
public static final String PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT = "htEntrySize";
public static final String PROXY_CACHE_ENQUEUE_METHOD_FREEMEM = null;
public static final String PROXY_CACHE_ENQUEUE_IDLESLEEP = "70_cachemanager_idlesleep";
public static final String PROXY_CACHE_ENQUEUE_BUSYSLEEP = "70_cachemanager_busysleep";
// 74_parsing
/**
* <p><code>public static final String <strong>INDEXER</strong> = "80_indexing"</code></p>

Write Preview
Loading…
Cancel
Save