diff --git a/htroot/ConfigHTCache_p.html b/htroot/ConfigHTCache_p.html new file mode 100644 index 000000000..e1ef22e96 --- /dev/null +++ b/htroot/ConfigHTCache_p.html @@ -0,0 +1,33 @@ + + + + YaCy '#[clientname]#': Indexing with Proxy + #%env/templates/metas.template%# + + + #%env/templates/header.template%# + #%env/templates/submenuConfig.template%# +

Hypertext Cache Configuration

+

+ The HTCache stores content retrieved by the HTTP and FTP protocol. Documents from smb:// and file:// locations are not cached. + The cache is a rotating cache: if it is full, then the oldest entries are deleted and new one can fill the space. +

+ +
+
HTCache Configuration +
+
+
+
+
#[actualCacheSize]# MB
+
+
MB
+
 
+
+
+
+
+ + #%env/templates/footer.template%# + + diff --git a/htroot/ConfigHTCache_p.java b/htroot/ConfigHTCache_p.java new file mode 100644 index 000000000..079ce9b5d --- /dev/null +++ b/htroot/ConfigHTCache_p.java @@ -0,0 +1,76 @@ +// ConfigHTCache_p.java +// --------------------------- +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004 +// +// $LastChangedDate: 2010-09-02 21:24:22 +0200 (Do, 02 Sep 2010) $ +// $LastChangedRevision: 7092 $ +// $LastChangedBy: orbiter $ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +// You must compile this file with +// javac -classpath .:../classes ProxyIndexingMonitor_p.java +// if the shell's current path is HTROOT + +import java.io.File; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.kelondro.logging.Log; + +import de.anomic.http.client.Cache; +import de.anomic.search.SwitchboardConstants; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class ConfigHTCache_p { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + // return variable that accumulates replacements + final serverObjects prop = new serverObjects(); + + String oldProxyCachePath, newProxyCachePath; + int newProxyCacheSize; + + if (post != null && post.containsKey("set")) try { + + // proxyCache - check and create the directory + oldProxyCachePath = env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT); + newProxyCachePath = post.get("HTCachePath", SwitchboardConstants.HTCACHE_PATH_DEFAULT); + newProxyCachePath = newProxyCachePath.replace('\\', '/'); + if (newProxyCachePath.endsWith("/")) { + newProxyCachePath = newProxyCachePath.substring(0, newProxyCachePath.length() - 1); + } + env.setConfig(SwitchboardConstants.HTCACHE_PATH, newProxyCachePath); + final File cache = env.getDataPath(SwitchboardConstants.HTCACHE_PATH, oldProxyCachePath); + if (!cache.isDirectory() && !cache.isFile()) cache.mkdirs(); + + // proxyCacheSize + newProxyCacheSize = post.getInt("maxCacheSize", 64); + if (newProxyCacheSize < 4) { newProxyCacheSize = 4; } + env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize); + Cache.setMaxCacheSize(newProxyCacheSize * 1024 * 1024); + } catch (final Exception e) { + Log.logException(e); + } + + prop.put("HTCachePath", env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT)); + prop.put("actualCacheSize", (Cache.getActualCacheSize() / 1024 / 1024)); + prop.put("maxCacheSize", env.getConfigLong(SwitchboardConstants.PROXY_CACHE_SIZE, 64)); + // return rewrite properties + return prop; + } +} \ No newline at end of file diff --git a/htroot/ProxyIndexingMonitor_p.java b/htroot/ProxyIndexingMonitor_p.java index 0e73b1354..40fb07ee3 100644 --- a/htroot/ProxyIndexingMonitor_p.java +++ b/htroot/ProxyIndexingMonitor_p.java @@ -95,7 +95,7 @@ public class ProxyIndexingMonitor_p { newProxyCacheSize = getStringLong(post.get(SwitchboardConstants.PROXY_CACHE_SIZE, "64")); if (getLong(newProxyCacheSize) < 4) { newProxyCacheSize = "4"; } env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize); - Cache.setCacheSize(Long.parseLong(newProxyCacheSize) * 1024 * 1024); + Cache.setMaxCacheSize(Long.parseLong(newProxyCacheSize) * 1024 * 1024); // implant these settings also into the crawling profile for the proxy if (sb.crawler.defaultProxyProfile == null) { diff --git a/htroot/env/templates/submenuConfig.template b/htroot/env/templates/submenuConfig.template index 5fe7a00f9..67e0c1853 100644 --- a/htroot/env/templates/submenuConfig.template +++ b/htroot/env/templates/submenuConfig.template @@ -16,6 +16,7 @@
  • Advanced Settings
  • Parser Configuration
  • Local robots.txt
    • +
  • Web Cache
  • Advanced Properties
    • \ No newline at end of file diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index a7c0c7595..f4b3ef691 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -96,11 +96,19 @@ public final class Cache { * This method changes the HTCache size.
    * @param the new cache size in bytes */ - public static void setCacheSize(final long newCacheSize) { + public static void setMaxCacheSize(final long newCacheSize) { maxCacheSize = newCacheSize; fileDBunbuffered.setMaxSize(maxCacheSize); } + /** + * get the current actual cache size + * @return + */ + public static long getActualCacheSize() { + return fileDBunbuffered.length(); + } + /** * close the databases */ diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 19b802708..b37a7da1b 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -885,6 +885,10 @@ public final class Switchboard extends serverSwitch { // clear statistic data this.crawlResults.clearStacks(); + // remove heuristics + setConfig("heuristic.site", false); + setConfig("heuristic.scroogle", false); + // relocate this.crawlQueues.relocate(this.queuesRoot); // cannot be closed because the busy threads are working with that object final File mySeedFile = new File(this.networkRoot, yacySeedDB.DBFILE_OWN_SEED); diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 56ab7d68a..43b41a303 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -379,6 +379,7 @@ public class ArrayStack implements BLOB { public void setMaxSize(long maxSize) { this.repositorySizeMax = maxSize; this.fileSizeLimit = Math.min((long) Integer.MAX_VALUE, maxSize / 10L); + executeLimits(); } private void executeLimits() {