(4, 0.9f);
static {
// no sorted output!
- performanceProfiles.put("defaults/background.settings", "slow (background)");
- performanceProfiles.put("defaults/normal.settings", "not so fast");
- performanceProfiles.put("defaults/yacy.init", "fast (YaCy only)");
+ performanceProfiles.put("defaults/yacy.init", "default (crawl)");
+ performanceProfiles.put("defaults/performance_dht.profile", "prefer DHT");
}
@@ -112,6 +111,10 @@ public class PerformanceQueues_p {
threads = switchboard.threadNames();
int c = 0;
long idleCycles, busyCycles, memshortageCycles;
+ // set profile?
+ final double multiplier = (post != null) && post.containsKey("multiplier") ? post.getDouble("multiplier", 1) : 1;
+ final boolean setProfile = (post != null && post.containsKey("submitdefault"));
+ final boolean setDelay = (post != null) && (post.containsKey("submitdelay"));
while (threads.hasNext()) {
threadName = threads.next();
thread = switchboard.getThread(threadName);
@@ -151,7 +154,7 @@ public class PerformanceQueues_p {
prop.putNum("table_" + c + "_execpercycle", (busyCycles == 0) ? -1 : exectime / busyCycles);
prop.putNum("table_" + c + "_memusepercycle", (busyCycles == 0) ? -1 : memuse / busyCycles / 1024);
- if ((post != null) && (post.containsKey("submitdelay"))) {
+ if (setDelay) {
// load with new values
idlesleep = post.getLong(threadName + "_idlesleep", 1000);
busysleep = post.getLong(threadName + "_busysleep", 100);
@@ -162,16 +165,12 @@ public class PerformanceQueues_p {
if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
- // on-the-fly re-configuration
- switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
- switchboard.setConfig(threadName + "_idlesleep", idlesleep);
- switchboard.setConfig(threadName + "_busysleep", busysleep);
- switchboard.setConfig(threadName + "_memprereq", memprereq);
- } if ((post != null) && (post.containsKey("submitdefault"))) {
+ onTheFlyReconfiguration(switchboard, threadName, idlesleep, busysleep, memprereq);
+ } if (setProfile) {
// load with new values
- idlesleep = Long.parseLong(d(defaultSettings.get(threadName + "_idlesleep"), "1000"));
- busysleep = Long.parseLong(d(defaultSettings.get(threadName + "_busysleep"), "100"));
- memprereq = Long.parseLong(d(defaultSettings.get(threadName + "_memprereq"), "0"));
+ idlesleep = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_idlesleep"), "1000")) * multiplier);
+ busysleep = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_busysleep"), "100")) * multiplier);
+ memprereq = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_memprereq"), "0")) * multiplier);
// check values to prevent short-cut loops
if (idlesleep < 1000) idlesleep = 1000;
@@ -180,11 +179,7 @@ public class PerformanceQueues_p {
if ((threadName.equals("61_globalcrawltrigger")) && (busysleep < 100)) busysleep = 100;
if ((threadName.equals("62_remotetriggeredcrawl")) && (busysleep < 100)) busysleep = 100;
- // on-the-fly re-configuration
- switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
- switchboard.setConfig(threadName + "_idlesleep", idlesleep);
- switchboard.setConfig(threadName + "_busysleep", busysleep);
- switchboard.setConfig(threadName + "_memprereq", memprereq);
+ onTheFlyReconfiguration(switchboard, threadName, idlesleep, busysleep, memprereq);
} else {
// load with old values
idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
@@ -301,6 +296,22 @@ public class PerformanceQueues_p {
// return rewrite values for templates
return prop;
}
+
+ /**
+ * @param switchboard
+ * @param threadName
+ * @param idlesleep
+ * @param busysleep
+ * @param memprereq
+ */
+ private static void onTheFlyReconfiguration(plasmaSwitchboard switchboard, String threadName, long idlesleep,
+ long busysleep, long memprereq) {
+ // on-the-fly re-configuration
+ switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
+ switchboard.setConfig(threadName + "_idlesleep", idlesleep);
+ switchboard.setConfig(threadName + "_busysleep", busysleep);
+ switchboard.setConfig(threadName + "_memprereq", memprereq);
+ }
private static String d(String a, String b) {
return (a == null) ? b : a;
diff --git a/htroot/WatchCrawler_p.html b/htroot/WatchCrawler_p.html
index 217fd75d8..736957a3c 100644
--- a/htroot/WatchCrawler_p.html
+++ b/htroot/WatchCrawler_p.html
@@ -12,7 +12,7 @@
#%env/templates/header.template%#
#%env/templates/submenuIndexCreate.template%#
Crawler Queues
- Next update in seconds. 
+
Next update in seconds.
@@ -24,19 +24,19 @@
| Indexing |
- |
+ #[indexingSize]# |
|
- |
+ #[indexingMax]# |
| Loader |
- |
+ #[loaderSize]# |
|
- |
+ #[loaderMax]# |
| Local Crawler |
- |
+ #[localCrawlSize]# |
![]()
@@ -46,7 +46,7 @@
|
| Limit Crawler |
- |
+ #[limitCrawlSize]# |
![]()
@@ -56,7 +56,7 @@
|
| Remote Crawler |
- |
+ #[remoteCrawlSize]# |
![]()
@@ -90,11 +90,11 @@
|
| Pages (URLs) |
- |
+ #[urlpublictextSize]# |
| RWIs (Words) |
- |
+ #[rwipublictextSize]# |
diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java
index 0cf2a80fb..b862c78e0 100644
--- a/htroot/WatchCrawler_p.java
+++ b/htroot/WatchCrawler_p.java
@@ -35,6 +35,7 @@ import java.util.Map;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import xml.queues_p;
import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.SitemapImporter;
@@ -62,14 +63,13 @@ public class WatchCrawler_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
plasmaSwitchboard sb = (plasmaSwitchboard) env;
- serverObjects prop = new serverObjects();
+ // inital values for AJAX Elements (without JavaScript)
+ serverObjects prop = queues_p.respond(header, post, env);
prop.put("forwardToCrawlStart", "0");
- if (post == null) {
- // not a crawl start, only monitoring
- prop.put("info", "0");
- } else {
- prop.put("info", "0");
+ prop.put("info", "0");
+ if (post != null) {
+ // a crawl start
if ((post.containsKey("autoforward")) &&
(sb.crawlQueues.coreCrawlJobSize() == 0) &&
diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java
index debe12dc1..f6f9463c2 100644
--- a/source/de/anomic/crawler/CrawlProfile.java
+++ b/source/de/anomic/crawler/CrawlProfile.java
@@ -48,9 +48,9 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
-import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java
index fbbdddb77..46e1c9257 100644
--- a/source/de/anomic/data/blogBoard.java
+++ b/source/de/anomic/data/blogBoard.java
@@ -65,8 +65,8 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java
index 8003d22a5..e51902a5c 100644
--- a/source/de/anomic/data/blogBoardComments.java
+++ b/source/de/anomic/data/blogBoardComments.java
@@ -64,8 +64,8 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.logging.serverLog;
diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java
index c2b08932f..0801156dd 100644
--- a/source/de/anomic/data/bookmarksDB.java
+++ b/source/de/anomic/data/bookmarksDB.java
@@ -75,8 +75,8 @@ import org.xml.sax.SAXException;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.index.indexWord;
-import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java
index 3c000bb97..3d9d796d9 100644
--- a/source/de/anomic/data/messageBoard.java
+++ b/source/de/anomic/data/messageBoard.java
@@ -49,8 +49,8 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.TimeZone;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java
index af3bbfb35..aa3eeb67f 100644
--- a/source/de/anomic/data/userDB.java
+++ b/source/de/anomic/data/userDB.java
@@ -55,9 +55,9 @@ import java.util.Map;
import java.util.Random;
import de.anomic.http.httpHeader;
+import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
-import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java
index ab7bf9984..e2b20b3ea 100644
--- a/source/de/anomic/data/wikiBoard.java
+++ b/source/de/anomic/data/wikiBoard.java
@@ -49,8 +49,8 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.TimeZone;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder;
diff --git a/source/de/anomic/kelondro/kelondroBLOBTree.java b/source/de/anomic/kelondro/kelondroBLOBTree.java
index b1be0d41e..396d51aa7 100644
--- a/source/de/anomic/kelondro/kelondroBLOBTree.java
+++ b/source/de/anomic/kelondro/kelondroBLOBTree.java
@@ -135,7 +135,7 @@ public class kelondroBLOBTree implements kelondroBLOB {
return key.getBytes();
}
- private String origKey(byte[] rawKey) {
+ String origKey(byte[] rawKey) {
int n = keylen - 1;
if (n >= rawKey.length) n = rawKey.length - 1;
while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--;
@@ -227,7 +227,7 @@ public class kelondroBLOBTree implements kelondroBLOB {
}
}
- private synchronized int get(String key, int pos) throws IOException {
+ synchronized int get(String key, int pos) throws IOException {
int reccnt = pos / reclen;
// read within a single record
byte[] buf = getValueCached(elementKey(key, reccnt));
diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java
index 08f4f15f3..709d6a64d 100644
--- a/source/de/anomic/kelondro/kelondroRowCollection.java
+++ b/source/de/anomic/kelondro/kelondroRowCollection.java
@@ -38,6 +38,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import de.anomic.server.NamePrefixThreadFactory;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.serverProcessor;
@@ -54,7 +55,7 @@ public class kelondroRowCollection {
static {
if (serverProcessor.useCPU > 1) {
- sortingthreadexecutor = Executors.newCachedThreadPool();
+ sortingthreadexecutor = Executors.newCachedThreadPool(new NamePrefixThreadFactory("sorting"));
} else {
sortingthreadexecutor = null;
}
diff --git a/source/de/anomic/kelondro/kelondroSplitTable.java b/source/de/anomic/kelondro/kelondroSplitTable.java
index 9da966f87..7d1c6648c 100644
--- a/source/de/anomic/kelondro/kelondroSplitTable.java
+++ b/source/de/anomic/kelondro/kelondroSplitTable.java
@@ -47,6 +47,7 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import de.anomic.server.NamePrefixThreadFactory;
import de.anomic.server.serverMemory;
import de.anomic.server.serverProcessor;
@@ -80,7 +81,7 @@ public class kelondroSplitTable implements kelondroIndex {
public void init(boolean resetOnFail) {
// init the thread pool for the keeperOf executor service
- this.executor = new ThreadPoolExecutor(serverProcessor.useCPU + 1, serverProcessor.useCPU + 1, 10, TimeUnit.SECONDS, new LinkedBlockingQueue());
+ this.executor = new ThreadPoolExecutor(serverProcessor.useCPU + 1, serverProcessor.useCPU + 1, 10, TimeUnit.SECONDS, new LinkedBlockingQueue(), new NamePrefixThreadFactory(tablename));
// initialized tables map
this.tables = new HashMap();
diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java
index 3abb6d4f3..0841864fa 100644
--- a/source/de/anomic/plasma/plasmaHTCache.java
+++ b/source/de/anomic/plasma/plasmaHTCache.java
@@ -71,8 +71,8 @@ import java.util.regex.Pattern;
import de.anomic.crawler.CrawlProfile;
import de.anomic.http.httpHeader;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.plasma.cache.IResourceInfo;
diff --git a/source/de/anomic/server/NamePrefixThreadFactory.java b/source/de/anomic/server/NamePrefixThreadFactory.java
new file mode 100644
index 000000000..7f942f62b
--- /dev/null
+++ b/source/de/anomic/server/NamePrefixThreadFactory.java
@@ -0,0 +1,69 @@
+// NamePrefixThreadFactory.java
+// (C) 2008 by Daniel Raap; danielr@users.berlios.de
+// first published 13.06.2008 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
+// $LastChangedRevision: 4558 $
+// $LastChangedBy: orbiter $
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package de.anomic.server;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
+public /**
+ * creates threads whose names begin with a specified prefix for identifing purpose
+ *
+ * @author daniel
+ */
+class NamePrefixThreadFactory implements ThreadFactory {
+
+ /**
+ * default as backend
+ */
+ private final static ThreadFactory defaultFactory = Executors.defaultThreadFactory();
+
+ /**
+ * pefix of each threadname
+ */
+ private final String prefix;
+
+ /**
+ * constructor
+ *
+ * @param prefix each thread is named 'prefix' + defaultName
+ */
+ public NamePrefixThreadFactory(final String prefix) {
+ this.prefix = prefix;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.util.concurrent.ThreadFactory#newThread(java.lang.Runnable)
+ */
+ public Thread newThread(final Runnable r) {
+ final Thread t = defaultFactory.newThread(r);
+ t.setName(this.prefix + "_" + t.getName());
+ return t;
+ }
+
+}
diff --git a/source/de/anomic/server/serverProcessor.java b/source/de/anomic/server/serverProcessor.java
index e4f197e2c..85bf07e18 100644
--- a/source/de/anomic/server/serverProcessor.java
+++ b/source/de/anomic/server/serverProcessor.java
@@ -56,7 +56,7 @@ public class serverProcessor {
this.input = new LinkedBlockingQueue(inputQueueSize);
this.output = output;
this.poolsize = poolsize;
- executor = Executors.newCachedThreadPool();
+ executor = Executors.newCachedThreadPool(new NamePrefixThreadFactory(jobExec));
for (int i = 0; i < poolsize; i++) {
executor.submit(new serverInstantBlockingThread(env, jobExec, input, output));
}
diff --git a/source/de/anomic/tools/diskUsage.java b/source/de/anomic/tools/diskUsage.java
index 5884f3457..09fd84b99 100644
--- a/source/de/anomic/tools/diskUsage.java
+++ b/source/de/anomic/tools/diskUsage.java
@@ -55,13 +55,13 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import de.anomic.plasma.plasmaSwitchboard;
// FIXME entfernen
import de.anomic.server.logging.serverLog;
-import de.anomic.plasma.plasmaSwitchboard;
public class diskUsage {
// FIXME entfernen
- private serverLog log = new serverLog("DISK USAGE");
+ serverLog log = new serverLog("DISK USAGE");
private static final HashMap diskUsages = new HashMap();
private static final ArrayList allVolumes = new ArrayList();
@@ -458,7 +458,7 @@ nextLine:
consoleError = false;
try {
- process = processBuilder.start();;
+ process = processBuilder.start();
inputStream = new consoleInterface(process.getInputStream());
errorStream = new consoleInterface(process.getErrorStream());
@@ -466,7 +466,7 @@ nextLine:
inputStream.start();
errorStream.start();
- int retval = process.waitFor();
+ /*int retval =*/ process.waitFor();
} catch(IOException iox) {
consoleError = true;
diff --git a/source/de/anomic/yacy/yacyAccessible.java b/source/de/anomic/yacy/yacyAccessible.java
index afc7289ed..2e281ae9d 100644
--- a/source/de/anomic/yacy/yacyAccessible.java
+++ b/source/de/anomic/yacy/yacyAccessible.java
@@ -50,7 +50,6 @@ package de.anomic.yacy;
import java.io.File;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.serverCore;
import de.anomic.server.serverFileUtils;
public class yacyAccessible {
diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java
index f14075c40..87976f5c1 100644
--- a/source/de/anomic/yacy/yacySeedDB.java
+++ b/source/de/anomic/yacy/yacySeedDB.java
@@ -65,8 +65,8 @@ import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
import de.anomic.http.httpd;
import de.anomic.http.httpdAlternativeDomainNames;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
diff --git a/source/yacy.java b/source/yacy.java
index 7586edd3b..6716d994a 100644
--- a/source/yacy.java
+++ b/source/yacy.java
@@ -78,8 +78,8 @@ import de.anomic.index.indexRWIRowEntry;
import de.anomic.index.indexRepositoryReference;
import de.anomic.index.indexURLReference;
import de.anomic.index.indexWord;
-import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree;
+import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroRowCollection;
@@ -198,14 +198,15 @@ public final class yacy {
}
// setting up logging
- f = new File(homePath, "DATA/LOG/"); if (!(f.exists())) f.mkdirs();
- if (!((new File(homePath, "DATA/LOG/yacy.logging")).exists())) try {
- serverFileUtils.copy(new File(homePath, "yacy.logging"), new File(homePath, "DATA/LOG/yacy.logging"));
+ f = new File(homePath, "DATA/LOG/yacy.logging");
+ if (!(new File(f.getPath()).exists())) f.mkdirs();
+ if (!f.exists()) try {
+ serverFileUtils.copy(new File(homePath, "yacy.logging"), f);
}catch (IOException e){
System.out.println("could not copy yacy.logging");
}
try{
- serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));
+ serverLog.configureLogging(homePath, f);
} catch (IOException e) {
System.out.println("could not find logging properties in homePath=" + homePath);
e.printStackTrace();