- WatchCrawler shows status without JavaScript

- Performance can be scaled + DHT-profile
- names for pool-threads
- some small refactorings


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4923 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent da4b1b5c0d
commit 68c38c2d34

@ -1,51 +0,0 @@
###
### YaCy Init File with background performance-settings
###
# current: 3 times of yacy.init
# performance-settings
# delay-times for permanent loops (milliseconds)
# the idlesleep is the pause that an proces sleeps if the last call to the
# process job was without execution of anything;
# the busysleep is the pause after a full job execution
# the prereq-value is a memory pre-requisite: that much bytes must
# be available/free in the heap; othervise the loop is not executed
# and another idlesleep is performed
20_dhtdistribution_idlesleep=90000
20_dhtdistribution_busysleep=30000
20_dhtdistribution_memprereq=6291456
30_peerping_idlesleep=360000
30_peerping_busysleep=360000
30_peerping_memprereq=1048576
40_peerseedcycle_idlesleep=5400000
40_peerseedcycle_busysleep=3600000
40_peerseedcycle_memprereq=2097152
50_localcrawl_idlesleep=6000
50_localcrawl_busysleep=150
50_localcrawl_busysleep__pro=3
50_localcrawl_memprereq=4194304
50_localcrawl_isPaused=false
60_remotecrawlloader_idlesleep=180000
60_remotecrawlloader_idlesleep__pro=30000
60_remotecrawlloader_busysleep=120000
60_remotecrawlloader_busysleep__pro=6000
60_remotecrawlloader_memprereq=2097152
60_remotecrawlloader_isPaused=false
62_remotetriggeredcrawl_idlesleep=30000
62_remotetriggeredcrawl_busysleep=3000
62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false
70_cachemanager_idlesleep=3000
70_cachemanager_busysleep=3
70_cachemanager_memprereq=1048576
80_indexing_idlesleep=3000
80_indexing_busysleep=30
80_indexing_busysleep__pro=3
80_indexing_memprereq=6291456
82_crawlstack_idlesleep=15000
82_crawlstack_busysleep=3
82_crawlstack_memprereq=1048576
90_cleanup_idlesleep=900000
90_cleanup_busysleep=900000
90_cleanup_memprereq=0

@ -1,7 +1,6 @@
### ###
### YaCy Init File with background performance-settings ### YaCy Init File
### ###
# current: 10 times of yacy.init
# performance-settings # performance-settings
# delay-times for permanent loops (milliseconds) # delay-times for permanent loops (milliseconds)
@ -11,41 +10,41 @@
# the prereq-value is a memory pre-requisite: that much bytes must # the prereq-value is a memory pre-requisite: that much bytes must
# be available/free in the heap; othervise the loop is not executed # be available/free in the heap; othervise the loop is not executed
# and another idlesleep is performed # and another idlesleep is performed
20_dhtdistribution_idlesleep=300000 20_dhtdistribution_idlesleep=5000
20_dhtdistribution_busysleep=100000 20_dhtdistribution_busysleep=2000
20_dhtdistribution_memprereq=6291456 20_dhtdistribution_memprereq=6291456
30_peerping_idlesleep=1200000 30_peerping_idlesleep=120000
30_peerping_busysleep=1200000 30_peerping_busysleep=120000
30_peerping_memprereq=1048576 30_peerping_memprereq=1048576
40_peerseedcycle_idlesleep=18000000 40_peerseedcycle_idlesleep=1800000
40_peerseedcycle_busysleep=12000000 40_peerseedcycle_busysleep=1200000
40_peerseedcycle_memprereq=2097152 40_peerseedcycle_memprereq=2097152
50_localcrawl_idlesleep=20000 50_localcrawl_idlesleep=4000
50_localcrawl_busysleep=5000 50_localcrawl_busysleep=500
50_localcrawl_busysleep__pro=500 50_localcrawl_busysleep__pro=100
50_localcrawl_memprereq=4194304 50_localcrawl_memprereq=4194304
50_localcrawl_isPaused=false 50_localcrawl_isPaused=false
60_remotecrawlloader_idlesleep=600000 60_remotecrawlloader_idlesleep=60000
60_remotecrawlloader_idlesleep__pro=100000 60_remotecrawlloader_idlesleep__pro=10000
60_remotecrawlloader_busysleep=400000 60_remotecrawlloader_busysleep=40000
60_remotecrawlloader_busysleep__pro=20000 60_remotecrawlloader_busysleep__pro=2000
60_remotecrawlloader_memprereq=2097152 60_remotecrawlloader_memprereq=2097152
60_remotecrawlloader_isPaused=false 60_remotecrawlloader_isPaused=false
62_remotetriggeredcrawl_idlesleep=100000 62_remotetriggeredcrawl_idlesleep=10000
62_remotetriggeredcrawl_busysleep=10000 62_remotetriggeredcrawl_busysleep=1000
62_remotetriggeredcrawl_memprereq=6291456 62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false 62_remotetriggeredcrawl_isPaused=false
70_cachemanager_idlesleep=10000 70_cachemanager_idlesleep=1000
70_cachemanager_busysleep=1000 70_cachemanager_busysleep=1
70_cachemanager_memprereq=1048576 70_cachemanager_memprereq=1048576
80_indexing_idlesleep=10000 80_indexing_idlesleep=1000
80_indexing_busysleep=100 80_indexing_busysleep=100
80_indexing_busysleep__pro=10 80_indexing_busysleep__pro=10
80_indexing_memprereq=6291456 80_indexing_memprereq=6291456
82_crawlstack_idlesleep=50000 82_crawlstack_idlesleep=5000
82_crawlstack_busysleep=100 82_crawlstack_busysleep=1
82_crawlstack_memprereq=1048576 82_crawlstack_memprereq=1048576
90_cleanup_idlesleep=3000000 90_cleanup_idlesleep=300000
90_cleanup_busysleep=3000000 90_cleanup_busysleep=300000
90_cleanup_memprereq=0 90_cleanup_memprereq=0

@ -59,8 +59,8 @@ import de.anomic.server.serverDomains;
import de.anomic.server.serverInstantBusyThread; import de.anomic.server.serverInstantBusyThread;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyAccessible; import de.anomic.yacy.yacyAccessible;
import de.anomic.yacy.yacySeed;
public class ConfigBasic { public class ConfigBasic {

@ -60,11 +60,11 @@
<input type="submit" name="submitdelay" value="Submit New Delay Values" />&nbsp;&nbsp;&nbsp; <input type="submit" name="submitdelay" value="Submit New Delay Values" />&nbsp;&nbsp;&nbsp;
Changes take effect immediately<br/> Changes take effect immediately<br/>
<input type="submit" name="submitdefault" value="Use Default Profile:" /> <input type="submit" name="submitdefault" value="Use Default Profile:" />
<select type="submit" name="defaultFile"> <select name="defaultFile">
#{profile}# #{profile}#
<option value="#[filename]#">#[description]#</option> <option value="#[filename]#">#[description]#</option>
#{/profile}# #{/profile}#
</select> </select> and use <select name="multiplier"><option value="0.5">200 %</option><option value="0.75">150 %</option><option value="1" selected="selected">100 %</option><option value="2">50 %</option><option value="3">33 %</option><option value="10">10 %</option></select> of the defiend performance.
</td> </td>
</tr> </tr>
</table> </table>

@ -65,9 +65,8 @@ public class PerformanceQueues_p {
private final static Map<String, String> performanceProfiles = new HashMap<String, String>(4, 0.9f); private final static Map<String, String> performanceProfiles = new HashMap<String, String>(4, 0.9f);
static { static {
// no sorted output! // no sorted output!
performanceProfiles.put("defaults/background.settings", "slow (background)"); performanceProfiles.put("defaults/yacy.init", "default (crawl)");
performanceProfiles.put("defaults/normal.settings", "not so fast"); performanceProfiles.put("defaults/performance_dht.profile", "prefer DHT");
performanceProfiles.put("defaults/yacy.init", "fast (YaCy only)");
} }
@ -112,6 +111,10 @@ public class PerformanceQueues_p {
threads = switchboard.threadNames(); threads = switchboard.threadNames();
int c = 0; int c = 0;
long idleCycles, busyCycles, memshortageCycles; long idleCycles, busyCycles, memshortageCycles;
// set profile?
final double multiplier = (post != null) && post.containsKey("multiplier") ? post.getDouble("multiplier", 1) : 1;
final boolean setProfile = (post != null && post.containsKey("submitdefault"));
final boolean setDelay = (post != null) && (post.containsKey("submitdelay"));
while (threads.hasNext()) { while (threads.hasNext()) {
threadName = threads.next(); threadName = threads.next();
thread = switchboard.getThread(threadName); thread = switchboard.getThread(threadName);
@ -151,7 +154,7 @@ public class PerformanceQueues_p {
prop.putNum("table_" + c + "_execpercycle", (busyCycles == 0) ? -1 : exectime / busyCycles); prop.putNum("table_" + c + "_execpercycle", (busyCycles == 0) ? -1 : exectime / busyCycles);
prop.putNum("table_" + c + "_memusepercycle", (busyCycles == 0) ? -1 : memuse / busyCycles / 1024); prop.putNum("table_" + c + "_memusepercycle", (busyCycles == 0) ? -1 : memuse / busyCycles / 1024);
if ((post != null) && (post.containsKey("submitdelay"))) { if (setDelay) {
// load with new values // load with new values
idlesleep = post.getLong(threadName + "_idlesleep", 1000); idlesleep = post.getLong(threadName + "_idlesleep", 1000);
busysleep = post.getLong(threadName + "_busysleep", 100); busysleep = post.getLong(threadName + "_busysleep", 100);
@ -162,16 +165,12 @@ public class PerformanceQueues_p {
if (idlesleep < 1000) idlesleep = 1000; if (idlesleep < 1000) idlesleep = 1000;
if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; } if (threadName.equals("10_httpd")) { idlesleep = 0; busysleep = 0; memprereq = 0; }
// on-the-fly re-configuration onTheFlyReconfiguration(switchboard, threadName, idlesleep, busysleep, memprereq);
switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq); } if (setProfile) {
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
switchboard.setConfig(threadName + "_memprereq", memprereq);
} if ((post != null) && (post.containsKey("submitdefault"))) {
// load with new values // load with new values
idlesleep = Long.parseLong(d(defaultSettings.get(threadName + "_idlesleep"), "1000")); idlesleep = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_idlesleep"), "1000")) * multiplier);
busysleep = Long.parseLong(d(defaultSettings.get(threadName + "_busysleep"), "100")); busysleep = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_busysleep"), "100")) * multiplier);
memprereq = Long.parseLong(d(defaultSettings.get(threadName + "_memprereq"), "0")); memprereq = (long) (Long.parseLong(d(defaultSettings.get(threadName + "_memprereq"), "0")) * multiplier);
// check values to prevent short-cut loops // check values to prevent short-cut loops
if (idlesleep < 1000) idlesleep = 1000; if (idlesleep < 1000) idlesleep = 1000;
@ -180,11 +179,7 @@ public class PerformanceQueues_p {
if ((threadName.equals("61_globalcrawltrigger")) && (busysleep < 100)) busysleep = 100; if ((threadName.equals("61_globalcrawltrigger")) && (busysleep < 100)) busysleep = 100;
if ((threadName.equals("62_remotetriggeredcrawl")) && (busysleep < 100)) busysleep = 100; if ((threadName.equals("62_remotetriggeredcrawl")) && (busysleep < 100)) busysleep = 100;
// on-the-fly re-configuration onTheFlyReconfiguration(switchboard, threadName, idlesleep, busysleep, memprereq);
switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
switchboard.setConfig(threadName + "_memprereq", memprereq);
} else { } else {
// load with old values // load with old values
idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000")); idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
@ -301,6 +296,22 @@ public class PerformanceQueues_p {
// return rewrite values for templates // return rewrite values for templates
return prop; return prop;
} }
/**
* @param switchboard
* @param threadName
* @param idlesleep
* @param busysleep
* @param memprereq
*/
private static void onTheFlyReconfiguration(plasmaSwitchboard switchboard, String threadName, long idlesleep,
long busysleep, long memprereq) {
// on-the-fly re-configuration
switchboard.setThreadPerformance(threadName, idlesleep, busysleep, memprereq);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
switchboard.setConfig(threadName + "_memprereq", memprereq);
}
private static String d(String a, String b) { private static String d(String a, String b) {
return (a == null) ? b : a; return (a == null) ? b : a;

@ -12,7 +12,7 @@
#%env/templates/header.template%# #%env/templates/header.template%#
#%env/templates/submenuIndexCreate.template%# #%env/templates/submenuIndexCreate.template%#
<h2>Crawler Queues</h2> <h2>Crawler Queues</h2>
<p> Next update in <input type="text" id="nextUpdate" onfocus="changeInterval()" onblur="newInterval()" size="2" /> seconds. <img src="/env/grafics/empty.gif" name="ajax" alt="empty"/> <p> Next update in <input type="text" id="nextUpdate" onfocus="changeInterval()" onblur="newInterval()" size="2" /> seconds. <img src="/env/grafics/empty.gif" name="ajax" alt="empty"/><noscript>(Please enable JavaScript to automatically update this page!)</noscript>
</p> </p>
<table border="0" cellpadding="2" cellspacing="1" class="watchCrawler"> <table border="0" cellpadding="2" cellspacing="1" class="watchCrawler">
<tbody> <tbody>
@ -24,19 +24,19 @@
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">Indexing</td> <td align="left">Indexing</td>
<td align="right"><span id="indexingqueuesize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="indexingqueuesize">#[indexingSize]#</span></td>
<td>&nbsp;</td> <td>&nbsp;</td>
<td align="right"><span id="indexingqueuemax">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="indexingqueuemax">#[indexingMax]#</span></td>
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">Loader</td> <td align="left">Loader</td>
<td align="right"><span id="loaderqueuesize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="loaderqueuesize">#[loaderSize]#</span></td>
<td>&nbsp;</td> <td>&nbsp;</td>
<td align="right"><span id="loaderqueuemax">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="loaderqueuemax">#[loaderMax]#</span></td>
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">Local Crawler</td> <td align="left">Local Crawler</td>
<td align="right"><span id="localcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="localcrawlerqueuesize">#[localCrawlSize]#</span></td>
<td> <td>
<a href="" id="localcrawlerstateA"> <a href="" id="localcrawlerstateA">
<img src="" alt="" style="width:12px; height:12px;" id="localcrawlerstateIMG" /> <img src="" alt="" style="width:12px; height:12px;" id="localcrawlerstateIMG" />
@ -46,7 +46,7 @@
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">Limit Crawler</td> <td align="left">Limit Crawler</td>
<td align="right"><span id="limitcrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="limitcrawlerqueuesize">#[limitCrawlSize]#</span></td>
<td> <td>
<a href="" title="" id="limitcrawlerstateA"> <a href="" title="" id="limitcrawlerstateA">
<img src="" alt="" style="width:12px; height:12px;" id="limitcrawlerstateIMG" /> <img src="" alt="" style="width:12px; height:12px;" id="limitcrawlerstateIMG" />
@ -56,7 +56,7 @@
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">Remote Crawler</td> <td align="left">Remote Crawler</td>
<td align="right"><span id="remotecrawlerqueuesize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="remotecrawlerqueuesize">#[remoteCrawlSize]#</span></td>
<td> <td>
<a href="" title="" id="remotecrawlerstateA"> <a href="" title="" id="remotecrawlerstateA">
<img src="" alt="" style="width:12px; height:12px;" id="remotecrawlerstateIMG" /> <img src="" alt="" style="width:12px; height:12px;" id="remotecrawlerstateIMG" />
@ -90,11 +90,11 @@
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">Pages (URLs)</td> <td align="left">Pages (URLs)</td>
<td align="right"><span id="urldbsize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="urldbsize">#[urlpublictextSize]#</span></td>
</tr> </tr>
<tr class="TableCellLight"> <tr class="TableCellLight">
<td align="left">RWIs (Words)</td> <td align="left">RWIs (Words)</td>
<td align="right"><span id="rwidbsize">&nbsp;&nbsp;&nbsp;</span></td> <td align="right"><span id="rwidbsize">#[rwipublictextSize]#</span></td>
</tr> </tr>
</tbody> </tbody>
</table> </table>

@ -35,6 +35,7 @@ import java.util.Map;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException; import java.util.regex.PatternSyntaxException;
import xml.queues_p;
import de.anomic.crawler.CrawlEntry; import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.SitemapImporter; import de.anomic.crawler.SitemapImporter;
@ -62,14 +63,13 @@ public class WatchCrawler_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements // return variable that accumulates replacements
plasmaSwitchboard sb = (plasmaSwitchboard) env; plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects(); // inital values for AJAX Elements (without JavaScript)
serverObjects prop = queues_p.respond(header, post, env);
prop.put("forwardToCrawlStart", "0"); prop.put("forwardToCrawlStart", "0");
if (post == null) { prop.put("info", "0");
// not a crawl start, only monitoring if (post != null) {
prop.put("info", "0"); // a crawl start
} else {
prop.put("info", "0");
if ((post.containsKey("autoforward")) && if ((post.containsKey("autoforward")) &&
(sb.crawlQueues.coreCrawlJobSize() == 0) && (sb.crawlQueues.coreCrawlJobSize() == 0) &&

@ -48,9 +48,9 @@ import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;

@ -65,8 +65,8 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;

@ -64,8 +64,8 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;

@ -75,8 +75,8 @@ import org.xml.sax.SAXException;
import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterWriter; import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.index.indexWord; import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;

@ -49,8 +49,8 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.TimeZone; import java.util.TimeZone;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;

@ -55,9 +55,9 @@ import java.util.Map;
import java.util.Random; import java.util.Random;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;

@ -49,8 +49,8 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.TimeZone; import java.util.TimeZone;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;

@ -135,7 +135,7 @@ public class kelondroBLOBTree implements kelondroBLOB {
return key.getBytes(); return key.getBytes();
} }
private String origKey(byte[] rawKey) { String origKey(byte[] rawKey) {
int n = keylen - 1; int n = keylen - 1;
if (n >= rawKey.length) n = rawKey.length - 1; if (n >= rawKey.length) n = rawKey.length - 1;
while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--; while ((n > 0) && (rawKey[n] == (byte) fillChar)) n--;
@ -227,7 +227,7 @@ public class kelondroBLOBTree implements kelondroBLOB {
} }
} }
private synchronized int get(String key, int pos) throws IOException { synchronized int get(String key, int pos) throws IOException {
int reccnt = pos / reclen; int reccnt = pos / reclen;
// read within a single record // read within a single record
byte[] buf = getValueCached(elementKey(key, reccnt)); byte[] buf = getValueCached(elementKey(key, reccnt));

@ -38,6 +38,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import de.anomic.server.NamePrefixThreadFactory;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory; import de.anomic.server.serverMemory;
import de.anomic.server.serverProcessor; import de.anomic.server.serverProcessor;
@ -54,7 +55,7 @@ public class kelondroRowCollection {
static { static {
if (serverProcessor.useCPU > 1) { if (serverProcessor.useCPU > 1) {
sortingthreadexecutor = Executors.newCachedThreadPool(); sortingthreadexecutor = Executors.newCachedThreadPool(new NamePrefixThreadFactory("sorting"));
} else { } else {
sortingthreadexecutor = null; sortingthreadexecutor = null;
} }

@ -47,6 +47,7 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import de.anomic.server.NamePrefixThreadFactory;
import de.anomic.server.serverMemory; import de.anomic.server.serverMemory;
import de.anomic.server.serverProcessor; import de.anomic.server.serverProcessor;
@ -80,7 +81,7 @@ public class kelondroSplitTable implements kelondroIndex {
public void init(boolean resetOnFail) { public void init(boolean resetOnFail) {
// init the thread pool for the keeperOf executor service // init the thread pool for the keeperOf executor service
this.executor = new ThreadPoolExecutor(serverProcessor.useCPU + 1, serverProcessor.useCPU + 1, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); this.executor = new ThreadPoolExecutor(serverProcessor.useCPU + 1, serverProcessor.useCPU + 1, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamePrefixThreadFactory(tablename));
// initialized tables map // initialized tables map
this.tables = new HashMap<String, kelondroIndex>(); this.tables = new HashMap<String, kelondroIndex>();

@ -71,8 +71,8 @@ import java.util.regex.Pattern;
import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.plasma.cache.IResourceInfo; import de.anomic.plasma.cache.IResourceInfo;

@ -0,0 +1,69 @@
// NamePrefixThreadFactory.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 13.06.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.server;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
public /**
* creates threads whose names begin with a specified prefix for identifing purpose
*
* @author daniel
*/
class NamePrefixThreadFactory implements ThreadFactory {
/**
* default as backend
*/
private final static ThreadFactory defaultFactory = Executors.defaultThreadFactory();
/**
* pefix of each threadname
*/
private final String prefix;
/**
* constructor
*
* @param prefix each thread is named 'prefix' + defaultName
*/
public NamePrefixThreadFactory(final String prefix) {
this.prefix = prefix;
}
/*
* (non-Javadoc)
*
* @see java.util.concurrent.ThreadFactory#newThread(java.lang.Runnable)
*/
public Thread newThread(final Runnable r) {
final Thread t = defaultFactory.newThread(r);
t.setName(this.prefix + "_" + t.getName());
return t;
}
}

@ -56,7 +56,7 @@ public class serverProcessor<J extends serverProcessorJob> {
this.input = new LinkedBlockingQueue<J>(inputQueueSize); this.input = new LinkedBlockingQueue<J>(inputQueueSize);
this.output = output; this.output = output;
this.poolsize = poolsize; this.poolsize = poolsize;
executor = Executors.newCachedThreadPool(); executor = Executors.newCachedThreadPool(new NamePrefixThreadFactory(jobExec));
for (int i = 0; i < poolsize; i++) { for (int i = 0; i < poolsize; i++) {
executor.submit(new serverInstantBlockingThread<J>(env, jobExec, input, output)); executor.submit(new serverInstantBlockingThread<J>(env, jobExec, input, output));
} }

@ -55,13 +55,13 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import de.anomic.plasma.plasmaSwitchboard;
// FIXME entfernen // FIXME entfernen
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.plasma.plasmaSwitchboard;
public class diskUsage { public class diskUsage {
// FIXME entfernen // FIXME entfernen
private serverLog log = new serverLog("DISK USAGE"); serverLog log = new serverLog("DISK USAGE");
private static final HashMap<String, long[]> diskUsages = new HashMap<String, long[]>(); private static final HashMap<String, long[]> diskUsages = new HashMap<String, long[]>();
private static final ArrayList<String> allVolumes = new ArrayList<String>(); private static final ArrayList<String> allVolumes = new ArrayList<String>();
@ -458,7 +458,7 @@ nextLine:
consoleError = false; consoleError = false;
try { try {
process = processBuilder.start();; process = processBuilder.start();
inputStream = new consoleInterface(process.getInputStream()); inputStream = new consoleInterface(process.getInputStream());
errorStream = new consoleInterface(process.getErrorStream()); errorStream = new consoleInterface(process.getErrorStream());
@ -466,7 +466,7 @@ nextLine:
inputStream.start(); inputStream.start();
errorStream.start(); errorStream.start();
int retval = process.waitFor(); /*int retval =*/ process.waitFor();
} catch(IOException iox) { } catch(IOException iox) {
consoleError = true; consoleError = true;

@ -50,7 +50,6 @@ package de.anomic.yacy;
import java.io.File; import java.io.File;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
public class yacyAccessible { public class yacyAccessible {

@ -65,8 +65,8 @@ import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.http.httpdAlternativeDomainNames; import de.anomic.http.httpdAlternativeDomainNames;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;

@ -78,8 +78,8 @@ import de.anomic.index.indexRWIRowEntry;
import de.anomic.index.indexRepositoryReference; import de.anomic.index.indexRepositoryReference;
import de.anomic.index.indexURLReference; import de.anomic.index.indexURLReference;
import de.anomic.index.indexWord; import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects; import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.kelondro.kelondroRowCollection; import de.anomic.kelondro.kelondroRowCollection;
@ -198,14 +198,15 @@ public final class yacy {
} }
// setting up logging // setting up logging
f = new File(homePath, "DATA/LOG/"); if (!(f.exists())) f.mkdirs(); f = new File(homePath, "DATA/LOG/yacy.logging");
if (!((new File(homePath, "DATA/LOG/yacy.logging")).exists())) try { if (!(new File(f.getPath()).exists())) f.mkdirs();
serverFileUtils.copy(new File(homePath, "yacy.logging"), new File(homePath, "DATA/LOG/yacy.logging")); if (!f.exists()) try {
serverFileUtils.copy(new File(homePath, "yacy.logging"), f);
}catch (IOException e){ }catch (IOException e){
System.out.println("could not copy yacy.logging"); System.out.println("could not copy yacy.logging");
} }
try{ try{
serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging")); serverLog.configureLogging(homePath, f);
} catch (IOException e) { } catch (IOException e) {
System.out.println("could not find logging properties in homePath=" + homePath); System.out.println("could not find logging properties in homePath=" + homePath);
e.printStackTrace(); e.printStackTrace();

Loading…
Cancel
Save