- more configuration for profiling graph (number of events)

- more logging for a shutdown: print reason and accessing IP into log


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6520 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 5f0f6b71b4
commit 8281e29963

@ -132,7 +132,7 @@ public class ConfigUpdate_p {
prop.put("candeploy_autoUpdate", "4");
} else {
yacyRelease.deployRelease(downloaded);
sb.terminate(5000);
sb.terminate(5000, "manual release update to " + downloaded.getName());
sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated");
prop.put("candeploy_autoUpdate", "1");
}

@ -24,11 +24,10 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.EventTracker;
import de.anomic.http.server.RequestHeader;
import de.anomic.server.serverObjects;
@ -41,15 +40,14 @@ public class PerformanceSearch_p {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final ArrayList<MemoryTracker.Event> events = MemoryTracker.history("SEARCH");
final Iterator<EventTracker.Event> events = EventTracker.getHistory("SEARCH");
int c = 0;
MemoryTracker.Event event;
ProfilingGraph.searchEvent search;
long lastt = 0;
if (events != null) synchronized (events) {
Iterator<MemoryTracker.Event> i = events.iterator();
while (i.hasNext()) {
event = i.next();
if (events != null) {
EventTracker.Event event;
ProfilingGraph.searchEvent search;
long lastt = 0;
while (events.hasNext()) {
event = events.next();
search = (ProfilingGraph.searchEvent) event.payload;
prop.put("table_" + c + "_query", search.queryID);
prop.put("table_" + c + "_event", search.processName);

@ -59,7 +59,7 @@ public class Steering {
if (post.containsKey("shutdown")) {
Log.logInfo("STEERING", "shutdown request from " + requestIP);
sb.terminate(100);
sb.terminate(100, "shutdown request from Steering; ip = " + requestIP);
prop.put("info", "3");
return prop;

@ -41,7 +41,7 @@ import net.yacy.kelondro.data.word.WordReference;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Bitfield;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.SortStack;
import net.yacy.kelondro.util.ISO639;
@ -225,7 +225,7 @@ public final class search {
//final Map<byte[], ReferenceContainer<WordReference>>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls));
final HashMap<byte[], ReferenceContainer<WordReference>> incc = indexSegment.termIndex().searchConjunction(theQuery.queryHashes, QueryParams.hashes2StringSet(urls));
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.COLLECTION, incc.size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.COLLECTION, incc.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
if (incc != null) {
final Iterator<Map.Entry<byte[], ReferenceContainer<WordReference>>> ci = incc.entrySet().iterator();
Map.Entry<byte[], ReferenceContainer<WordReference>> entry;
@ -342,7 +342,7 @@ public final class search {
refstr.append(",").append(e.name);
}
prop.put("references", (refstr.length() > 0) ? refstr.substring(1) : refstr.toString());
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
}
prop.put("indexabstract", indexabstract.toString());
@ -369,7 +369,7 @@ public final class search {
}
prop.put("links", links.toString());
prop.put("linkcount", accu.size());
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
}
// add information about forward peers

@ -44,9 +44,9 @@ import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Bitfield;
import net.yacy.kelondro.util.Domains;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.SetTools;
import net.yacy.kelondro.util.ISO639;
import net.yacy.repository.LoaderDispatcher;
@ -461,7 +461,7 @@ public class yacysearch {
authenticated,
indexSegment,
ranking);
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.INITIALIZATION, 0, 0), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.INITIALIZATION, 0, 0), false, 30000, ProfilingGraph.maxTime);
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(10000);

@ -31,8 +31,8 @@ import java.util.ArrayList;
import java.util.TreeSet;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryTracker;
import de.anomic.http.server.HeaderFramework;
import de.anomic.http.server.RequestHeader;
@ -153,7 +153,7 @@ public class yacysearchitem {
prop.put("content_description", desc);
prop.putXML("content_description-xml", desc);
prop.putJSON("content_description-json", desc);
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + item, 0, 0), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + item, 0, 0), false, 30000, ProfilingGraph.maxTime);
return prop;
}

@ -27,7 +27,7 @@
import java.util.ArrayList;
import java.util.Iterator;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.EventTracker;
import de.anomic.http.server.RequestHeader;
import de.anomic.search.QueryParams;
@ -146,7 +146,7 @@ public class yacysearchtrailer {
prop.put("nav-about_body", aboutBody);
}
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false, 30000, ProfilingGraph.maxTime);
return prop;
}

@ -53,12 +53,12 @@ public class DocumentIndex extends Segment {
private static final RankingProfile textRankingDefault = new RankingProfile(ContentDomain.TEXT);
//private Bitfield zeroConstraint = new Bitfield(4);
private final static File poison = new File(".");
private BlockingQueue<File> queue;
final static File poison = new File(".");
BlockingQueue<File> queue;
private Worker[] worker;
private CallbackListener callback;
CallbackListener callback;
private static final ThreadGroup workerThreadGroup = new ThreadGroup("workerThreadGroup");
static final ThreadGroup workerThreadGroup = new ThreadGroup("workerThreadGroup");
public DocumentIndex(Log log, final File segmentPath, CallbackListener callback, int cachesize) throws IOException {
super(log, segmentPath, cachesize, targetFileSize * 4 - 1, false, false);

@ -52,8 +52,8 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.TermSearch;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.SortStack;
import de.anomic.yacy.graphics.ProfilingGraph;
@ -135,7 +135,7 @@ public final class RankingProcess extends Thread {
query.maxDistance);
this.localSearchInclusion = search.inclusion();
final ReferenceContainer<WordReference> index = search.joined();
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
if (index.isEmpty()) {
return;
}
@ -164,7 +164,7 @@ public final class RankingProcess extends Thread {
// normalize entries
final BlockingQueue<WordReferenceVars> decodedEntries = this.order.normalizeWith(index);
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
// iterate over normalized entries and select some that are better than currently stored
timer = System.currentTimeMillis();
@ -259,7 +259,7 @@ public final class RankingProcess extends Thread {
}
//if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), SearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
}
/**

@ -39,7 +39,7 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.SetTools;
import net.yacy.kelondro.util.SortStack;
import net.yacy.kelondro.util.SortStore;
@ -95,8 +95,7 @@ public class ResultFetcher {
// start worker threads to fetch urls and snippets
this.workerThreads = null;
deployWorker(Math.min(10, query.itemsPerPage), query.neededResults());
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), this.workerThreads.length + " online snippet fetch threads started", 0, 0), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), this.workerThreads.length + " online snippet fetch threads started", 0, 0), false, 30000, ProfilingGraph.maxTime);
}
public void deployWorker(int deployCount, int neededResults) {
@ -281,7 +280,7 @@ public class ResultFetcher {
public ResultEntry oneResult(final int item) {
// check if we already retrieved this item
// (happens if a search pages is accessed a second time)
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "obtain one result entry - start", 0, 0), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "obtain one result entry - start", 0, 0), false, 30000, ProfilingGraph.maxTime);
if (this.result.sizeStore() > item) {
// we have the wanted result already in the result array .. return that
return this.result.element(item).element;

@ -38,8 +38,8 @@ import net.yacy.kelondro.data.word.WordReferenceVars;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.SetTools;
import de.anomic.crawler.ResultURLs;
@ -139,7 +139,7 @@ public final class SearchEvent {
(query.domType == QueryParams.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes);
if (this.primarySearchThreads != null) {
if (this.primarySearchThreads.length > fetchpeers) this.rankedCache.moreFeeders(this.primarySearchThreads.length - fetchpeers);
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
// finished searching
Log.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
} else {
@ -179,7 +179,7 @@ public final class SearchEvent {
IACount.put(wordhash, Integer.valueOf(container.size()));
IAResults.put(wordhash, ReferenceContainer.compressIndex(container, null, 1000).toString());
}
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMap().size(), System.currentTimeMillis() - timer), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMap().size(), System.currentTimeMillis() - timer), false, 30000, ProfilingGraph.maxTime);
}
// start worker threads to fetch urls and snippets
@ -188,7 +188,7 @@ public final class SearchEvent {
// clean up events
SearchEventCache.cleanupEvents(false);
MemoryTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "event-cleanup", 0, 0), false);
EventTracker.update("SEARCH", new ProfilingGraph.searchEvent(query.id(true), "event-cleanup", 0, 0), false, 30000, ProfilingGraph.maxTime);
// store this search to a cache so it can be re-used
if (MemoryControl.available() < 1024 * 1024 * 10) SearchEventCache.cleanupEvents(true);

@ -128,6 +128,7 @@ import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.DateFormatter;
import net.yacy.kelondro.util.Domains;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.MemoryTracker;
@ -193,6 +194,7 @@ import de.anomic.yacy.yacyUpdateLocation;
import de.anomic.yacy.yacyRelease;
import de.anomic.yacy.dht.Dispatcher;
import de.anomic.yacy.dht.PeerSelection;
import de.anomic.yacy.graphics.ProfilingGraph;
import de.anomic.yacy.graphics.WebStructureGraph;
public final class Switchboard extends serverSwitch {
@ -1486,7 +1488,7 @@ public final class Switchboard extends serverSwitch {
log.logInfo("AUTO-UPDATE: omiting update because download failed (file cannot be found, is too small or signature is bad)");
} else {
yacyRelease.deployRelease(downloaded);
terminate(5000);
terminate(5000, "auto-update to install " + downloaded.getName());
log.logInfo("AUTO-UPDATE: deploy and restart initiated");
}
}
@ -1765,10 +1767,10 @@ public final class Switchboard extends serverSwitch {
if (System.currentTimeMillis() - lastPPMUpdate > 20000) {
// we don't want to do this too often
updateMySeed();
MemoryTracker.update("ppm", Long.valueOf(currentPPM()), true);
EventTracker.update("ppm", Long.valueOf(currentPPM()), true, 30000, ProfilingGraph.maxTime);
lastPPMUpdate = System.currentTimeMillis();
}
MemoryTracker.update("indexed", queueEntry.url().toNormalform(true, false), false);
EventTracker.update("indexed", queueEntry.url().toNormalform(true, false), false, 30000, ProfilingGraph.maxTime);
// if this was performed for a remote crawl request, notify requester
if ((processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null)) {
@ -2005,7 +2007,7 @@ public final class Switchboard extends serverSwitch {
}
public int currentPPM() {
return MemoryTracker.countEvents("indexed", 20000) * 3;
return EventTracker.countEvents("indexed", 20000) * 3;
}
public String makeDefaultPeerName() {
@ -2127,13 +2129,15 @@ public final class Switchboard extends serverSwitch {
else if (this.terminate || curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ...");
}
public void terminate(final long delay) {
public void terminate(final long delay, String reason) {
if (delay <= 0) throw new IllegalArgumentException("The shutdown delay must be greater than 0.");
(new delayedShutdown(this,delay)).start();
log.logInfo("caught delayed terminate request: " + reason);
(new delayedShutdown(this, delay, reason)).start();
}
public void terminate() {
public void terminate(String reason) {
this.terminate = true;
log.logInfo("caught terminate request: " + reason);
this.shutdownSync.release();
}
@ -2171,9 +2175,11 @@ public final class Switchboard extends serverSwitch {
class delayedShutdown extends Thread {
private final Switchboard sb;
private final long delay;
public delayedShutdown(final Switchboard sb, final long delay) {
private final String reason;
public delayedShutdown(final Switchboard sb, final long delay, final String reason) {
this.sb = sb;
this.delay = delay;
this.reason = reason;
}
public void run() {
@ -2184,6 +2190,6 @@ class delayedShutdown extends Thread {
} catch (final Exception e) {
Log.logException(e);
}
this.sb.terminate();
this.sb.terminate(reason);
}
}

@ -93,13 +93,13 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
public static final String LF_STRING = new String(new byte[]{LF});
public static final Class<?>[] stringType = {"".getClass()}; // set up some reflection
public static final long startupTime = System.currentTimeMillis();
private static final ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup");
private static final HashMap<String, Object> commandObjMethodCache = new HashMap<String, Object>(5);
static final ThreadGroup sessionThreadGroup = new ThreadGroup("sessionThreadGroup");
static final HashMap<String, Object> commandObjMethodCache = new HashMap<String, Object>(5);
/**
* will be increased with each session and is used to return a hash code
*/
private static int sessionCounter = 0;
static int sessionCounter = 0;
// static variables
private static final long keepAliveTimeout = 60000; // time that a connection is kept alive if requested with a keepAlive statement
@ -123,18 +123,18 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
/**
* specifies if the server should try to do a restart
*/
private boolean forceRestart = false;
boolean forceRestart = false;
public static boolean useStaticIP = false;
protected Log log;
private SSLSocketFactory sslSocketFactory = null;
private ServerSocket socket; // listener
private final int timeout; // connection time-out of the socket
private serverHandler handlerPrototype; // the command class (a serverHandler)
serverHandler handlerPrototype; // the command class (a serverHandler)
private final serverSwitch switchboard; // the command class switchboard
private HashMap<String, String> denyHost;
private int commandMaxLength;
HashMap<String, String> denyHost;
int commandMaxLength;
private int maxBusySessions;
private long lastAutoTermination;

@ -380,7 +380,6 @@ public class serverSwitch {
newThread.setMemPreReqisite(initialMemoryPreRequisite);
setConfig(threadName + "_memprereq", initialMemoryPreRequisite);
}
newThread.setLog(log);
newThread.setDescription(threadShortDescription, threadLongDescription, threadMonitorURL);
workerThreads.put(threadName, newThread);
// start the thread

@ -136,7 +136,7 @@ public final class Tray {
menuItem = new MenuItem(label);
menuItem.addActionListener(new ActionListener() {
public void actionPerformed(final ActionEvent e) {
sb.terminate();
sb.terminate("shutdown from tray");
}
});
menu.add(menuItem);

@ -26,26 +26,27 @@
package de.anomic.yacy.graphics;
import java.util.ArrayList;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import net.yacy.kelondro.util.MemoryTracker;
import net.yacy.kelondro.util.MemoryTracker.Event;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.EventTracker.Event;
import net.yacy.visualization.RasterPlotter;
import net.yacy.visualization.ChartPlotter;
public class ProfilingGraph {
private static ChartPlotter bufferChart = null;
public static long maxTime = 600000L;
public static long maxPayload(final String eventname, final long min) {
final ArrayList<Event> list = MemoryTracker.history(eventname);
final Iterator<Event> list = EventTracker.getHistory(eventname);
if (list == null) return min;
long max = min, l;
synchronized (list) {
for (MemoryTracker.Event event: list) {
EventTracker.Event event;
while (list.hasNext()) {
event = list.next();
l = ((Long) event.payload).longValue();
if (l > max) max = l;
}
@ -102,49 +103,61 @@ public class ProfilingGraph {
}
*/
// draw memory
ArrayList<Event> events = MemoryTracker.history("memory");
Iterator<Event> events = EventTracker.getHistory("memory");
x0 = 1; y0 = 0;
if (events != null) synchronized (events) {for (MemoryTracker.Event event: events) {
time = event.time - now;
bytes = ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = (int) (bytes / 1024 / 1024);
chart.setColor("AAAAFF");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x1, y1, 2, null, 0);
chart.setColor("0000FF");
if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x0, y0, x1, y1);
x0 = x1; y0 = y1;
}}
if (events != null) {
EventTracker.Event event;
while (events.hasNext()) {
event = events.next();
time = event.time - now;
bytes = ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = (int) (bytes / 1024 / 1024);
chart.setColor("AAAAFF");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x1, y1, 2, null, 0);
chart.setColor("0000FF");
if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_RIGHT, x0, y0, x1, y1);
x0 = x1; y0 = y1;
}
}
// draw wordcache
events = MemoryTracker.history("wordcache");
events = EventTracker.getHistory("wordcache");
x0 = 1; y0 = 0;
if (events != null) synchronized (events) {for (MemoryTracker.Event event: events) {
time = event.time - now;
words = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = words;
chart.setColor("228822");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x1, y1, 2, null, 315);
chart.setColor("008800");
if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x0, y0, x1, y1);
x0 = x1; y0 = y1;
}}
if (events != null) {
EventTracker.Event event;
while (events.hasNext()) {
event = events.next();
time = event.time - now;
words = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = words;
chart.setColor("228822");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x1, y1, 2, null, 315);
chart.setColor("008800");
if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_LEFT, x0, y0, x1, y1);
x0 = x1; y0 = y1;
}
}
// draw ppm
events = MemoryTracker.history("ppm");
events = EventTracker.getHistory("ppm");
x0 = 1; y0 = 0;
if (events != null) synchronized (events) {for (MemoryTracker.Event event: events) {
time = event.time - now;
ppm = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = ppm;
chart.setColor("AA8888");
if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x0, y0, x1, y1);
chart.setColor("AA2222");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0);
x0 = x1; y0 = y1;
}}
if (events != null) {
EventTracker.Event event;
while (events.hasNext()) {
event = events.next();
time = event.time - now;
ppm = (int) ((Long) event.payload).longValue();
x1 = (int) (time/1000);
y1 = ppm;
chart.setColor("AA8888");
if (x0 < 0) chart.chartLine(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x0, y0, x1, y1);
chart.setColor("AA2222");
chart.chartDot(ChartPlotter.DIMENSION_BOTTOM, ChartPlotter.DIMENSION_ANOT0, x1, y1, 2, ppm + " PPM", 0);
x0 = x1; y0 = y1;
}
}
bufferChart = chart;
} catch (final ConcurrentModificationException cme) {

@ -58,7 +58,8 @@ public class WebStructureGraph {
private final Log log;
private final File rankingPath, structureFile;
private final String crlFile, crgFile;
private TreeMap<String, String> structure_old, structure_new; // <b64hash(6)>','<host> to <date-yyyymmdd(8)>{<target-b64hash(6)><target-count-hex(4)>}*
TreeMap<String, String> structure_old; // <b64hash(6)>','<host> to <date-yyyymmdd(8)>{<target-b64hash(6)><target-count-hex(4)>}*
TreeMap<String, String> structure_new;
public WebStructureGraph(final Log log, final File rankingPath, final String crlFile, final String crgFile, final File structureFile) {
this.log = log;

@ -416,7 +416,7 @@ public final class yacyRelease extends yacyVersion {
Log.logInfo("RESTART", "wrote restart-script to " + scriptFile.getAbsolutePath());
OS.execAsynchronous(scriptFile);
Log.logInfo("RESTART", "script is running");
sb.terminate(5000);
sb.terminate(5000, "windows restart");
} catch (final IOException e) {
Log.logSevere("RESTART", "restart failed", e);
}
@ -466,7 +466,7 @@ public final class yacyRelease extends yacyVersion {
Log.logInfo("RESTART", "wrote restart-script to " + scriptFile.getAbsolutePath());
OS.execAsynchronous(scriptFile);
Log.logInfo("RESTART", "script is running");
sb.terminate(5000);
sb.terminate(5000, "unix restart");
} catch (final IOException e) {
Log.logSevere("RESTART", "restart failed", e);
}
@ -567,7 +567,7 @@ public final class yacyRelease extends yacyVersion {
OS.execAsynchronous(scriptFile);
Log.logInfo("UPDATE", "script is running");
sb.setConfig("update.time.deploy", System.currentTimeMillis());
sb.terminate(5000);
sb.terminate(5000, "auto-deploy for " + releaseFile.getName());
} catch (final IOException e) {
Log.logSevere("UPDATE", "update failed", e);
}

@ -160,7 +160,6 @@ public class xlsParser extends AbstractParser implements Idiom {
*/
Log.logException(e);
final String errorMsg = "Unable to parse the xls document '" + location + "':" + e.getMessage();
theLogger.logSevere(errorMsg);
throw new ParserException(errorMsg, location);
}
}

@ -31,6 +31,8 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import de.anomic.yacy.graphics.ProfilingGraph;
import net.yacy.kelondro.index.ARC;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.SimpleARC;
@ -39,8 +41,8 @@ import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.order.Order;
import net.yacy.kelondro.util.ByteArray;
import net.yacy.kelondro.util.EventTracker;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.MemoryTracker;
/*
@ -108,7 +110,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public void add(ReferenceContainer<ReferenceType> newEntries) throws IOException {
this.ram.add(newEntries);
if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) {
MemoryTracker.update("wordcache", Long.valueOf(this.ram.size()), true);
EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime);
cleanCache();
}
}
@ -116,7 +118,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
public void add(byte[] termHash, ReferenceType entry) throws IOException {
this.ram.add(termHash, entry);
if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) {
MemoryTracker.update("wordcache", Long.valueOf(this.ram.size()), true);
EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime);
cleanCache();
}
}

@ -0,0 +1,123 @@
// EventTracker.java
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 17.11.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-11-05 21:28:37 +0100 (Do, 05 Nov 2009) $
// $LastChangedRevision: 6458 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.util;
import java.util.Iterator;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
public class EventTracker {
private final static Map<String, Queue<Event>> historyMaps = new ConcurrentHashMap<String, Queue<Event>>();
private final static Map<String, Long> eventAccess = new ConcurrentHashMap<String, Long>(); // value: last time when this was accessed
public final static void update(
final String eventName,
final Object eventPayload,
boolean useProtection,
int maxQueueSize,
long maxQueueAge) {
// check protection against too heavy access
if (useProtection) {
Long lastAcc = eventAccess.get(eventName);
if (lastAcc == null) {
eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis()));
} else {
long time = System.currentTimeMillis();
if (time - lastAcc.longValue() < 1000) {
return; // protect against too heavy load
}
eventAccess.put(eventName, Long.valueOf(time));
}
}
// get event history container
Queue<Event> history = historyMaps.get(eventName);
// create history
if (history == null) {
history = new ConcurrentLinkedQueue<Event>();
// update entry
history.offer(new Event(eventPayload));
// store map
historyMaps.put(eventName, history);
return;
}
// update history
synchronized (history) {
// update entry
history.offer(new Event(eventPayload));
// clean up too old entries
int tp = history.size() - maxQueueSize;
while (tp-- > 0) history.poll();
if (history.size() % 10 == 0) { // reduce number of System.currentTimeMillis() calls
Event e;
final long now = System.currentTimeMillis();
while (history.size() > 0) {
e = history.peek();
if (now - e.time < maxQueueAge) break;
history.poll();
}
}
}
}
public final static Iterator<Event> getHistory(final String eventName) {
Queue<Event> list = historyMaps.get(eventName);
if (list == null) return null;
return list.iterator();
}
public final static int countEvents(final String eventName, long time) {
Iterator<Event> event = getHistory(eventName);
if (event == null) return 0;
long now = System.currentTimeMillis();
int count = 0;
while (event.hasNext()) {
if (now - event.next().time < time) count++;
}
return count;
}
public final static class Event {
public Object payload;
public long time;
public Event(final Object payload) {
this.payload = payload;
this.time = System.currentTimeMillis();
}
}
}

@ -30,7 +30,8 @@ import net.yacy.kelondro.util.MemoryControl;
public abstract class AbstractBlockingThread<J extends WorkflowJob> extends AbstractThread implements BlockingThread<J> {
private WorkflowProcessor<J> manager = null;
private final static Log log = new Log("BlockingThread");
public void setManager(final WorkflowProcessor<J> manager) {
this.manager = manager;
}

@ -33,6 +33,7 @@ import net.yacy.kelondro.util.MemoryControl;
public abstract class AbstractBusyThread extends AbstractThread implements BusyThread {
private final static Log log = new Log("BusyThread");
private long startup = 0, intermission = 0, idlePause = 0, busyPause = 0;
private long idletime = 0, memprereq = 0;
private long idleCycles = 0, busyCycles = 0, outofmemoryCycles = 0;
@ -167,7 +168,7 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
idletime += System.currentTimeMillis() - timestamp;
} catch (final SocketException e) {
// in case that a socket is interrupted, this method must die silently (shutdown)
this.log.logFine("socket-job interrupted: " + e.getMessage());
log.logFine("socket-job interrupted: " + e.getMessage());
} catch (final Exception e) {
// handle exceptions: thread must not die on any unexpected exceptions
// if the exception is too bad it should call terminate()
@ -201,16 +202,16 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
Thread.sleep(millis);
//}
} catch (final InterruptedException e) {
if (this.log != null)
this.log.logConfig("thread '" + this.getName() + "' interrupted because of shutdown.");
if (log != null)
log.logConfig("thread '" + this.getName() + "' interrupted because of shutdown.");
}
}
public void notifyThread() {
if (this.syncObject != null) {
synchronized (this.syncObject) {
if (this.log != null)
if (this.log.isFine()) this.log.logFine("thread '" + this.getName()
if (log != null)
if (log.isFine()) log.logFine("thread '" + this.getName()
+ "' has received a notification from thread '"
+ Thread.currentThread().getName() + "'.");
this.syncObject.notifyAll();

@ -39,8 +39,8 @@ import net.yacy.kelondro.logging.Log;
public abstract class AbstractThread extends Thread implements WorkflowThread {
private static Log log = new Log("WorkflowThread");
protected boolean running = true;
protected Log log = null;
protected long busytime = 0, memuse = 0;
private long blockPause = 0;
private String shortDescr = "", longDescr = "";
@ -100,12 +100,6 @@ public abstract class AbstractThread extends Thread implements WorkflowThread {
// returns the sum of all memory usage differences before and after one busy job
return memuse;
}
public final void setLog(final Log log) {
// defines a log where process states can be written to
this.log = log;
}
public boolean shutdownInProgress() {
return !this.running || Thread.currentThread().isInterrupted();

@ -151,7 +151,6 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT
thread.setIdleSleep(-1);
thread.setBusySleep(-1);
thread.setMemPreReqisite(0);
thread.setLog(log);
thread.start();
return thread;
}

@ -24,8 +24,6 @@
package net.yacy.kelondro.workflow;
import net.yacy.kelondro.logging.Log;
public interface WorkflowThread {
// -------------------------------------------------------
@ -58,9 +56,6 @@ public interface WorkflowThread {
public long getMemoryUse();
// returns the sum of all memory usage differences before and after one busy job
public void setLog(Log log);
// defines a log where process states can be written to
public void jobExceptionHandler(Exception e);
// handles any action necessary during job execution

@ -316,8 +316,6 @@ public final class yacy {
// first start the server
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
//server.start();
// repair log settings, gets overwritten with "PLASMA" in deployThread
server.setLog(new Log("SERVER"));
// open the browser window
final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
@ -515,10 +513,10 @@ public final class yacy {
return config;
}
public static void shutdown() {
public static void shutdown(String reason) {
if (sb != null) {
// YaCy is running in the same runtime. we can shutdown via interrupt
sb.terminate();
sb.terminate(reason);
} else {
final File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
shutdown(applicationRoot);
@ -1056,7 +1054,7 @@ class shutdownHookThread extends Thread {
// sending the yacy main thread a shutdown signal
Log.logFine("SHUTDOWN","Signaling shutdown to the switchboard.");
this.sb.terminate();
this.sb.terminate("shutdown hook");
// waiting for the yacy thread to finish execution
Log.logFine("SHUTDOWN","Waiting for main thread to finish.");

Loading…
Cancel
Save