From e2e7949feb8c3b27d67d4620d7f4bf3d16915e4b Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 13 Mar 2009 00:13:47 +0000 Subject: [PATCH] replaced old PPM computation with a better one that simply sums up events that had been stored in the profiling table. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5706 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Network.java | 2 +- htroot/yacy/search.java | 6 ++-- htroot/yacysearch.java | 2 +- htroot/yacysearchitem.java | 2 +- htroot/yacysearchtrailer.java | 2 +- .../kelondro/index/IntegerHandleIndex.java | 9 +++--- .../kelondro/index/ObjectIndexDaemon.java | 26 ++++++++--------- .../de/anomic/plasma/plasmaSearchEvent.java | 12 ++++---- .../plasma/plasmaSearchRankingProcess.java | 8 +++--- .../de/anomic/plasma/plasmaSwitchboard.java | 28 ++++--------------- source/de/anomic/plasma/plasmaWordIndex.java | 4 +-- source/de/anomic/server/serverProfiling.java | 18 +++++++++--- 12 files changed, 55 insertions(+), 64 deletions(-) diff --git a/htroot/Network.java b/htroot/Network.java index c2baf1992..c910de9d4 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -138,7 +138,7 @@ public class Network { prop.putNum("table_my-rU", Long.parseLong(seed.get(yacySeed.URL_IN, "0"))); prop.putNum("table_my-ppm", myppm); prop.putNum("table_my-qph", Math.round(100d * myqph) / 100d); - prop.putNum("table_my-totalppm", sb.totalPPM); + prop.putNum("table_my-totalppm", sb.currentPPM()); prop.putNum("table_my-totalqph", Math.round(6000d * sb.totalQPM) / 100d); prop.putNum("table_my-seeds", Long.parseLong(seed.get(yacySeed.SCOUNT, "0"))); prop.putNum("table_my-connects", Double.parseDouble(seed.get(yacySeed.CCOUNT, "0"))); diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index a6ed02d72..534f1b117 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -187,7 +187,7 @@ public final class search { final long timer = System.currentTimeMillis(); final Map[] containers = sb.webIndex.localSearchContainers(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2Set(urls)); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer), false); if (containers != null) { final Iterator> ci = containers[0].entrySet().iterator(); Map.Entry entry; @@ -279,7 +279,7 @@ public final class search { refstr.append(",").append(j.next()); } prop.put("references", (refstr.length() > 0) ? refstr.substring(1) : refstr.toString()); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer), false); } prop.put("indexabstract", indexabstract.toString()); @@ -306,7 +306,7 @@ public final class search { } prop.put("links", links.toString()); prop.put("linkcount", accu.size()); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer), false); } // add information about forward peers diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 260e6bd77..eb480d297 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -380,7 +380,7 @@ public class yacysearch { yacyURL.TLD_any_zone_filter, client, authenticated); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.INITIALIZATION, 0, 0)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.INITIALIZATION, 0, 0), false); // tell all threads to do nothing for a specific time sb.intermissionAllThreads(10000); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 6d850f6fb..a245d68c3 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -143,7 +143,7 @@ public class yacysearchitem { final plasmaSnippetCache.TextSnippet snippet = result.textSnippet(); prop.put("content_description", (snippet == null) ? "" : snippet.getLineMarked(theQuery.fullqueryHashes)); prop.putXML("content_description-xml", (snippet == null) ? "" : snippet.getLineMarked(theQuery.fullqueryHashes)); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + item, 0, 0)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + item, 0, 0), false); return prop; } diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 04bf4840d..8d85d7784 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -109,7 +109,7 @@ public class yacysearchtrailer { } prop.put("words", hintcount); } - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + "bottomline", 0, 0)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false); return prop; } diff --git a/source/de/anomic/kelondro/index/IntegerHandleIndex.java b/source/de/anomic/kelondro/index/IntegerHandleIndex.java index ba18264a0..f007f087c 100644 --- a/source/de/anomic/kelondro/index/IntegerHandleIndex.java +++ b/source/de/anomic/kelondro/index/IntegerHandleIndex.java @@ -325,11 +325,10 @@ public class IntegerHandleIndex { public static void main(String[] args) { int count = (args.length == 0) ? 1000000 : Integer.parseInt(args[0]); - System.out.println("Starting test with " + count + " objects, minimum memory: " + (count * 16) + " bytes; " + MemoryControl.available( -) + " available"); - + System.out.println("Starting test with " + count + " objects, minimum memory: " + (count * 16) + " bytes; " + MemoryControl.available() + " available"); Random r = new Random(0); long start = System.currentTimeMillis(); + System.gc(); // for resource measurement long a = MemoryControl.available(); IntegerHandleIndex idx = new IntegerHandleIndex(12, Base64Order.enhancedCoder, 0); @@ -357,8 +356,7 @@ public class IntegerHandleIndex { if (d == null) hm.put(hash, 1); else hm.put(hash, d + 1); } long timej = ((long) count) * 1000L / (System.currentTimeMillis() - start); - System.out.println("Result HashMap: " +timej + " inc per second; " + count -+ " loops."); + System.out.println("Result HashMap: " +timej + " inc per second; " + count + " loops."); System.gc(); long memj = a - MemoryControl.available(); System.out.println("Used Memory: " + memj + " bytes"); @@ -366,6 +364,7 @@ public class IntegerHandleIndex { System.out.println("Geschwindigkeitsfaktor j/k: " + (timej / timek)); System.out.println("Speicherplatzfaktor j/k: " + (memj / memk)); System.exit(0); + } } diff --git a/source/de/anomic/kelondro/index/ObjectIndexDaemon.java b/source/de/anomic/kelondro/index/ObjectIndexDaemon.java index 2e5311234..fd74efa39 100644 --- a/source/de/anomic/kelondro/index/ObjectIndexDaemon.java +++ b/source/de/anomic/kelondro/index/ObjectIndexDaemon.java @@ -32,7 +32,7 @@ import de.anomic.kelondro.index.Row.Entry; import de.anomic.kelondro.index.Row.Queue; import de.anomic.kelondro.order.CloneableIterator; -public class ObjectIndexDaemon implements ObjectIndex { +public class ObjectIndexDaemon { private Row.Entry poison; private PutScheduler putScheduler; @@ -71,11 +71,11 @@ public class ObjectIndexDaemon implements ObjectIndex { } } - public void addUnique(Entry row) throws IOException { + public void addUnique(Entry row) { index.addUnique(row); } - public void addUnique(List rows) throws IOException { + public void addUnique(List rows) { index.addUnique(rows); } @@ -100,7 +100,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return this.index.filename(); } - public Entry get(byte[] key) throws IOException { + public Entry get(byte[] key) { Entry entry = this.queue.get(key); if (entry != null) return entry; return this.index.get(key); @@ -112,7 +112,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return this.index.has(key); } - public Entry replace(Entry row) throws IOException { + public Entry replace(Entry row) { Entry entry = get(row.getPrimaryKeyBytes()); try { this.queue.put(row); @@ -122,7 +122,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return entry; } - public void put(Entry row) throws IOException { + public void put(Entry row) { try { this.queue.put(row); } catch (InterruptedException e) { @@ -130,7 +130,7 @@ public class ObjectIndexDaemon implements ObjectIndex { } } - public void put(List rows) throws IOException { + public void put(List rows) { for (Entry entry: rows) try { this.queue.put(entry); } catch (InterruptedException e) { @@ -138,14 +138,14 @@ public class ObjectIndexDaemon implements ObjectIndex { } } - public Entry remove(byte[] key) throws IOException { + public Entry remove(byte[] key) { Entry entry = this.queue.delete(key); if (entry == null) return this.index.remove(key); this.index.remove(key); return entry; } - public synchronized ArrayList removeDoubles() throws IOException { + public synchronized ArrayList removeDoubles() { try { this.queue.put(poison); this.putScheduler.join(); @@ -159,7 +159,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return d; } - public synchronized Entry removeOne() throws IOException { + public synchronized Entry removeOne() { try { this.queue.put(poison); this.putScheduler.join(); @@ -177,7 +177,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return this.index.row(); } - public synchronized CloneableIterator keys(boolean up, byte[] firstKey) throws IOException { + public synchronized CloneableIterator keys(boolean up, byte[] firstKey) { try { this.queue.put(poison); this.putScheduler.join(); @@ -191,7 +191,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return keys; } - public synchronized CloneableIterator rows(boolean up, byte[] firstKey) throws IOException { + public synchronized CloneableIterator rows(boolean up, byte[] firstKey) { try { this.queue.put(poison); this.putScheduler.join(); @@ -205,7 +205,7 @@ public class ObjectIndexDaemon implements ObjectIndex { return rows; } - public CloneableIterator rows() throws IOException { + public CloneableIterator rows() { return rows(true, null); } diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index 73af76a57..83b5e9f83 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -155,7 +155,7 @@ public final class plasmaSearchEvent { query.ranking, query.constraint, (query.domType == plasmaSearchQuery.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false); // meanwhile do a local search localSearchThread = new localSearchProcess(); @@ -192,7 +192,7 @@ public final class plasmaSearchEvent { IACount.put(wordhash, Integer.valueOf(container.size())); IAResults.put(wordhash, ReferenceContainer.compressIndex(container, null, 1000).toString()); } - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMaps()[0].size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMaps()[0].size(), System.currentTimeMillis() - timer), false); } } @@ -203,18 +203,18 @@ public final class plasmaSearchEvent { this.workerThreads[i] = new resultWorker(i, 10000, 2); this.workerThreads[i].start(); } - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "online snippet fetch threads started", 0, 0)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "online snippet fetch threads started", 0, 0), false); } else { final long timer = System.currentTimeMillis(); // use only a single worker thread, thats enough resultWorker worker = new resultWorker(0, 3000, 0); worker.run(); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "offline snippet fetch", result.size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "offline snippet fetch", result.size(), System.currentTimeMillis() - timer), false); } // clean up events cleanupEvents(false); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "event-cleanup", 0, 0)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "event-cleanup", 0, 0), false); // store this search to a cache so it can be re-used if (MemoryControl.available() < 1024 * 1024 * 10) cleanupEvents(true); @@ -599,7 +599,7 @@ public final class plasmaSearchEvent { public ResultEntry oneResult(final int item) { // check if we already retrieved this item (happens if a search // pages is accessed a second time) - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "obtain one result entry - start", 0, 0)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "obtain one result entry - start", 0, 0), false); if (this.result.sizeStore() > item) { // we have the wanted result already in the result array .. return that return this.result.element(item).element; diff --git a/source/de/anomic/plasma/plasmaSearchRankingProcess.java b/source/de/anomic/plasma/plasmaSearchRankingProcess.java index 013a4df85..0bbb9a85d 100644 --- a/source/de/anomic/plasma/plasmaSearchRankingProcess.java +++ b/source/de/anomic/plasma/plasmaSearchRankingProcess.java @@ -111,7 +111,7 @@ public final class plasmaSearchRankingProcess { long timer = System.currentTimeMillis(); this.localSearchContainerMaps = wordIndex.localSearchContainers(query.queryHashes, query.excludeHashes, null); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.COLLECTION, this.localSearchContainerMaps[0].size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.COLLECTION, this.localSearchContainerMaps[0].size(), System.currentTimeMillis() - timer), false); // join and exclude the local result timer = System.currentTimeMillis(); @@ -120,7 +120,7 @@ public final class plasmaSearchRankingProcess { this.localSearchContainerMaps[0].values(), this.localSearchContainerMaps[1].values(), query.maxDistance); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer), false); if (index.size() == 0) { return; } @@ -145,7 +145,7 @@ public final class plasmaSearchRankingProcess { // normalize entries final ArrayList decodedEntries = this.order.normalizeWith(index); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer), false); // iterate over normalized entries and select some that are better than currently stored timer = System.currentTimeMillis(); @@ -211,7 +211,7 @@ public final class plasmaSearchRankingProcess { } //if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true); - serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer)); + serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer), false); } private boolean testFlags(final Reference ientry) { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 5d2041d95..42fa091d2 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -223,12 +223,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch localSearches; // array of search result properties as HashMaps public ArrayList remoteSearches; // array of search result properties as HashMaps public HashMap> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time) - public long lastseedcheckuptime = -1; public long indexedPages = 0; - public long lastindexedPages = 0; public double requestedQueries = 0d; - public double lastrequestedQueries = 0d; - public int totalPPM = 0; public double totalQPM = 0d; public TreeMap clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used public URLLicense licensedURLs; @@ -1677,10 +1673,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 30000) { // we don't want to do this too often updateMySeed(); - serverProfiling.update("ppm", Long.valueOf(currentPPM())); + serverProfiling.update("ppm", Long.valueOf(currentPPM()), true); lastPPMUpdate = System.currentTimeMillis(); } - serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false)); + serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false), false); // if this was performed for a remote crawl request, notify requester final yacySeed initiatorPeer = queueEntry.initiatorPeer(); @@ -1975,11 +1971,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch 300 || uptimediff <= 0 || lastseedcheckuptime == -1 ) { - lastseedcheckuptime = uptime; - lastindexedPages = indexedPages; - lastrequestedQueries = requestedQueries; - } - //the speed of indexing (pages/minute) of the peer - totalPPM = (int) (indexedPages * 60 / Math.max(uptime, 1)); - webIndex.seedDB.mySeed().put(yacySeed.ISPEED, Long.toString(Math.round(Math.max(indexedcdiff, 0f) * 60f / Math.max(uptimediff, 1f)))); + final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000; + webIndex.seedDB.mySeed().put(yacySeed.ISPEED, Integer.toString(currentPPM())); totalQPM = requestedQueries * 60d / Math.max(uptime, 1d); webIndex.seedDB.mySeed().put(yacySeed.RSPEED, Double.toString(totalQPM /*Math.max((float) requestcdiff, 0f) * 60f / Math.max((float) uptimediff, 1f)*/ )); diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 8f35f214f..eba23f6ab 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -467,7 +467,7 @@ public final class plasmaWordIndex implements Index { // flush elements that are too big. This flushing depends on the fact that the flush rule // selects the biggest elements first for flushing. If it does not for any reason, the following // loop would not terminate. - serverProfiling.update("wordcache", Long.valueOf(cs)); + serverProfiling.update("wordcache", Long.valueOf(cs), true); // To ensure termination an additional counter is used int l = 0; while (theCache.size() > 0 && (l++ < 100) && (theCache.maxURLinCache() > wCacheMaxChunk)) { @@ -479,7 +479,7 @@ public final class plasmaWordIndex implements Index { (MemoryControl.available() < collections.minMem()))) { flushCacheOne(theCache); } - if (cacheSize() != cs) serverProfiling.update("wordcache", Long.valueOf(cacheSize())); + if (cacheSize() != cs) serverProfiling.update("wordcache", Long.valueOf(cacheSize()), true); } } diff --git a/source/de/anomic/server/serverProfiling.java b/source/de/anomic/server/serverProfiling.java index f6b951061..bd63361a4 100644 --- a/source/de/anomic/server/serverProfiling.java +++ b/source/de/anomic/server/serverProfiling.java @@ -40,7 +40,7 @@ public class serverProfiling extends Thread { private static serverProfiling systemProfiler = null; public static void startSystemProfiling() { - systemProfiler = new serverProfiling(1000); + systemProfiler = new serverProfiling(1500); systemProfiler.start(); } @@ -58,7 +58,7 @@ public class serverProfiling extends Thread { public void run() { while (running) { - update("memory", Long.valueOf(MemoryControl.used())); + update("memory", Long.valueOf(MemoryControl.used()), true); try { Thread.sleep(this.delaytime); } catch (final InterruptedException e) { @@ -67,13 +67,13 @@ public class serverProfiling extends Thread { } } - public static void update(final String eventName, final Object eventPayload) { + public static void update(final String eventName, final Object eventPayload, boolean useProtection) { // get event history container Long lastAcc = eventAccess.get(eventName); if (lastAcc == null) { eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis())); } else { - if (System.currentTimeMillis() - lastAcc.longValue() > 1000) { + if (!useProtection || System.currentTimeMillis() - lastAcc.longValue() > 1000) { eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis())); } else { return; // protect against too heavy load @@ -108,6 +108,16 @@ public class serverProfiling extends Thread { return (historyMaps.containsKey(eventName) ? (historyMaps.get(eventName)) : new ConcurrentLinkedQueue()).iterator(); } + public static int countEvents(final String eventName, long time) { + Iterator i = history(eventName); + long now = System.currentTimeMillis(); + int count = 0; + while (i.hasNext()) { + if (now - i.next().time < time) count++; + } + return count; + } + public static class Event { public Object payload; public long time;