boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

replaced old PPM computation with a better one that simply sums up events that had been stored in the profiling table.

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5706 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent f6d989aa04
commit e2e7949feb
12 changed files with 55 additions and 64 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Network.java
Unescape View File

@ -138,7 +138,7 @@ public class Network {
prop.putNum("table_my-rU", Long.parseLong(seed.get(yacySeed.URL_IN, "0")));
prop.putNum("table_my-ppm", myppm);
prop.putNum("table_my-qph", Math.round(100d * myqph) / 100d);
prop.putNum("table_my-totalppm", sb.totalPPM);
prop.putNum("table_my-totalppm", sb.currentPPM());
prop.putNum("table_my-totalqph", Math.round(6000d * sb.totalQPM) / 100d);
prop.putNum("table_my-seeds", Long.parseLong(seed.get(yacySeed.SCOUNT, "0")));
prop.putNum("table_my-connects", Double.parseDouble(seed.get(yacySeed.CCOUNT, "0")));

6
htroot/yacy/search.java
Unescape View File

@ -187,7 +187,7 @@ public final class search {
final long timer = System.currentTimeMillis();
final Map<String, ReferenceContainer>[] containers = sb.webIndex.localSearchContainers(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2Set(urls));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer), false);
if (containers != null) {
final Iterator<Map.Entry<String, ReferenceContainer>> ci = containers[0].entrySet().iterator();
Map.Entry<String, ReferenceContainer> entry;
@ -279,7 +279,7 @@ public final class search {
refstr.append(",").append(j.next());
}
prop.put("references", (refstr.length() > 0) ? refstr.substring(1) : refstr.toString());
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "reference collection", ws.size(), System.currentTimeMillis() - timer), false);
}
prop.put("indexabstract", indexabstract.toString());
@ -306,7 +306,7 @@ public final class search {
}
prop.put("links", links.toString());
prop.put("linkcount", accu.size());
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), "result list preparation", accu.size(), System.currentTimeMillis() - timer), false);
}
// add information about forward peers

2
htroot/yacysearch.java
Unescape View File

@ -380,7 +380,7 @@ public class yacysearch {
yacyURL.TLD_any_zone_filter,
client,
authenticated);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.INITIALIZATION, 0, 0));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.INITIALIZATION, 0, 0), false);
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(10000);

2
htroot/yacysearchitem.java
Unescape View File

@ -143,7 +143,7 @@ public class yacysearchitem {
final plasmaSnippetCache.TextSnippet snippet = result.textSnippet();
prop.put("content_description", (snippet == null) ? "" : snippet.getLineMarked(theQuery.fullqueryHashes));
prop.putXML("content_description-xml", (snippet == null) ? "" : snippet.getLineMarked(theQuery.fullqueryHashes));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + item, 0, 0));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + item, 0, 0), false);
return prop;
}

2
htroot/yacysearchtrailer.java
Unescape View File

@ -109,7 +109,7 @@ public class yacysearchtrailer {
}
prop.put("words", hintcount);
}
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + "bottomline", 0, 0));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.FINALIZATION + "-" + "bottomline", 0, 0), false);
return prop;
}

9
source/de/anomic/kelondro/index/IntegerHandleIndex.java
Unescape View File

@ -325,11 +325,10 @@ public class IntegerHandleIndex {
public static void main(String[] args) {
int count = (args.length == 0) ? 1000000 : Integer.parseInt(args[0]);
System.out.println("Starting test with " + count + " objects, minimum memory: " + (count * 16) + " bytes; " + MemoryControl.available(
) + " available");
System.out.println("Starting test with " + count + " objects, minimum memory: " + (count * 16) + " bytes; " + MemoryControl.available() + " available");
Random r = new Random(0);
long start = System.currentTimeMillis();
System.gc(); // for resource measurement
long a = MemoryControl.available();
IntegerHandleIndex idx = new IntegerHandleIndex(12, Base64Order.enhancedCoder, 0);
@ -357,8 +356,7 @@ public class IntegerHandleIndex {
if (d == null) hm.put(hash, 1); else hm.put(hash, d + 1);
}
long timej = ((long) count) * 1000L / (System.currentTimeMillis() - start);
System.out.println("Result HashMap: " +timej + " inc per second; " + count
+ " loops.");
System.out.println("Result HashMap: " +timej + " inc per second; " + count + " loops.");
System.gc();
long memj = a - MemoryControl.available();
System.out.println("Used Memory: " + memj + " bytes");
@ -366,6 +364,7 @@ public class IntegerHandleIndex {
System.out.println("Geschwindigkeitsfaktor j/k: " + (timej / timek));
System.out.println("Speicherplatzfaktor j/k: " + (memj / memk));
System.exit(0);
}
}

26
source/de/anomic/kelondro/index/ObjectIndexDaemon.java
Unescape View File

@ -32,7 +32,7 @@ import de.anomic.kelondro.index.Row.Entry;
import de.anomic.kelondro.index.Row.Queue;
import de.anomic.kelondro.order.CloneableIterator;
public class ObjectIndexDaemon implements ObjectIndex {
public class ObjectIndexDaemon {
private Row.Entry poison;
private PutScheduler putScheduler;
@ -71,11 +71,11 @@ public class ObjectIndexDaemon implements ObjectIndex {
}
}
public void addUnique(Entry row) throws IOException {
public void addUnique(Entry row) {
index.addUnique(row);
}
public void addUnique(List<Entry> rows) throws IOException {
public void addUnique(List<Entry> rows) {
index.addUnique(rows);
}
@ -100,7 +100,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return this.index.filename();
}
public Entry get(byte[] key) throws IOException {
public Entry get(byte[] key) {
Entry entry = this.queue.get(key);
if (entry != null) return entry;
return this.index.get(key);
@ -112,7 +112,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return this.index.has(key);
}
public Entry replace(Entry row) throws IOException {
public Entry replace(Entry row) {
Entry entry = get(row.getPrimaryKeyBytes());
try {
this.queue.put(row);
@ -122,7 +122,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return entry;
}
public void put(Entry row) throws IOException {
public void put(Entry row) {
try {
this.queue.put(row);
} catch (InterruptedException e) {
@ -130,7 +130,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
}
}
public void put(List<Entry> rows) throws IOException {
public void put(List<Entry> rows) {
for (Entry entry: rows) try {
this.queue.put(entry);
} catch (InterruptedException e) {
@ -138,14 +138,14 @@ public class ObjectIndexDaemon implements ObjectIndex {
}
}
public Entry remove(byte[] key) throws IOException {
public Entry remove(byte[] key) {
Entry entry = this.queue.delete(key);
if (entry == null) return this.index.remove(key);
this.index.remove(key);
return entry;
}
public synchronized ArrayList<RowCollection> removeDoubles() throws IOException {
public synchronized ArrayList<RowCollection> removeDoubles() {
try {
this.queue.put(poison);
this.putScheduler.join();
@ -159,7 +159,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return d;
}
public synchronized Entry removeOne() throws IOException {
public synchronized Entry removeOne() {
try {
this.queue.put(poison);
this.putScheduler.join();
@ -177,7 +177,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return this.index.row();
}
public synchronized CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException {
public synchronized CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) {
try {
this.queue.put(poison);
this.putScheduler.join();
@ -191,7 +191,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return keys;
}
public synchronized CloneableIterator<Entry> rows(boolean up, byte[] firstKey) throws IOException {
public synchronized CloneableIterator<Entry> rows(boolean up, byte[] firstKey) {
try {
this.queue.put(poison);
this.putScheduler.join();
@ -205,7 +205,7 @@ public class ObjectIndexDaemon implements ObjectIndex {
return rows;
}
public CloneableIterator<Entry> rows() throws IOException {
public CloneableIterator<Entry> rows() {
return rows(true, null);
}

12
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -155,7 +155,7 @@ public final class plasmaSearchEvent {
query.ranking,
query.constraint,
(query.domType == plasmaSearchQuery.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "remote search thread start", this.primarySearchThreads.length, System.currentTimeMillis() - timer), false);
// meanwhile do a local search
localSearchThread = new localSearchProcess();
@ -192,7 +192,7 @@ public final class plasmaSearchEvent {
IACount.put(wordhash, Integer.valueOf(container.size()));
IAResults.put(wordhash, ReferenceContainer.compressIndex(container, null, 1000).toString());
}
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMaps()[0].size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "abstract generation", this.rankedCache.searchContainerMaps()[0].size(), System.currentTimeMillis() - timer), false);
}
}
@ -203,18 +203,18 @@ public final class plasmaSearchEvent {
this.workerThreads[i] = new resultWorker(i, 10000, 2);
this.workerThreads[i].start();
}
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "online snippet fetch threads started", 0, 0));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "online snippet fetch threads started", 0, 0), false);
} else {
final long timer = System.currentTimeMillis();
// use only a single worker thread, thats enough
resultWorker worker = new resultWorker(0, 3000, 0);
worker.run();
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "offline snippet fetch", result.size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "offline snippet fetch", result.size(), System.currentTimeMillis() - timer), false);
}
// clean up events
cleanupEvents(false);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "event-cleanup", 0, 0));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "event-cleanup", 0, 0), false);
// store this search to a cache so it can be re-used
if (MemoryControl.available() < 1024 * 1024 * 10) cleanupEvents(true);
@ -599,7 +599,7 @@ public final class plasmaSearchEvent {
public ResultEntry oneResult(final int item) {
// check if we already retrieved this item (happens if a search
// pages is accessed a second time)
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "obtain one result entry - start", 0, 0));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), "obtain one result entry - start", 0, 0), false);
if (this.result.sizeStore() > item) {
// we have the wanted result already in the result array .. return that
return this.result.element(item).element;

8
source/de/anomic/plasma/plasmaSearchRankingProcess.java
Unescape View File

@ -111,7 +111,7 @@ public final class plasmaSearchRankingProcess {
long timer = System.currentTimeMillis();
this.localSearchContainerMaps = wordIndex.localSearchContainers(query.queryHashes, query.excludeHashes, null);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.COLLECTION, this.localSearchContainerMaps[0].size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.COLLECTION, this.localSearchContainerMaps[0].size(), System.currentTimeMillis() - timer), false);
// join and exclude the local result
timer = System.currentTimeMillis();
@ -120,7 +120,7 @@ public final class plasmaSearchRankingProcess {
this.localSearchContainerMaps[0].values(),
this.localSearchContainerMaps[1].values(),
query.maxDistance);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.JOIN, index.size(), System.currentTimeMillis() - timer), false);
if (index.size() == 0) {
return;
}
@ -145,7 +145,7 @@ public final class plasmaSearchRankingProcess {
// normalize entries
final ArrayList<ReferenceVars> decodedEntries = this.order.normalizeWith(index);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.NORMALIZING, index.size(), System.currentTimeMillis() - timer), false);
// iterate over normalized entries and select some that are better than currently stored
timer = System.currentTimeMillis();
@ -211,7 +211,7 @@ public final class plasmaSearchRankingProcess {
}
//if ((query.neededResults() > 0) && (container.size() > query.neededResults())) remove(true, true);
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(query.id(true), plasmaSearchEvent.PRESORT, index.size(), System.currentTimeMillis() - timer), false);
}
private boolean testFlags(final Reference ientry) {

28
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -223,12 +223,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
public ArrayList<plasmaSearchQuery> localSearches; // array of search result properties as HashMaps
public ArrayList<plasmaSearchQuery> remoteSearches; // array of search result properties as HashMaps
public HashMap<String, TreeSet<Long>> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time)
public long lastseedcheckuptime = -1;
public long indexedPages = 0;
public long lastindexedPages = 0;
public double requestedQueries = 0d;
public double lastrequestedQueries = 0d;
public int totalPPM = 0;
public double totalQPM = 0d;
public TreeMap<String, String> clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used
public URLLicense licensedURLs;
@ -1677,10 +1673,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
if (System.currentTimeMillis() - lastPPMUpdate > 30000) {
// we don't want to do this too often
updateMySeed();
serverProfiling.update("ppm", Long.valueOf(currentPPM()));
serverProfiling.update("ppm", Long.valueOf(currentPPM()), true);
lastPPMUpdate = System.currentTimeMillis();
}
serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false));
serverProfiling.update("indexed", queueEntry.url().toNormalform(true, false), false);
// if this was performed for a remote crawl request, notify requester
final yacySeed initiatorPeer = queueEntry.initiatorPeer();
@ -1975,11 +1971,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
}
public int currentPPM() {
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
final long uptimediff = uptime - lastseedcheckuptime;
final long indexedcdiff = indexedPages - lastindexedPages;
totalPPM = (int) (indexedPages * 60 / Math.max(uptime, 1));
return Math.round(Math.max(indexedcdiff, 0f) * 60f / Math.max(uptimediff, 1f));
return serverProfiling.countEvents("indexed", 30000) * 2;
}
public void updateMySeed() {
@ -1990,19 +1982,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
webIndex.seedDB.mySeed().put(yacySeed.NAME, getConfig("peerName", "nameless"));
webIndex.seedDB.mySeed().put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(getConfig("port", "8080"))));
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
final long uptimediff = uptime - lastseedcheckuptime;
final long indexedcdiff = indexedPages - lastindexedPages;
//double requestcdiff = requestedQueries - lastrequestedQueries;
if (uptimediff > 300 || uptimediff <= 0 || lastseedcheckuptime == -1 ) {
lastseedcheckuptime = uptime;
lastindexedPages = indexedPages;
lastrequestedQueries = requestedQueries;
}
//the speed of indexing (pages/minute) of the peer
totalPPM = (int) (indexedPages * 60 / Math.max(uptime, 1));
webIndex.seedDB.mySeed().put(yacySeed.ISPEED, Long.toString(Math.round(Math.max(indexedcdiff, 0f) * 60f / Math.max(uptimediff, 1f))));
final long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
webIndex.seedDB.mySeed().put(yacySeed.ISPEED, Integer.toString(currentPPM()));
totalQPM = requestedQueries * 60d / Math.max(uptime, 1d);
webIndex.seedDB.mySeed().put(yacySeed.RSPEED, Double.toString(totalQPM /*Math.max((float) requestcdiff, 0f) * 60f / Math.max((float) uptimediff, 1f)*/ ));

4
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -467,7 +467,7 @@ public final class plasmaWordIndex implements Index {
// flush elements that are too big. This flushing depends on the fact that the flush rule
// selects the biggest elements first for flushing. If it does not for any reason, the following
// loop would not terminate.
serverProfiling.update("wordcache", Long.valueOf(cs));
serverProfiling.update("wordcache", Long.valueOf(cs), true);
// To ensure termination an additional counter is used
int l = 0;
while (theCache.size() > 0 && (l++ < 100) && (theCache.maxURLinCache() > wCacheMaxChunk)) {
@ -479,7 +479,7 @@ public final class plasmaWordIndex implements Index {
(MemoryControl.available() < collections.minMem()))) {
flushCacheOne(theCache);
}
if (cacheSize() != cs) serverProfiling.update("wordcache", Long.valueOf(cacheSize()));
if (cacheSize() != cs) serverProfiling.update("wordcache", Long.valueOf(cacheSize()), true);
}
}

18
source/de/anomic/server/serverProfiling.java
Unescape View File

@ -40,7 +40,7 @@ public class serverProfiling extends Thread {
private static serverProfiling systemProfiler = null;
public static void startSystemProfiling() {
systemProfiler = new serverProfiling(1000);
systemProfiler = new serverProfiling(1500);
systemProfiler.start();
}
@ -58,7 +58,7 @@ public class serverProfiling extends Thread {
public void run() {
while (running) {
update("memory", Long.valueOf(MemoryControl.used()));
update("memory", Long.valueOf(MemoryControl.used()), true);
try {
Thread.sleep(this.delaytime);
} catch (final InterruptedException e) {
@ -67,13 +67,13 @@ public class serverProfiling extends Thread {
}
}
public static void update(final String eventName, final Object eventPayload) {
public static void update(final String eventName, final Object eventPayload, boolean useProtection) {
// get event history container
Long lastAcc = eventAccess.get(eventName);
if (lastAcc == null) {
eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis()));
} else {
if (System.currentTimeMillis() - lastAcc.longValue() > 1000) {
if (!useProtection || System.currentTimeMillis() - lastAcc.longValue() > 1000) {
eventAccess.put(eventName, Long.valueOf(System.currentTimeMillis()));
} else {
return; // protect against too heavy load
@ -108,6 +108,16 @@ public class serverProfiling extends Thread {
return (historyMaps.containsKey(eventName) ? (historyMaps.get(eventName)) : new ConcurrentLinkedQueue<Event>()).iterator();
}
public static int countEvents(final String eventName, long time) {
Iterator<Event> i = history(eventName);
long now = System.currentTimeMillis();
int count = 0;
while (i.hasNext()) {
if (now - i.next().time < time) count++;
}
return count;
}
public static class Event {
public Object payload;
public long time;

Write Preview
Loading…
Cancel
Save