boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced logging

Browse Source 
- more logging output
- fix in log line preparation
- added filter to log page
- some small bugfixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2707 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 08aa9d4c07
commit db294687ea
11 changed files with 117 additions and 66 deletions
Show Stats Download Patch File Download Diff File

1
htroot/ViewLog_p.html
Unescape View File

@ -12,6 +12,7 @@
<fieldset>
<input type="text" name="lines" id="lines" size ="4" value="#[lines]#" maxlength="9" /><label for="lines"> Lines</label>
<input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>
<input type="text" name="filter" id="filter" size ="30" value="#[filter]#" maxlength="60" />
<input type="submit" value="refresh" />
</fieldset>
</form>

20
htroot/ViewLog_p.java
Unescape View File

@ -48,6 +48,7 @@
import java.util.logging.Handler;
import java.util.logging.Logger;
import java.util.regex.PatternSyntaxException;
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
@ -60,7 +61,8 @@ public class ViewLog_p {
serverObjects prop = new serverObjects();
String[] log = new String[0];
boolean reversed = false;
int lines = 50;
int lines = 200;
String filter = ".*.*";
if(post != null){
if(post.containsKey("mode") && ((String)post.get("mode")).equals("reversed")){
@ -69,6 +71,9 @@ public class ViewLog_p {
if(post.containsKey("lines")){
lines = Integer.parseInt((String)post.get("lines"));
}
if(post.containsKey("filter")){
filter = (String)post.get("filter");
}
}
@ -83,21 +88,26 @@ public class ViewLog_p {
prop.put("reverseChecked", reversed ? 1 : 0);
prop.put("lines", lines);
prop.put("filter", filter);
int level = 0;
int lc = 0;
for (int i=0; i < log.length; i++) {
String nextLogLine = log[i];
String nextLogLine = log[i].trim();
try {if (!(nextLogLine.matches(filter))) continue;} catch (PatternSyntaxException e) {}
if (nextLogLine.startsWith("E ")) level = 4;
else if (nextLogLine.startsWith("W ")) level = 3;
else if (nextLogLine.startsWith("S ")) level = 2;
else if (nextLogLine.startsWith("I ")) level = 1;
else if (nextLogLine.startsWith("D ")) level = 0;
prop.put("log_" + i + "_level",level);
prop.put("log_" + i + "_line", nextLogLine);
prop.put("log_" + lc + "_level",level);
prop.put("log_" + lc + "_line", nextLogLine);
lc++;
}
prop.put("log",log.length);
prop.put("log",lc);
// return rewrite properties
return prop;

2
htroot/env/base.css vendored
Unescape View File

@ -341,7 +341,7 @@ dl.pairs dd {
/* for pages: */
body#ViewLog div.log {
height:480px;
height:600px;
overflow:scroll;
}

2
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -214,7 +214,7 @@ public class kelondroRowCollection {
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) {
ensureSize(chunkcount + 1);
System.arraycopy(a, 0, chunkcache, rowdef.objectsize() * chunkcount, l);
System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l);
chunkcount++;
}
this.lastTimeWrote = System.currentTimeMillis();

37
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -447,7 +447,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
System.out.println("RESULT SIZE: " + c.size());
*/
/*
// performance test for put
long start = System.currentTimeMillis();
kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
Random random = new Random(0);
@ -459,6 +460,40 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
*/
// remove test
long start = System.currentTimeMillis();
kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
byte[] key;
int testsize = 5000;
byte[][] delkeys = new byte[testsize / 5][];
Random random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
if (i % 5 != 0) continue;
delkeys[i / 5] = key;
}
random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
c.put(c.rowdef.newEntry(new byte[][]{key, key}));
if (i % 1000 == 0) {
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape();
}
}
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape();
random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
if (i % 5 == 0) continue;
if (c.get(key) == null) System.out.println("missing entry " + new String(key));
}
c.shape();
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
public static byte[] randomHash(final long r0, final long r1) {

9
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -70,7 +70,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
private plasmaCrawlLURL urlStore;
private plasmaSnippetCache snippetCache;
private indexContainer rcContainers; // cache for results
private int rcContainerCount;
private int rcContainerFlushCount;
private Map rcAbstracts; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
private plasmaSearchTimingProfile profileLocal, profileGlobal;
private boolean postsort;
@ -92,7 +92,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
this.urlStore = urlStore;
this.snippetCache = snippetCache;
this.rcContainers = new indexContainer(null);
this.rcContainerCount = 0;
this.rcContainerFlushCount = 0;
this.rcAbstracts = new TreeMap();
this.profileLocal = localTiming;
this.profileGlobal = remoteTiming;
@ -434,6 +434,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// this must be called after search results had been computed
// it is wise to call this within a separate thread because
// this method waits until all threads are finished
serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
int remaining = 0;
if (primarySearchThreads == null) return;
@ -456,7 +457,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
//log.logFine("FINISHED FLUSH RESULTS PROCESS for query " + query.hashes(","));
}
serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
// finally delete the temporary index
rcContainers = null;
@ -483,7 +484,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
rcContainers.clear();
}
}
rcContainerCount += count;
rcContainerFlushCount += count;
}
}

3
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -363,8 +363,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
if (container == null) {
container = dhtInCache.getContainer(wordHash, urlselection, true, -1);
} else {
indexContainer ic = dhtInCache.getContainer(wordHash, urlselection, true, -1);
if (ic != null) container.add(ic, -1);
container.add(dhtInCache.getContainer(wordHash, urlselection, true, -1), -1);
}
// get from collection index

2
source/de/anomic/server/logging/GuiHandler.java
Unescape View File

@ -201,7 +201,7 @@ public class GuiHandler extends Handler{
for (int i = 0; i < lineCount; i++) {
int ix = (reversed) ?
Math.abs((start-i)%this.buffer.length) :
(start+i)%this.buffer.length;
(start - lineCount + i) % this.buffer.length;
record = this.buffer[ix];
logMessages.add(logFormatter.format(record));
}

13
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -364,7 +364,7 @@ public final class yacyClient {
}
}
public static int search(
public static String[] search(
String wordhashes,
String urlhashes,
String prefer,
@ -454,12 +454,12 @@ public final class yacyClient {
} catch (IOException e) {
yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
yacyCore.peerActions.peerDeparture(targetPeer);
return 0;
return null;
}
if (result.size() == 0) {
yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
return 0;
return null;
}
// compute all computation times
@ -497,6 +497,7 @@ public final class yacyClient {
// insert results to containers
plasmaCrawlLURL.Entry urlEntry;
String[] urls = new String[results];
for (int n = 0; n < results; n++) {
// get one single search result
urlEntry = urlManager.newEntry((String) result.get("resource" + n), true);
@ -540,6 +541,8 @@ public final class yacyClient {
for (int m = 0; m < words; m++) {
container[m].add(new indexEntry[]{entry}, System.currentTimeMillis());
}
// store url hash for statistics
urls[n] = urlEntry.hash();
}
// insert the containers to the index
@ -574,11 +577,11 @@ public final class yacyClient {
searchtime = totalrequesttime;
}
yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references"));
return results;
return urls;
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e);
e.printStackTrace();
return 0;
return null;
}
}

20
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -70,7 +70,7 @@ public class yacySearch extends Thread {
final private plasmaURLPattern blacklist;
final private plasmaSnippetCache snippetCache;
final private yacySeed targetPeer;
private int links;
private String[] urls;
private int maxDistance;
final private plasmaSearchTimingProfile timingProfile;
final private plasmaSearchRankingProfile rankingProfile;
@ -94,18 +94,20 @@ public class yacySearch extends Thread {
this.blacklist = blacklist;
this.snippetCache = snippetCache;
this.targetPeer = targetPeer;
this.links = -1;
this.urls = null;
this.maxDistance = maxDistance;
this.timingProfile = (plasmaSearchTimingProfile) timingProfile.clone();
this.rankingProfile = rankingProfile;
}
public void run() {
this.links = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes);
if (links != 0) {
yacyCore.seedDB.mySeed.incRI(links);
yacyCore.seedDB.mySeed.incRU(links);
this.urls = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
StringBuffer urllist = new StringBuffer(this.urls.length * 13);
for (int i = 0; i < this.urls.length; i++) urllist.append(this.urls[i]).append(' ');
yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + urls.length + " links for word hash " + wordhashes + ": " + new String(urllist));
if (urls != null) {
yacyCore.seedDB.mySeed.incRI(urls.length);
yacyCore.seedDB.mySeed.incRU(urls.length);
}
}
@ -117,7 +119,7 @@ public class yacySearch extends Thread {
}
public int links() {
return this.links;
return this.urls.length;
}
public plasmaSearchTimingProfile timingProfile() {
@ -241,7 +243,7 @@ public class yacySearch extends Thread {
public static int collectedLinks(yacySearch[] searchThreads) {
int links = 0;
for (int i = 0; i < searchThreads.length; i++) {
if (!(searchThreads[i].isAlive())) links += searchThreads[i].links;
if (!(searchThreads[i].isAlive())) links += searchThreads[i].urls.length;
}
return links;
}

Write Preview
Loading…
Cancel
Save