boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced logging

Browse Source 
- more logging output
- fix in log line preparation
- added filter to log page
- some small bugfixes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2707 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent 08aa9d4c07
commit db294687ea
11 changed files with 117 additions and 66 deletions
Show Stats Download Patch File Download Diff File

72
.classpath
Unescape View File

@ -1,36 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="source"/>
<classpathentry excluding="htdocsdefault/|locale/|proxymsg/|yacy/|env/|env/" output="htroot" kind="src" path="htroot"/>
<classpathentry kind="src" path="htroot/htdocsdefault"/>
<classpathentry kind="src" path="htroot/yacy"/>
<classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-collections.jar"/>
<classpathentry kind="lib" path="lib/commons-pool-1.2.jar"/>
<classpathentry kind="lib" path="libx/axis-ant.jar"/>
<classpathentry kind="lib" path="libx/axis.jar"/>
<classpathentry kind="lib" path="libx/bzip2.jar"/>
<classpathentry kind="lib" path="libx/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="libx/commons-discovery.jar"/>
<classpathentry kind="lib" path="libx/commons-jxpath-1.1.jar"/>
<classpathentry kind="lib" path="libx/commons-logging.jar"/>
<classpathentry kind="lib" path="libx/informa-0.6.0.jar"/>
<classpathentry kind="lib" path="libx/jakarta-oro-2.0.7.jar"/>
<classpathentry kind="lib" path="libx/jaxrpc.jar"/>
<classpathentry kind="lib" path="libx/jdom.jar"/>
<classpathentry kind="lib" path="libx/jmimemagic-0.1.0.jar"/>
<classpathentry kind="lib" path="libx/jrpm-head.jar"/>
<classpathentry kind="lib" path="libx/jrpm-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="libx/jsch-0.1.21.jar"/>
<classpathentry kind="lib" path="libx/log4j-1.2.9.jar"/>
<classpathentry kind="lib" path="libx/odf_utils_05_11_29.jar"/>
<classpathentry kind="lib" path="libx/PDFBox-0.7.2.jar"/>
<classpathentry kind="lib" path="libx/saaj.jar"/>
<classpathentry kind="lib" path="libx/sbbi-upnplib-1.0.3.jar"/>
<classpathentry kind="lib" path="libx/tar.jar"/>
<classpathentry kind="lib" path="libx/tm-extractors-0.4.jar"/>
<classpathentry kind="lib" path="libx/wsdl4j.jar"/>
<classpathentry kind="lib" path="libx/xerces.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="source"/>
<classpathentry excluding="htdocsdefault/|locale/|proxymsg/|yacy/|env/|env/" output="htroot" kind="src" path="htroot"/>
<classpathentry kind="src" path="htroot/htdocsdefault"/>
<classpathentry kind="src" path="htroot/yacy"/>
<classpathentry kind="src" path="htroot/env"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="lib/commons-collections.jar"/>
<classpathentry kind="lib" path="lib/commons-pool-1.2.jar"/>
<classpathentry kind="lib" path="libx/axis-ant.jar"/>
<classpathentry kind="lib" path="libx/axis.jar"/>
<classpathentry kind="lib" path="libx/bzip2.jar"/>
<classpathentry kind="lib" path="libx/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="libx/commons-discovery.jar"/>
<classpathentry kind="lib" path="libx/commons-jxpath-1.1.jar"/>
<classpathentry kind="lib" path="libx/commons-logging.jar"/>
<classpathentry kind="lib" path="libx/informa-0.6.0.jar"/>
<classpathentry kind="lib" path="libx/jakarta-oro-2.0.7.jar"/>
<classpathentry kind="lib" path="libx/jaxrpc.jar"/>
<classpathentry kind="lib" path="libx/jdom.jar"/>
<classpathentry kind="lib" path="libx/jmimemagic-0.1.0.jar"/>
<classpathentry kind="lib" path="libx/jrpm-head.jar"/>
<classpathentry kind="lib" path="libx/jrpm-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="libx/jsch-0.1.21.jar"/>
<classpathentry kind="lib" path="libx/log4j-1.2.9.jar"/>
<classpathentry kind="lib" path="libx/odf_utils_05_11_29.jar"/>
<classpathentry kind="lib" path="libx/PDFBox-0.7.2.jar"/>
<classpathentry kind="lib" path="libx/saaj.jar"/>
<classpathentry kind="lib" path="libx/sbbi-upnplib-1.0.3.jar"/>
<classpathentry kind="lib" path="libx/tar.jar"/>
<classpathentry kind="lib" path="libx/tm-extractors-0.4.jar"/>
<classpathentry kind="lib" path="libx/wsdl4j.jar"/>
<classpathentry kind="lib" path="libx/xerces.jar"/>
<classpathentry kind="output" path="gen"/>
</classpath>

1
htroot/ViewLog_p.html
Unescape View File

@ -12,6 +12,7 @@
<fieldset>
<input type="text" name="lines" id="lines" size ="4" value="#[lines]#" maxlength="9" /><label for="lines"> Lines</label>
<input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>
<input type="text" name="filter" id="filter" size ="30" value="#[filter]#" maxlength="60" />
<input type="submit" value="refresh" />
</fieldset>
</form>

20
htroot/ViewLog_p.java
Unescape View File

@ -48,6 +48,7 @@
import java.util.logging.Handler;
import java.util.logging.Logger;
import java.util.regex.PatternSyntaxException;
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
@ -60,7 +61,8 @@ public class ViewLog_p {
serverObjects prop = new serverObjects();
String[] log = new String[0];
boolean reversed = false;
int lines = 50;
int lines = 200;
String filter = ".*.*";
if(post != null){
if(post.containsKey("mode") && ((String)post.get("mode")).equals("reversed")){
@ -69,6 +71,9 @@ public class ViewLog_p {
if(post.containsKey("lines")){
lines = Integer.parseInt((String)post.get("lines"));
}
if(post.containsKey("filter")){
filter = (String)post.get("filter");
}
}
@ -83,21 +88,26 @@ public class ViewLog_p {
prop.put("reverseChecked", reversed ? 1 : 0);
prop.put("lines", lines);
prop.put("filter", filter);
int level = 0;
int lc = 0;
for (int i=0; i < log.length; i++) {
String nextLogLine = log[i];
String nextLogLine = log[i].trim();
try {if (!(nextLogLine.matches(filter))) continue;} catch (PatternSyntaxException e) {}
if (nextLogLine.startsWith("E ")) level = 4;
else if (nextLogLine.startsWith("W ")) level = 3;
else if (nextLogLine.startsWith("S ")) level = 2;
else if (nextLogLine.startsWith("I ")) level = 1;
else if (nextLogLine.startsWith("D ")) level = 0;
prop.put("log_" + i + "_level",level);
prop.put("log_" + i + "_line", nextLogLine);
prop.put("log_" + lc + "_level",level);
prop.put("log_" + lc + "_line", nextLogLine);
lc++;
}
prop.put("log",log.length);
prop.put("log",lc);
// return rewrite properties
return prop;

2
htroot/env/base.css vendored
Unescape View File

@ -341,7 +341,7 @@ dl.pairs dd {
/* for pages: */
body#ViewLog div.log {
height:480px;
height:600px;
overflow:scroll;
}

2
source/de/anomic/kelondro/kelondroRowCollection.java
Unescape View File

@ -214,7 +214,7 @@ public class kelondroRowCollection {
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) {
ensureSize(chunkcount + 1);
System.arraycopy(a, 0, chunkcache, rowdef.objectsize() * chunkcount, l);
System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l);
chunkcount++;
}
this.lastTimeWrote = System.currentTimeMillis();

37
source/de/anomic/kelondro/kelondroRowSet.java
Unescape View File

@ -447,7 +447,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
System.out.println("RESULT SIZE: " + c.size());
*/
/*
// performance test for put
long start = System.currentTimeMillis();
kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
Random random = new Random(0);
@ -459,6 +460,40 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
*/
// remove test
long start = System.currentTimeMillis();
kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
byte[] key;
int testsize = 5000;
byte[][] delkeys = new byte[testsize / 5][];
Random random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
if (i % 5 != 0) continue;
delkeys[i / 5] = key;
}
random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
c.put(c.rowdef.newEntry(new byte[][]{key, key}));
if (i % 1000 == 0) {
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape();
}
}
for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
c.shape();
random = new Random(0);
for (int i = 0; i < testsize; i++) {
key = randomHash(random);
if (i % 5 == 0) continue;
if (c.get(key) == null) System.out.println("missing entry " + new String(key));
}
c.shape();
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
public static byte[] randomHash(final long r0, final long r1) {

11
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -70,7 +70,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
private plasmaCrawlLURL urlStore;
private plasmaSnippetCache snippetCache;
private indexContainer rcContainers; // cache for results
private int rcContainerCount;
private int rcContainerFlushCount;
private Map rcAbstracts; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
private plasmaSearchTimingProfile profileLocal, profileGlobal;
private boolean postsort;
@ -92,7 +92,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
this.urlStore = urlStore;
this.snippetCache = snippetCache;
this.rcContainers = new indexContainer(null);
this.rcContainerCount = 0;
this.rcContainerFlushCount = 0;
this.rcAbstracts = new TreeMap();
this.profileLocal = localTiming;
this.profileGlobal = remoteTiming;
@ -434,7 +434,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// this must be called after search results had been computed
// it is wise to call this within a separate thread because
// this method waits until all threads are finished
serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
int remaining = 0;
if (primarySearchThreads == null) return;
long starttime = System.currentTimeMillis();
@ -456,7 +457,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
//log.logFine("FINISHED FLUSH RESULTS PROCESS for query " + query.hashes(","));
}
serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
// finally delete the temporary index
rcContainers = null;
@ -483,7 +484,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
rcContainers.clear();
}
}
rcContainerCount += count;
rcContainerFlushCount += count;
}
}

3
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -363,8 +363,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
if (container == null) {
container = dhtInCache.getContainer(wordHash, urlselection, true, -1);
} else {
indexContainer ic = dhtInCache.getContainer(wordHash, urlselection, true, -1);
if (ic != null) container.add(ic, -1);
container.add(dhtInCache.getContainer(wordHash, urlselection, true, -1), -1);
}
// get from collection index

2
source/de/anomic/server/logging/GuiHandler.java
Unescape View File

@ -201,7 +201,7 @@ public class GuiHandler extends Handler{
for (int i = 0; i < lineCount; i++) {
int ix = (reversed) ?
Math.abs((start-i)%this.buffer.length) :
(start+i)%this.buffer.length;
(start - lineCount + i) % this.buffer.length;
record = this.buffer[ix];
logMessages.add(logFormatter.format(record));
}

13
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -364,7 +364,7 @@ public final class yacyClient {
}
}
public static int search(
public static String[] search(
String wordhashes,
String urlhashes,
String prefer,
@ -454,12 +454,12 @@ public final class yacyClient {
} catch (IOException e) {
yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
yacyCore.peerActions.peerDeparture(targetPeer);
return 0;
return null;
}
if (result.size() == 0) {
yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
return 0;
return null;
}
// compute all computation times
@ -497,6 +497,7 @@ public final class yacyClient {
// insert results to containers
plasmaCrawlLURL.Entry urlEntry;
String[] urls = new String[results];
for (int n = 0; n < results; n++) {
// get one single search result
urlEntry = urlManager.newEntry((String) result.get("resource" + n), true);
@ -540,6 +541,8 @@ public final class yacyClient {
for (int m = 0; m < words; m++) {
container[m].add(new indexEntry[]{entry}, System.currentTimeMillis());
}
// store url hash for statistics
urls[n] = urlEntry.hash();
}
// insert the containers to the index
@ -574,11 +577,11 @@ public final class yacyClient {
searchtime = totalrequesttime;
}
yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references"));
return results;
return urls;
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e);
e.printStackTrace();
return 0;
return null;
}
}

20
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -70,7 +70,7 @@ public class yacySearch extends Thread {
final private plasmaURLPattern blacklist;
final private plasmaSnippetCache snippetCache;
final private yacySeed targetPeer;
private int links;
private String[] urls;
private int maxDistance;
final private plasmaSearchTimingProfile timingProfile;
final private plasmaSearchRankingProfile rankingProfile;
@ -94,18 +94,20 @@ public class yacySearch extends Thread {
this.blacklist = blacklist;
this.snippetCache = snippetCache;
this.targetPeer = targetPeer;
this.links = -1;
this.urls = null;
this.maxDistance = maxDistance;
this.timingProfile = (plasmaSearchTimingProfile) timingProfile.clone();
this.rankingProfile = rankingProfile;
}
public void run() {
this.links = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes);
if (links != 0) {
yacyCore.seedDB.mySeed.incRI(links);
yacyCore.seedDB.mySeed.incRU(links);
this.urls = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
StringBuffer urllist = new StringBuffer(this.urls.length * 13);
for (int i = 0; i < this.urls.length; i++) urllist.append(this.urls[i]).append(' ');
yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + urls.length + " links for word hash " + wordhashes + ": " + new String(urllist));
if (urls != null) {
yacyCore.seedDB.mySeed.incRI(urls.length);
yacyCore.seedDB.mySeed.incRU(urls.length);
}
}
@ -117,7 +119,7 @@ public class yacySearch extends Thread {
}
public int links() {
return this.links;
return this.urls.length;
}
public plasmaSearchTimingProfile timingProfile() {
@ -241,7 +243,7 @@ public class yacySearch extends Thread {
public static int collectedLinks(yacySearch[] searchThreads) {
int links = 0;
for (int i = 0; i < searchThreads.length; i++) {
if (!(searchThreads[i].isAlive())) links += searchThreads[i].links;
if (!(searchThreads[i].isAlive())) links += searchThreads[i].urls.length;
}
return links;
}

Write Preview
Loading…
Cancel
Save