diff --git a/.classpath b/.classpath
index c9eba99d5..28b75dfde 100644
--- a/.classpath
+++ b/.classpath
@@ -1,36 +1,36 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/htroot/ViewLog_p.html b/htroot/ViewLog_p.html
index 93d9138d5..0cc33c260 100644
--- a/htroot/ViewLog_p.html
+++ b/htroot/ViewLog_p.html
@@ -12,6 +12,7 @@
diff --git a/htroot/ViewLog_p.java b/htroot/ViewLog_p.java
index 1f59be1f4..050a28ea6 100644
--- a/htroot/ViewLog_p.java
+++ b/htroot/ViewLog_p.java
@@ -48,6 +48,7 @@
import java.util.logging.Handler;
import java.util.logging.Logger;
+import java.util.regex.PatternSyntaxException;
import de.anomic.http.httpHeader;
import de.anomic.server.serverObjects;
@@ -60,7 +61,8 @@ public class ViewLog_p {
serverObjects prop = new serverObjects();
String[] log = new String[0];
boolean reversed = false;
- int lines = 50;
+ int lines = 200;
+ String filter = ".*.*";
if(post != null){
if(post.containsKey("mode") && ((String)post.get("mode")).equals("reversed")){
@@ -69,6 +71,9 @@ public class ViewLog_p {
if(post.containsKey("lines")){
lines = Integer.parseInt((String)post.get("lines"));
}
+ if(post.containsKey("filter")){
+ filter = (String)post.get("filter");
+ }
}
@@ -83,21 +88,26 @@ public class ViewLog_p {
prop.put("reverseChecked", reversed ? 1 : 0);
prop.put("lines", lines);
+ prop.put("filter", filter);
int level = 0;
+ int lc = 0;
for (int i=0; i < log.length; i++) {
- String nextLogLine = log[i];
+ String nextLogLine = log[i].trim();
+ try {if (!(nextLogLine.matches(filter))) continue;} catch (PatternSyntaxException e) {}
+
if (nextLogLine.startsWith("E ")) level = 4;
else if (nextLogLine.startsWith("W ")) level = 3;
else if (nextLogLine.startsWith("S ")) level = 2;
else if (nextLogLine.startsWith("I ")) level = 1;
else if (nextLogLine.startsWith("D ")) level = 0;
- prop.put("log_" + i + "_level",level);
- prop.put("log_" + i + "_line", nextLogLine);
+ prop.put("log_" + lc + "_level",level);
+ prop.put("log_" + lc + "_line", nextLogLine);
+ lc++;
}
- prop.put("log",log.length);
+ prop.put("log",lc);
// return rewrite properties
return prop;
diff --git a/htroot/env/base.css b/htroot/env/base.css
index 277def36d..6108c96e8 100644
--- a/htroot/env/base.css
+++ b/htroot/env/base.css
@@ -341,7 +341,7 @@ dl.pairs dd {
/* for pages: */
body#ViewLog div.log {
- height:480px;
+ height:600px;
overflow:scroll;
}
diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java
index d0d62a940..2fb0a4459 100644
--- a/source/de/anomic/kelondro/kelondroRowCollection.java
+++ b/source/de/anomic/kelondro/kelondroRowCollection.java
@@ -214,7 +214,7 @@ public class kelondroRowCollection {
int l = Math.min(rowdef.objectsize(), Math.min(alength, a.length - astart));
synchronized (chunkcache) {
ensureSize(chunkcount + 1);
- System.arraycopy(a, 0, chunkcache, rowdef.objectsize() * chunkcount, l);
+ System.arraycopy(a, astart, chunkcache, rowdef.objectsize() * chunkcount, l);
chunkcount++;
}
this.lastTimeWrote = System.currentTimeMillis();
diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java
index 68985b1e8..fc00799a1 100644
--- a/source/de/anomic/kelondro/kelondroRowSet.java
+++ b/source/de/anomic/kelondro/kelondroRowSet.java
@@ -447,7 +447,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
System.out.println("RESULT SIZE: " + c.size());
*/
-
+ /*
+ // performance test for put
long start = System.currentTimeMillis();
kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
Random random = new Random(0);
@@ -459,6 +460,40 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd
}
System.out.println("RESULT SIZE: " + c.size());
System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
+ */
+
+ // remove test
+ long start = System.currentTimeMillis();
+ kelondroRowSet c = new kelondroRowSet(new kelondroRow("byte[] a-12, byte[] b-12"), 0);
+ byte[] key;
+ int testsize = 5000;
+ byte[][] delkeys = new byte[testsize / 5][];
+ Random random = new Random(0);
+ for (int i = 0; i < testsize; i++) {
+ key = randomHash(random);
+ if (i % 5 != 0) continue;
+ delkeys[i / 5] = key;
+ }
+ random = new Random(0);
+ for (int i = 0; i < testsize; i++) {
+ key = randomHash(random);
+ c.put(c.rowdef.newEntry(new byte[][]{key, key}));
+ if (i % 1000 == 0) {
+ for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
+ c.shape();
+ }
+ }
+ for (int j = 0; j < delkeys.length; j++) c.remove(delkeys[j]);
+ c.shape();
+ random = new Random(0);
+ for (int i = 0; i < testsize; i++) {
+ key = randomHash(random);
+ if (i % 5 == 0) continue;
+ if (c.get(key) == null) System.out.println("missing entry " + new String(key));
+ }
+ c.shape();
+ System.out.println("RESULT SIZE: " + c.size());
+ System.out.println("Time: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
}
public static byte[] randomHash(final long r0, final long r1) {
diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java
index bd4463586..71da580a2 100644
--- a/source/de/anomic/plasma/plasmaSearchEvent.java
+++ b/source/de/anomic/plasma/plasmaSearchEvent.java
@@ -70,7 +70,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
private plasmaCrawlLURL urlStore;
private plasmaSnippetCache snippetCache;
private indexContainer rcContainers; // cache for results
- private int rcContainerCount;
+ private int rcContainerFlushCount;
private Map rcAbstracts; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
private plasmaSearchTimingProfile profileLocal, profileGlobal;
private boolean postsort;
@@ -92,7 +92,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
this.urlStore = urlStore;
this.snippetCache = snippetCache;
this.rcContainers = new indexContainer(null);
- this.rcContainerCount = 0;
+ this.rcContainerFlushCount = 0;
this.rcAbstracts = new TreeMap();
this.profileLocal = localTiming;
this.profileGlobal = remoteTiming;
@@ -434,7 +434,8 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
// this must be called after search results had been computed
// it is wise to call this within a separate thread because
// this method waits until all threads are finished
-
+ serverLog.logFine("PLASMA", "STARTED FLUSHING GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
+
int remaining = 0;
if (primarySearchThreads == null) return;
long starttime = System.currentTimeMillis();
@@ -456,7 +457,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
//log.logFine("FINISHED FLUSH RESULTS PROCESS for query " + query.hashes(","));
}
- serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
+ serverLog.logFine("PLASMA", "FINISHED FLUSHING " + rcContainerFlushCount + " GLOBAL SEARCH RESULTS FOR SEARCH " + query.queryWords);
// finally delete the temporary index
rcContainers = null;
@@ -483,7 +484,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
rcContainers.clear();
}
}
- rcContainerCount += count;
+ rcContainerFlushCount += count;
}
}
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java
index 001f564d8..df03367d6 100644
--- a/source/de/anomic/plasma/plasmaWordIndex.java
+++ b/source/de/anomic/plasma/plasmaWordIndex.java
@@ -363,8 +363,7 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
if (container == null) {
container = dhtInCache.getContainer(wordHash, urlselection, true, -1);
} else {
- indexContainer ic = dhtInCache.getContainer(wordHash, urlselection, true, -1);
- if (ic != null) container.add(ic, -1);
+ container.add(dhtInCache.getContainer(wordHash, urlselection, true, -1), -1);
}
// get from collection index
diff --git a/source/de/anomic/server/logging/GuiHandler.java b/source/de/anomic/server/logging/GuiHandler.java
index c509ffcc4..b53b86865 100644
--- a/source/de/anomic/server/logging/GuiHandler.java
+++ b/source/de/anomic/server/logging/GuiHandler.java
@@ -201,7 +201,7 @@ public class GuiHandler extends Handler{
for (int i = 0; i < lineCount; i++) {
int ix = (reversed) ?
Math.abs((start-i)%this.buffer.length) :
- (start+i)%this.buffer.length;
+ (start - lineCount + i) % this.buffer.length;
record = this.buffer[ix];
logMessages.add(logFormatter.format(record));
}
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java
index 67ef67438..6da6f90a7 100644
--- a/source/de/anomic/yacy/yacyClient.java
+++ b/source/de/anomic/yacy/yacyClient.java
@@ -364,7 +364,7 @@ public final class yacyClient {
}
}
- public static int search(
+ public static String[] search(
String wordhashes,
String urlhashes,
String prefer,
@@ -454,12 +454,12 @@ public final class yacyClient {
} catch (IOException e) {
yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
yacyCore.peerActions.peerDeparture(targetPeer);
- return 0;
+ return null;
}
if (result.size() == 0) {
yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes));
- return 0;
+ return null;
}
// compute all computation times
@@ -497,6 +497,7 @@ public final class yacyClient {
// insert results to containers
plasmaCrawlLURL.Entry urlEntry;
+ String[] urls = new String[results];
for (int n = 0; n < results; n++) {
// get one single search result
urlEntry = urlManager.newEntry((String) result.get("resource" + n), true);
@@ -540,6 +541,8 @@ public final class yacyClient {
for (int m = 0; m < words; m++) {
container[m].add(new indexEntry[]{entry}, System.currentTimeMillis());
}
+ // store url hash for statistics
+ urls[n] = urlEntry.hash();
}
// insert the containers to the index
@@ -574,11 +577,11 @@ public final class yacyClient {
searchtime = totalrequesttime;
}
yacyCore.log.logFine("SEARCH " + results + " URLS FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes) + ", duetime=" + duetime + ", searchtime=" + searchtime + ", netdelay=" + (totalrequesttime - searchtime) + ", references=" + result.get("references"));
- return results;
+ return urls;
} catch (Exception e) {
yacyCore.log.logSevere("yacyClient.search error: '" + targetPeer.get(yacySeed.NAME, "anonymous") + "' failed - " + e);
e.printStackTrace();
- return 0;
+ return null;
}
}
diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java
index 27fa4ce07..2cfb7ac83 100644
--- a/source/de/anomic/yacy/yacySearch.java
+++ b/source/de/anomic/yacy/yacySearch.java
@@ -70,7 +70,7 @@ public class yacySearch extends Thread {
final private plasmaURLPattern blacklist;
final private plasmaSnippetCache snippetCache;
final private yacySeed targetPeer;
- private int links;
+ private String[] urls;
private int maxDistance;
final private plasmaSearchTimingProfile timingProfile;
final private plasmaSearchRankingProfile rankingProfile;
@@ -94,18 +94,20 @@ public class yacySearch extends Thread {
this.blacklist = blacklist;
this.snippetCache = snippetCache;
this.targetPeer = targetPeer;
- this.links = -1;
+ this.urls = null;
this.maxDistance = maxDistance;
this.timingProfile = (plasmaSearchTimingProfile) timingProfile.clone();
this.rankingProfile = rankingProfile;
}
public void run() {
- this.links = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
- yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes);
- if (links != 0) {
- yacyCore.seedDB.mySeed.incRI(links);
- yacyCore.seedDB.mySeed.incRU(links);
+ this.urls = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile);
+ StringBuffer urllist = new StringBuffer(this.urls.length * 13);
+ for (int i = 0; i < this.urls.length; i++) urllist.append(this.urls[i]).append(' ');
+ yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + urls.length + " links for word hash " + wordhashes + ": " + new String(urllist));
+ if (urls != null) {
+ yacyCore.seedDB.mySeed.incRI(urls.length);
+ yacyCore.seedDB.mySeed.incRU(urls.length);
}
}
@@ -117,7 +119,7 @@ public class yacySearch extends Thread {
}
public int links() {
- return this.links;
+ return this.urls.length;
}
public plasmaSearchTimingProfile timingProfile() {
@@ -241,7 +243,7 @@ public class yacySearch extends Thread {
public static int collectedLinks(yacySearch[] searchThreads) {
int links = 0;
for (int i = 0; i < searchThreads.length; i++) {
- if (!(searchThreads[i].isAlive())) links += searchThreads[i].links;
+ if (!(searchThreads[i].isAlive())) links += searchThreads[i].urls.length;
}
return links;
}