boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

enhanced secondary search: index abstracts decompression is now much faster and does not cause strong CPU load after several searches with more than one word

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7565 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent bf27a72d53
commit 619b561a4a
6 changed files with 56 additions and 29 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/http/server/ServerSideIncludes.java
Unescape View File

@ -67,7 +67,7 @@ public class ServerSideIncludes {
if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) {
final int q = in.indexOf("\"".getBytes(), off + 22);
if (q > 0) {
final String path = in.toString(off + 22, q);
final String path = in.toString(off + 22, q - off - 22);
writeContent(path, out, authorization, requesthost);
}
}

32
source/de/anomic/search/SearchEvent.java
Unescape View File

@ -377,12 +377,12 @@ public final class SearchEvent {
// cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
// this relation contains the information where specific urls can be found in specific peers
SortedMap<String, SortedMap<String, String>> abstractsCache;
SortedMap<String, SortedMap<String, StringBuilder>> abstractsCache;
SortedSet<String> checkedPeers;
Semaphore trigger;
public SecondarySearchSuperviser() {
this.abstractsCache = new TreeMap<String, SortedMap<String, String>>();
this.abstractsCache = new TreeMap<String, SortedMap<String, StringBuilder>>();
this.checkedPeers = new TreeSet<String>();
this.trigger = new Semaphore(0);
}
@ -392,8 +392,8 @@ public final class SearchEvent {
* @param wordhash
* @param singleAbstract // a mapping from url-hashes to a string of peer-hashes
*/
public void addAbstract(String wordhash, final TreeMap<String, String> singleAbstract) {
final SortedMap<String, String> oldAbstract;
public void addAbstract(String wordhash, final TreeMap<String, StringBuilder> singleAbstract) {
final SortedMap<String, StringBuilder> oldAbstract;
synchronized (abstractsCache) {
oldAbstract = abstractsCache.get(wordhash);
if (oldAbstract == null) {
@ -405,12 +405,12 @@ public final class SearchEvent {
// extend the abstracts in the cache: join the single abstracts
new Thread() {
public void run() {
for (final Map.Entry<String, String> oneref: singleAbstract.entrySet()) {
for (final Map.Entry<String, StringBuilder> oneref: singleAbstract.entrySet()) {
final String urlhash = oneref.getKey();
final String peerlistNew = oneref.getValue();
final StringBuilder peerlistNew = oneref.getValue();
synchronized (oldAbstract) {
final String peerlistOld = oldAbstract.put(urlhash, peerlistNew);
if (peerlistOld != null) oldAbstract.put(urlhash, peerlistOld + peerlistNew);
final StringBuilder peerlistOld = oldAbstract.put(urlhash, peerlistNew);
if (peerlistOld != null) peerlistOld.append(peerlistNew);
}
}
}
@ -423,13 +423,14 @@ public final class SearchEvent {
}
private String wordsFromPeer(final String peerhash, final String urls) {
Map.Entry<String, SortedMap<String, String>> entry;
String word, peerlist, url, wordlist = "";
SortedMap<String, String> urlPeerlist;
Map.Entry<String, SortedMap<String, StringBuilder>> entry;
String word, url, wordlist = "";
StringBuilder peerlist;
SortedMap<String, StringBuilder> urlPeerlist;
int p;
boolean hasURL;
synchronized (this) {
final Iterator<Map.Entry <String, SortedMap<String, String>>> i = this.abstractsCache.entrySet().iterator();
final Iterator<Map.Entry <String, SortedMap<String, StringBuilder>>> i = this.abstractsCache.entrySet().iterator();
while (i.hasNext()) {
entry = i.next();
word = entry.getKey();
@ -482,17 +483,18 @@ public final class SearchEvent {
if (abstractsCache.size() != query.queryHashes.size()) return;
// join all the urlhash:peerlist relations: the resulting map has values with a combined peer-list list
final SortedMap<String, String> abstractJoin = SetTools.joinConstructive(abstractsCache.values(), true);
final SortedMap<String, StringBuilder> abstractJoin = SetTools.joinConstructive(abstractsCache.values(), true);
if (abstractJoin.isEmpty()) return;
// the join result is now a urlhash: peer-list relation
// generate a list of peers that have the urls for the joined search result
final SortedMap<String, String> secondarySearchURLs = new TreeMap<String, String>(); // a (peerhash:urlhash-liststring) mapping
String url, urls, peer, peerlist;
String url, urls, peer;
StringBuilder peerlist;
final String mypeerhash = peers.mySeed().hash;
boolean mypeerinvolved = false;
int mypeercount;
for (Map.Entry<String, String> entry: abstractJoin.entrySet()) {
for (Map.Entry<String, StringBuilder> entry: abstractJoin.entrySet()) {
url = entry.getKey();
peerlist = entry.getValue();
//System.out.println("DEBUG-INDEXABSTRACT: url " + url + ": from peers " + peerlist);

2
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -222,7 +222,7 @@ public class yacySearch extends Thread {
if (targetPeer == null || targetPeer.hash == null) return null;
if (clusterselection != null) targetPeer.setAlternativeAddress(clusterselection.get(targetPeer.hash.getBytes()));
final yacySearch searchThread = new yacySearch(
wordhashes, "", urlhashes, Pattern.compile(""), Pattern.compile(".*"), "", "", "", 0, time, 9999, true, 0, targetPeer,
wordhashes, "", urlhashes, Pattern.compile(""), Pattern.compile(".*"), "", "", "", 20, time, 9999, true, 0, targetPeer,
indexSegment, peers, containerCache, null, blacklist, rankingProfile, constraint);
searchThread.start();
return searchThread;

25
source/net/yacy/kelondro/rwi/ReferenceContainer.java
Unescape View File

@ -556,23 +556,34 @@ public class ReferenceContainer<ReferenceType extends Reference> extends RowSet
return bb;
}
public static final TreeMap<String, String> decompressIndex(ByteBuffer ci, final String peerhash) {
TreeMap<String, String> target = new TreeMap<String, String>();
public static final TreeMap<String, StringBuilder> decompressIndex(ByteBuffer ci, final String peerhash) {
TreeMap<String, StringBuilder> target = new TreeMap<String, StringBuilder>();
// target is a mapping from url-hashes to a string of peer-hashes
if (ci.byteAt(0) != '{' || ci.byteAt(ci.length() - 1) != '}') return target;
//System.out.println("DEBUG-DECOMPRESS: input is " + ci.toString());
ci = ci.trim(1, ci.length() - 2);
String dom, url, peers;
String dom, url;
StringBuilder peers;
StringBuilder urlsb;
while ((ci.length() >= 13) && (ci.byteAt(6) == ':')) {
assert ci.length() >= 6 : "ci.length() = " + ci.length();
dom = ci.toString(0, 6);
dom = ci.toStringBuilder(0, 6, 6).toString();
ci.trim(7);
while ((ci.length() > 0) && (ci.byteAt(0) != ',')) {
assert ci.length() >= 6 : "ci.length() = " + ci.length();
url = ci.toString(0, 6) + dom;
urlsb = ci.toStringBuilder(0, 6, 12);
urlsb.append(dom);
url = urlsb.toString();
ci.trim(6);
peers = target.put(url, peerhash);
if (peers != null) target.put(url, peers + peerhash);
peers = target.get(url);
if (peers == null) {
peers = new StringBuilder(24);
peers.append(peerhash);
target.put(url, peers);
} else {
peers.append(peerhash);
}
//System.out.println("DEBUG-DECOMPRESS: " + url + ":" + target.get(url));
}
if (ci.byteAt(0) == ',') ci.trim(1);

16
source/net/yacy/kelondro/util/ByteBuffer.java
Unescape View File

@ -302,7 +302,8 @@ public final class ByteBuffer extends OutputStream {
}
public ByteBuffer trim(final int start) {
trim(start, this.length - start);
this.offset += start;
this.length -= start;
return this;
}
@ -402,8 +403,17 @@ public final class ByteBuffer extends OutputStream {
}
}
public String toString(final int left, final int rightbound) {
return UTF8.String(buffer, offset + left, rightbound - left);
public String toString(final int left, final int length) {
return UTF8.String(buffer, offset + left, length);
}
public StringBuilder toStringBuilder(final int left, final int length, final int sblength) {
assert sblength >= length;
final StringBuilder sb = new StringBuilder(sblength);
int i = 0;
sb.setLength(length);
for (int j = left; j < left + length; j++) sb.setCharAt(i++, (char) buffer[j]);
return sb;
}
public Properties propParser(final String charset) {

8
source/net/yacy/kelondro/util/SetTools.java
Unescape View File

@ -150,7 +150,9 @@ public final class SetTools {
}
if (mobj2 != null) {
if (mentry1.getValue() instanceof String) {
result.put(mentry1.getKey(), (B) ((concatStrings) ? (mentry1.getValue() + (String) mobj2) : mentry1.getValue()));
result.put(mentry1.getKey(), (B) ((concatStrings) ? (((String) mentry1.getValue()) + (String) mobj2) : (String) mentry1.getValue()));
} else if (mentry1.getValue() instanceof StringBuilder) {
result.put(mentry1.getKey(), (B) ((concatStrings) ? (((StringBuilder) mentry1.getValue()).append((StringBuilder) mobj2)) : mentry1.getValue()));
} else {
result.put(mentry1.getKey(), mentry1.getValue());
}
@ -183,7 +185,9 @@ public final class SetTools {
if (mi2.hasNext()) mentry2 = mi2.next(); else break;
} else {
if (mentry1.getValue() instanceof String) {
result.put(mentry1.getKey(), (B) ((concatStrings) ? ((String) mentry1.getValue() + (String) mentry2.getValue()) : (String) mentry1.getValue()));
result.put(mentry1.getKey(), (B) ((concatStrings) ? (((String) mentry1.getValue()) + (String) mentry2.getValue()) : (String) mentry1.getValue()));
} else if (mentry1.getValue() instanceof StringBuilder) {
result.put(mentry1.getKey(), (B) ((concatStrings) ? (((StringBuilder) mentry1.getValue()).append((StringBuilder) mentry2.getValue())) : (StringBuilder) mentry1.getValue()));
} else {
result.put(mentry1.getKey(), mentry1.getValue());
}

Write Preview
Loading…
Cancel
Save