boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git://github.com/f1ori/yacy

Browse Source
pull/1/head
admin 13 years ago
parent 1b58bcfc34 c9216d5adf
commit 066eee883e
5 changed files with 53 additions and 64 deletions
Show Stats Download Patch File Download Diff File

3
htroot/yacy/search.java
Unescape View File

@ -38,7 +38,6 @@ import java.util.regex.Pattern;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
@ -333,7 +332,7 @@ public final class search {
final Iterator<Map.Entry<byte[], Integer>> i = theSearch.abstractsCount();
while (i.hasNext()) {
entry = i.next();
indexcount.append("indexcount.").append(UTF8.String(entry.getKey())).append('=').append((entry.getValue()).toString()).append(serverCore.CRLF_STRING);
indexcount.append("indexcount.").append(ASCII.String(entry.getKey())).append('=').append((entry.getValue()).toString()).append(serverCore.CRLF_STRING);
}
if (abstractSet != null) {
// if a specific index-abstract is demanded, attach it here

14
source/net/yacy/kelondro/util/ByteBuffer.java
Unescape View File

@ -415,7 +415,7 @@ public final class ByteBuffer extends OutputStream {
final StringBuilder sb = new StringBuilder(sblength);
int i = 0;
sb.setLength(length);
for (int j = left; j < left + length; j++) sb.setCharAt(i++, (char) this.buffer[j]);
for (int j = left; j < left + length; j++) sb.setCharAt(i++, (char) this.buffer[this.offset + j]);
return sb;
}
@ -506,18 +506,6 @@ public final class ByteBuffer extends OutputStream {
this.offset = 0;
}
public void reset(final int newSize) {
resize(newSize);
this.reset();
}
public void resize(final int newSize) {
if(newSize < 0) throw new IllegalArgumentException("Illegal array size: " + newSize);
final byte[] v = new byte[newSize];
System.arraycopy(this.buffer,0,v,0,newSize > this.buffer.length ? this.buffer.length : newSize);
this.buffer = v;
}
public void writeTo(final OutputStream dest) throws IOException {
dest.write(this.buffer, this.offset, this.length);
dest.flush();

7
source/net/yacy/peers/Protocol.java
Unescape View File

@ -482,6 +482,9 @@ public final class Protocol {
// computation time
final long totalrequesttime = System.currentTimeMillis() - timestamp;
final boolean thisIsASecondarySearch = urlhashes.length() > 0;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;
// create containers
final int words = wordhashes.length() / Word.commonHashLength;
assert words > 0 : "wordhashes = " + wordhashes;
@ -568,8 +571,6 @@ public final class Protocol {
} catch (final Exception e) {
Log.logException(e);
}
final boolean thisIsASecondarySearch = urlhashes.length() > 0;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;
Network.log.logInfo("remote search: peer " + target.getName() + " sent " + container[0].size() + "/" + result.joincount + " references for " + (thisIsASecondarySearch ? "a secondary search" : "joined word queries"));
@ -651,7 +652,7 @@ public final class Protocol {
final boolean global,
final int partitions,
final String hostname,
String hostaddress,
final String hostaddress,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final RankingProfile rankingProfile,
final Bitfield constraint) throws IOException {

38
source/net/yacy/peers/RemoteSearch.java
Unescape View File

@ -64,7 +64,7 @@ public class RemoteSearch extends Thread {
public RemoteSearch(
final String wordhashes, final String excludehashes,
final String urlhashes,
final String urlhashes, // this is the field that is filled during a secondary search to restrict to specific urls that are to be retrieved
final Pattern prefer,
final Pattern filter,
final Pattern snippet,
@ -112,30 +112,30 @@ public class RemoteSearch extends Thread {
public void run() {
try {
this.urls = Protocol.search(
peers.mySeed(),
wordhashes, excludehashes, urlhashes,
prefer, filter, snippet,
language, sitehash, authorhash,
count, time, maxDistance, global, partitions,
targetPeer, indexSegment, containerCache, secondarySearchSuperviser,
blacklist, rankingProfile, constraint);
if (urls >= 0) {
this.peers.mySeed(),
this.wordhashes, this.excludehashes, this.urlhashes,
this.prefer, this.filter, this.snippet,
this.language, this.sitehash, this.authorhash,
this.count, this.time, this.maxDistance, this.global, this.partitions,
this.targetPeer, this.indexSegment, this.containerCache, this.secondarySearchSuperviser,
this.blacklist, this.rankingProfile, this.constraint);
if (this.urls >= 0) {
// urls is an array of url hashes. this is only used for log output
if (urlhashes != null && urlhashes.length() > 0) Network.log.logInfo("SECONDARY REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + this.urls + " links for word hash " + wordhashes);
peers.mySeed().incRI(urls);
peers.mySeed().incRU(urls);
if (this.urlhashes != null && this.urlhashes.length() > 0) Network.log.logInfo("SECONDARY REMOTE SEARCH - remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName() + " contributed " + this.urls + " links for word hash " + this.wordhashes);
this.peers.mySeed().incRI(this.urls);
this.peers.mySeed().incRU(this.urls);
} else {
Network.log.logInfo("REMOTE SEARCH - no answer from remote peer " + targetPeer.hash + ":" + targetPeer.getName());
Network.log.logInfo("REMOTE SEARCH - no answer from remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName());
}
} catch (final Exception e) {
Log.logException(e);
} finally {
containerCache.oneFeederTerminated();
this.containerCache.oneFeederTerminated();
}
}
public static String set2string(final HandleSet hashes) {
StringBuilder wh = new StringBuilder(hashes.size() * 12);
final StringBuilder wh = new StringBuilder(hashes.size() * 12);
final Iterator<byte[]> iter = hashes.iterator();
while (iter.hasNext()) { wh.append(ASCII.String(iter.next())); }
return wh.toString();
@ -150,7 +150,7 @@ public class RemoteSearch extends Thread {
}
public Seed target() {
return targetPeer;
return this.targetPeer;
}
public static RemoteSearch[] primaryRemoteSearches(
@ -159,7 +159,7 @@ public class RemoteSearch extends Thread {
final String language,
final String sitehash,
final String authorhash,
final int count, long time, final int maxDist,
final int count, final long time, final int maxDist,
final Segment indexSegment,
final SeedDB peers,
final RWIProcess containerCache,
@ -186,7 +186,7 @@ public class RemoteSearch extends Thread {
burstMultiwordPercent)
: PeerSelection.selectClusterPeers(peers, clusterselection);
if (targetPeers == null) return new RemoteSearch[0];
int targets = targetPeers.length;
final int targets = targetPeers.length;
if (targets == 0) return new RemoteSearch[0];
final RemoteSearch[] searchThreads = new RemoteSearch[targets];
for (int i = 0; i < targets; i++) {
@ -198,7 +198,7 @@ public class RemoteSearch extends Thread {
count, time, maxDist, true, targets, targetPeers[i],
indexSegment, peers, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint);
searchThreads[i].start();
} catch (OutOfMemoryError e) {
} catch (final OutOfMemoryError e) {
Log.logException(e);
break;
}

21
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -383,11 +383,11 @@ public final class SearchEvent {
return this.resultFetcher.oneResult(item, timeout);
}
boolean secondarySearchStartet = false;
//boolean secondarySearchStartet = false;
public static class HeuristicResult /*implements Comparable<HeuristicResult>*/ {
public final byte[] urlhash; public final String heuristicName; public final boolean redundant;
public HeuristicResult(final byte[] urlhash, final String heuristicName, final boolean redundant) {
private final byte[] urlhash; public final String heuristicName; public final boolean redundant;
private HeuristicResult(final byte[] urlhash, final String heuristicName, final boolean redundant) {
this.urlhash = urlhash; this.heuristicName = heuristicName; this.redundant = redundant;
}/*
public int compareTo(HeuristicResult o) {
@ -405,9 +405,9 @@ public final class SearchEvent {
// cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
// this relation contains the information where specific urls can be found in specific peers
SortedMap<String, SortedMap<String, StringBuilder>> abstractsCache;
SortedSet<String> checkedPeers;
Semaphore trigger;
private final SortedMap<String, SortedMap<String, StringBuilder>> abstractsCache;
private final SortedSet<String> checkedPeers;
private final Semaphore trigger;
public SecondarySearchSuperviser() {
this.abstractsCache = new TreeMap<String, SortedMap<String, StringBuilder>>();
@ -491,7 +491,7 @@ public final class SearchEvent {
}
} catch (final InterruptedException e) {
// the thread was interrupted
// do nohing
// do nothing
}
// the time-out was reached
}
@ -500,10 +500,11 @@ public final class SearchEvent {
if (this.abstractsCache == null || this.abstractsCache.size() != SearchEvent.this.query.queryHashes.size()) return; // secondary search not possible (yet)
// catch up index abstracts and join them; then call peers again to submit their urls
/*
System.out.println("DEBUG-INDEXABSTRACT: " + abstractsCache.size() + " word references caught, " + query.queryHashes.size() + " needed");
for (Map.Entry<String, TreeMap<String, String>> entry: abstractsCache.entrySet()) {
System.out.println("DEBUG-INDEXABSTRACT: hash " + entry.getKey() + ": " + ((query.queryHashes.has(entry.getKey().getBytes()) ? "NEEDED" : "NOT NEEDED") + "; " + entry.getValue().size() + " entries"));
System.out.println("DEBUG-INDEXABSTRACT: " + this.abstractsCache.size() + " word references caught, " + SearchEvent.this.query.queryHashes.size() + " needed");
for (final Map.Entry<String, SortedMap<String, StringBuilder>> entry: this.abstractsCache.entrySet()) {
System.out.println("DEBUG-INDEXABSTRACT: hash " + entry.getKey() + ": " + ((SearchEvent.this.query.queryHashes.has(entry.getKey().getBytes()) ? "NEEDED" : "NOT NEEDED") + "; " + entry.getValue().size() + " entries"));
}
*/

Write Preview
Loading…
Cancel
Save