Merge branch 'master' of git://github.com/f1ori/yacy

pull/1/head
admin 13 years ago
commit 066eee883e

@ -38,7 +38,6 @@ import java.util.regex.Pattern;
import net.yacy.cora.document.ASCII; import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSMessage;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
@ -333,7 +332,7 @@ public final class search {
final Iterator<Map.Entry<byte[], Integer>> i = theSearch.abstractsCount(); final Iterator<Map.Entry<byte[], Integer>> i = theSearch.abstractsCount();
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
indexcount.append("indexcount.").append(UTF8.String(entry.getKey())).append('=').append((entry.getValue()).toString()).append(serverCore.CRLF_STRING); indexcount.append("indexcount.").append(ASCII.String(entry.getKey())).append('=').append((entry.getValue()).toString()).append(serverCore.CRLF_STRING);
} }
if (abstractSet != null) { if (abstractSet != null) {
// if a specific index-abstract is demanded, attach it here // if a specific index-abstract is demanded, attach it here

@ -415,7 +415,7 @@ public final class ByteBuffer extends OutputStream {
final StringBuilder sb = new StringBuilder(sblength); final StringBuilder sb = new StringBuilder(sblength);
int i = 0; int i = 0;
sb.setLength(length); sb.setLength(length);
for (int j = left; j < left + length; j++) sb.setCharAt(i++, (char) this.buffer[j]); for (int j = left; j < left + length; j++) sb.setCharAt(i++, (char) this.buffer[this.offset + j]);
return sb; return sb;
} }
@ -506,18 +506,6 @@ public final class ByteBuffer extends OutputStream {
this.offset = 0; this.offset = 0;
} }
public void reset(final int newSize) {
resize(newSize);
this.reset();
}
public void resize(final int newSize) {
if(newSize < 0) throw new IllegalArgumentException("Illegal array size: " + newSize);
final byte[] v = new byte[newSize];
System.arraycopy(this.buffer,0,v,0,newSize > this.buffer.length ? this.buffer.length : newSize);
this.buffer = v;
}
public void writeTo(final OutputStream dest) throws IOException { public void writeTo(final OutputStream dest) throws IOException {
dest.write(this.buffer, this.offset, this.length); dest.write(this.buffer, this.offset, this.length);
dest.flush(); dest.flush();

@ -482,6 +482,9 @@ public final class Protocol {
// computation time // computation time
final long totalrequesttime = System.currentTimeMillis() - timestamp; final long totalrequesttime = System.currentTimeMillis() - timestamp;
final boolean thisIsASecondarySearch = urlhashes.length() > 0;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;
// create containers // create containers
final int words = wordhashes.length() / Word.commonHashLength; final int words = wordhashes.length() / Word.commonHashLength;
assert words > 0 : "wordhashes = " + wordhashes; assert words > 0 : "wordhashes = " + wordhashes;
@ -568,8 +571,6 @@ public final class Protocol {
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
} }
final boolean thisIsASecondarySearch = urlhashes.length() > 0;
assert !thisIsASecondarySearch || secondarySearchSuperviser == null;
Network.log.logInfo("remote search: peer " + target.getName() + " sent " + container[0].size() + "/" + result.joincount + " references for " + (thisIsASecondarySearch ? "a secondary search" : "joined word queries")); Network.log.logInfo("remote search: peer " + target.getName() + " sent " + container[0].size() + "/" + result.joincount + " references for " + (thisIsASecondarySearch ? "a secondary search" : "joined word queries"));
@ -651,7 +652,7 @@ public final class Protocol {
final boolean global, final boolean global,
final int partitions, final int partitions,
final String hostname, final String hostname,
String hostaddress, final String hostaddress,
final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser, final SearchEvent.SecondarySearchSuperviser secondarySearchSuperviser,
final RankingProfile rankingProfile, final RankingProfile rankingProfile,
final Bitfield constraint) throws IOException { final Bitfield constraint) throws IOException {

@ -64,7 +64,7 @@ public class RemoteSearch extends Thread {
public RemoteSearch( public RemoteSearch(
final String wordhashes, final String excludehashes, final String wordhashes, final String excludehashes,
final String urlhashes, final String urlhashes, // this is the field that is filled during a secondary search to restrict to specific urls that are to be retrieved
final Pattern prefer, final Pattern prefer,
final Pattern filter, final Pattern filter,
final Pattern snippet, final Pattern snippet,
@ -112,30 +112,30 @@ public class RemoteSearch extends Thread {
public void run() { public void run() {
try { try {
this.urls = Protocol.search( this.urls = Protocol.search(
peers.mySeed(), this.peers.mySeed(),
wordhashes, excludehashes, urlhashes, this.wordhashes, this.excludehashes, this.urlhashes,
prefer, filter, snippet, this.prefer, this.filter, this.snippet,
language, sitehash, authorhash, this.language, this.sitehash, this.authorhash,
count, time, maxDistance, global, partitions, this.count, this.time, this.maxDistance, this.global, this.partitions,
targetPeer, indexSegment, containerCache, secondarySearchSuperviser, this.targetPeer, this.indexSegment, this.containerCache, this.secondarySearchSuperviser,
blacklist, rankingProfile, constraint); this.blacklist, this.rankingProfile, this.constraint);
if (urls >= 0) { if (this.urls >= 0) {
// urls is an array of url hashes. this is only used for log output // urls is an array of url hashes. this is only used for log output
if (urlhashes != null && urlhashes.length() > 0) Network.log.logInfo("SECONDARY REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + this.urls + " links for word hash " + wordhashes); if (this.urlhashes != null && this.urlhashes.length() > 0) Network.log.logInfo("SECONDARY REMOTE SEARCH - remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName() + " contributed " + this.urls + " links for word hash " + this.wordhashes);
peers.mySeed().incRI(urls); this.peers.mySeed().incRI(this.urls);
peers.mySeed().incRU(urls); this.peers.mySeed().incRU(this.urls);
} else { } else {
Network.log.logInfo("REMOTE SEARCH - no answer from remote peer " + targetPeer.hash + ":" + targetPeer.getName()); Network.log.logInfo("REMOTE SEARCH - no answer from remote peer " + this.targetPeer.hash + ":" + this.targetPeer.getName());
} }
} catch (final Exception e) { } catch (final Exception e) {
Log.logException(e); Log.logException(e);
} finally { } finally {
containerCache.oneFeederTerminated(); this.containerCache.oneFeederTerminated();
} }
} }
public static String set2string(final HandleSet hashes) { public static String set2string(final HandleSet hashes) {
StringBuilder wh = new StringBuilder(hashes.size() * 12); final StringBuilder wh = new StringBuilder(hashes.size() * 12);
final Iterator<byte[]> iter = hashes.iterator(); final Iterator<byte[]> iter = hashes.iterator();
while (iter.hasNext()) { wh.append(ASCII.String(iter.next())); } while (iter.hasNext()) { wh.append(ASCII.String(iter.next())); }
return wh.toString(); return wh.toString();
@ -150,7 +150,7 @@ public class RemoteSearch extends Thread {
} }
public Seed target() { public Seed target() {
return targetPeer; return this.targetPeer;
} }
public static RemoteSearch[] primaryRemoteSearches( public static RemoteSearch[] primaryRemoteSearches(
@ -159,7 +159,7 @@ public class RemoteSearch extends Thread {
final String language, final String language,
final String sitehash, final String sitehash,
final String authorhash, final String authorhash,
final int count, long time, final int maxDist, final int count, final long time, final int maxDist,
final Segment indexSegment, final Segment indexSegment,
final SeedDB peers, final SeedDB peers,
final RWIProcess containerCache, final RWIProcess containerCache,
@ -186,7 +186,7 @@ public class RemoteSearch extends Thread {
burstMultiwordPercent) burstMultiwordPercent)
: PeerSelection.selectClusterPeers(peers, clusterselection); : PeerSelection.selectClusterPeers(peers, clusterselection);
if (targetPeers == null) return new RemoteSearch[0]; if (targetPeers == null) return new RemoteSearch[0];
int targets = targetPeers.length; final int targets = targetPeers.length;
if (targets == 0) return new RemoteSearch[0]; if (targets == 0) return new RemoteSearch[0];
final RemoteSearch[] searchThreads = new RemoteSearch[targets]; final RemoteSearch[] searchThreads = new RemoteSearch[targets];
for (int i = 0; i < targets; i++) { for (int i = 0; i < targets; i++) {
@ -198,7 +198,7 @@ public class RemoteSearch extends Thread {
count, time, maxDist, true, targets, targetPeers[i], count, time, maxDist, true, targets, targetPeers[i],
indexSegment, peers, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint); indexSegment, peers, containerCache, secondarySearchSuperviser, blacklist, rankingProfile, constraint);
searchThreads[i].start(); searchThreads[i].start();
} catch (OutOfMemoryError e) { } catch (final OutOfMemoryError e) {
Log.logException(e); Log.logException(e);
break; break;
} }

@ -383,11 +383,11 @@ public final class SearchEvent {
return this.resultFetcher.oneResult(item, timeout); return this.resultFetcher.oneResult(item, timeout);
} }
boolean secondarySearchStartet = false; //boolean secondarySearchStartet = false;
public static class HeuristicResult /*implements Comparable<HeuristicResult>*/ { public static class HeuristicResult /*implements Comparable<HeuristicResult>*/ {
public final byte[] urlhash; public final String heuristicName; public final boolean redundant; private final byte[] urlhash; public final String heuristicName; public final boolean redundant;
public HeuristicResult(final byte[] urlhash, final String heuristicName, final boolean redundant) { private HeuristicResult(final byte[] urlhash, final String heuristicName, final boolean redundant) {
this.urlhash = urlhash; this.heuristicName = heuristicName; this.redundant = redundant; this.urlhash = urlhash; this.heuristicName = heuristicName; this.redundant = redundant;
}/* }/*
public int compareTo(HeuristicResult o) { public int compareTo(HeuristicResult o) {
@ -405,9 +405,9 @@ public final class SearchEvent {
// cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
// this relation contains the information where specific urls can be found in specific peers // this relation contains the information where specific urls can be found in specific peers
SortedMap<String, SortedMap<String, StringBuilder>> abstractsCache; private final SortedMap<String, SortedMap<String, StringBuilder>> abstractsCache;
SortedSet<String> checkedPeers; private final SortedSet<String> checkedPeers;
Semaphore trigger; private final Semaphore trigger;
public SecondarySearchSuperviser() { public SecondarySearchSuperviser() {
this.abstractsCache = new TreeMap<String, SortedMap<String, StringBuilder>>(); this.abstractsCache = new TreeMap<String, SortedMap<String, StringBuilder>>();
@ -491,7 +491,7 @@ public final class SearchEvent {
} }
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
// the thread was interrupted // the thread was interrupted
// do nohing // do nothing
} }
// the time-out was reached // the time-out was reached
} }
@ -500,10 +500,11 @@ public final class SearchEvent {
if (this.abstractsCache == null || this.abstractsCache.size() != SearchEvent.this.query.queryHashes.size()) return; // secondary search not possible (yet) if (this.abstractsCache == null || this.abstractsCache.size() != SearchEvent.this.query.queryHashes.size()) return; // secondary search not possible (yet)
// catch up index abstracts and join them; then call peers again to submit their urls // catch up index abstracts and join them; then call peers again to submit their urls
/* /*
System.out.println("DEBUG-INDEXABSTRACT: " + abstractsCache.size() + " word references caught, " + query.queryHashes.size() + " needed"); System.out.println("DEBUG-INDEXABSTRACT: " + this.abstractsCache.size() + " word references caught, " + SearchEvent.this.query.queryHashes.size() + " needed");
for (Map.Entry<String, TreeMap<String, String>> entry: abstractsCache.entrySet()) { for (final Map.Entry<String, SortedMap<String, StringBuilder>> entry: this.abstractsCache.entrySet()) {
System.out.println("DEBUG-INDEXABSTRACT: hash " + entry.getKey() + ": " + ((query.queryHashes.has(entry.getKey().getBytes()) ? "NEEDED" : "NOT NEEDED") + "; " + entry.getValue().size() + " entries")); System.out.println("DEBUG-INDEXABSTRACT: hash " + entry.getKey() + ": " + ((SearchEvent.this.query.queryHashes.has(entry.getKey().getBytes()) ? "NEEDED" : "NOT NEEDED") + "; " + entry.getValue().size() + " entries"));
} }
*/ */

Loading…
Cancel
Save