From da621e827e53a2bfed4cf6135e3941a1b55d1021 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 28 May 2013 16:26:38 +0200 Subject: [PATCH] prevent NPE in case RWI is disabled --- htroot/PerformanceQueues_p.java | 15 ++++--- htroot/yacy/query.java | 2 +- htroot/yacy/search.java | 2 +- htroot/yacysearch.java | 2 +- source/net/yacy/peers/Dispatcher.java | 6 +-- source/net/yacy/peers/Protocol.java | 39 ++++++++++++++++++- source/net/yacy/search/index/Segment.java | 2 +- source/net/yacy/search/query/SearchEvent.java | 6 ++- .../net/yacy/search/snippet/ResultEntry.java | 2 +- 9 files changed, 59 insertions(+), 17 deletions(-) diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index df8948a2f..0a6cc2a49 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -31,6 +31,8 @@ import java.util.Map; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.kelondro.data.word.WordReference; +import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; @@ -132,6 +134,7 @@ public class PerformanceQueues_p { sb.setConfig("performanceSpeed", post.getInt("profileSpeed", 100)); } + IndexCell rwi = indexSegment.termIndex(); while (threads.hasNext()) { threadName = threads.next(); thread = sb.getThread(threadName); @@ -215,7 +218,7 @@ public class PerformanceQueues_p { // prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ "0"); prop.put("table_" + c + "_disabled", threadName.equals("10_httpd") ? "1" : "0" ); // httpd hardcoded defaults prop.put("table_" + c + "_recommendation", threadName.endsWith("_indexing") ? "1" : "0"); - prop.putNum("table_" + c + "_recommendation_value", threadName.endsWith("_indexing") ? (indexSegment.termIndex().minMem() / 1024) : 0); + prop.putNum("table_" + c + "_recommendation_value", rwi == null ? 0 : threadName.endsWith("_indexing") ? (rwi.minMem() / 1024) : 0); c++; } prop.put("table", c); @@ -245,7 +248,7 @@ public class PerformanceQueues_p { if ((post != null) && (post.containsKey("cacheSizeSubmit"))) { final int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000); sb.setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount)); - indexSegment.termIndex().setBufferMaxWordCount(wordCacheMaxCount); + if (rwi != null) rwi.setBufferMaxWordCount(wordCacheMaxCount); } if ((post != null) && (post.containsKey("poolConfig"))) { @@ -301,10 +304,10 @@ public class PerformanceQueues_p { // table cache settings prop.putNum("wordCacheSize", indexSegment.RWIBufferCount()); - prop.putNum("wordCacheSizeKBytes", indexSegment.termIndex().getBufferSizeBytes()/1024); - prop.putNum("maxURLinCache", indexSegment.termIndex().getBufferMaxReferences()); - prop.putNum("maxAgeOfCache", indexSegment.termIndex().getBufferMaxAge() / 1000 / 60); // minutes - prop.putNum("minAgeOfCache", indexSegment.termIndex().getBufferMinAge() / 1000 / 60); // minutes + prop.putNum("wordCacheSizeKBytes", rwi == null ? 0 : rwi.getBufferSizeBytes()/1024); + prop.putNum("maxURLinCache", rwi == null ? 0 : rwi.getBufferMaxReferences()); + prop.putNum("maxAgeOfCache", rwi == null ? 0 : rwi.getBufferMaxAge() / 1000 / 60); // minutes + prop.putNum("minAgeOfCache", rwi == null ? 0 : rwi.getBufferMinAge() / 1000 / 60); // minutes prop.putNum("maxWaitingWordFlush", sb.getConfigLong("maxWaitingWordFlush", 180)); prop.put("wordCacheMaxCount", sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000)); prop.put("crawlPauseProxy", sb.getConfigLong(SwitchboardConstants.PROXY_ONLINE_CAUTION_DELAY, 30000)); diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index c91c43007..35ce03125 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -95,7 +95,7 @@ public final class query { if (obj.equals("rwiurlcount")) try { // the total number of different urls in the rwi is returned // shall contain a word hash, the number of assigned lurls to this hash is returned - prop.put("response", sb.index.termIndex().get(env.getBytes(), null).size()); + prop.put("response", sb.index.termIndex() == null ? 0 : sb.index.termIndex().get(env.getBytes(), null).size()); return prop; } catch (final IOException e) { Log.logException(e); diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 418b5d6f3..148499108 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -258,7 +258,7 @@ public final class search { final long timer = System.currentTimeMillis(); //final Map>[] containers = sb.indexSegment.index().searchTerm(theQuery.queryHashes, theQuery.excludeHashes, plasmaSearchQuery.hashes2StringSet(urls)); - final TreeMap> incc = indexSegment.termIndex().searchConjunction(theQuery.getQueryGoal().getIncludeHashes(), QueryParams.hashes2Set(urls)); + final TreeMap> incc = indexSegment.termIndex() == null ? new TreeMap>() : indexSegment.termIndex().searchConjunction(theQuery.getQueryGoal().getIncludeHashes(), QueryParams.hashes2Set(urls)); EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true), SearchEventType.COLLECTION, "", incc.size(), System.currentTimeMillis() - timer), false); if (incc != null) { diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 1a53ed5b6..bf9b29e16 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -540,7 +540,7 @@ public class yacysearch { // delete the index entry locally final String delHash = post.get("deleteref", ""); // urlhash - indexSegment.termIndex().remove(qg.getIncludeHashes(), delHash.getBytes()); + if (indexSegment.termIndex() != null) indexSegment.termIndex().remove(qg.getIncludeHashes(), delHash.getBytes()); // make new news message with negative voting if ( !sb.isRobinsonMode() ) { diff --git a/source/net/yacy/peers/Dispatcher.java b/source/net/yacy/peers/Dispatcher.java index 0897efbab..242052994 100644 --- a/source/net/yacy/peers/Dispatcher.java +++ b/source/net/yacy/peers/Dispatcher.java @@ -169,7 +169,7 @@ public class Dispatcher { final ArrayList> containers = new ArrayList>(maxContainerCount); - final Iterator> indexContainerIterator = this.segment.termIndex().referenceContainerIterator(hash, true, true, ram); // very important that rotation is true here + final Iterator> indexContainerIterator = this.segment.termIndex() == null ? new ArrayList>().iterator() : this.segment.termIndex().referenceContainerIterator(hash, true, true, ram); // very important that rotation is true here ReferenceContainer container; int refcount = 0; @@ -201,7 +201,7 @@ public class Dispatcher { it = c.entries(); while (it.hasNext()) try { urlHashes.put(it.next().urlhash()); } catch (final SpaceExceededException e) { Log.logException(e); } if (this.log.isFine()) this.log.logFine("selected " + urlHashes.size() + " urls for word '" + ASCII.String(c.getTermHash()) + "'"); - if (!urlHashes.isEmpty()) this.segment.termIndex().remove(c.getTermHash(), urlHashes); + if (this.segment.termIndex() != null && !urlHashes.isEmpty()) this.segment.termIndex().remove(c.getTermHash(), urlHashes); } rc = containers; } else { @@ -209,7 +209,7 @@ public class Dispatcher { // but to avoid race conditions return the results from the deletes rc = new ArrayList>(containers.size()); for (final ReferenceContainer c: containers) { - container = this.segment.termIndex().remove(c.getTermHash()); // be aware this might be null! + container = this.segment.termIndex() == null ? null : this.segment.termIndex().remove(c.getTermHash()); // be aware this might be null! if (container != null && !container.isEmpty()) { if (this.log.isFine()) this.log.logFine("selected " + container.size() + " urls for word '" + ASCII.String(c.getTermHash()) + "'"); rc.add(container); diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 957422471..50e5be0f7 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -394,7 +394,44 @@ public final class Protocol { return count; } - + /* + private int readSeeds(String prefix) { + String seedStr; + while ( (seedStr = result.get("seed" + i++)) != null ) { + // integrate new seed into own database + // the first seed, "seed0" is the seed of the responding peer + if ( seedStr.length() > Seed.maxsize ) { + Network.log.logInfo("hello/client: rejected contacting seed; too large (" + + seedStr.length() + + " > " + + Seed.maxsize + + ")"); + } else { + try { + if ( i == 1 ) { + final int p = address.indexOf(':'); + if ( p < 0 ) { + return -1; + } + InetAddress ia = Domains.dnsResolve(address.substring(0, p)); + if (ia == null) continue; + final String host = ia.getHostAddress(); + s = Seed.genRemoteSeed(seedStr, false, host); + } else { + s = Seed.genRemoteSeed(seedStr, false, null); + } + if ( peerActions.peerArrival(s, (i == 1)) ) { + count++; + } + } catch ( final IOException e ) { + Network.log.logInfo("hello/client: rejected contacting seed; bad (" + + e.getMessage() + + ")"); + } + } + } + } +*/ public static Seed querySeed(final Seed target, final String seedHash) { // prepare request final String salt = crypt.randomSalt(); diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index d32b2e6e1..8d534e1f5 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -713,7 +713,7 @@ public class Segment { // delete all word references int count = 0; - if (words != null) count = termIndex().remove(Word.words2hashesHandles(words), urlhash); + if (words != null && termIndex() != null) count = termIndex().remove(Word.words2hashesHandles(words), urlhash); // finally delete the url entry itself fulltext().remove(urlhash); diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 463b8a954..adc8017fc 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -1026,8 +1026,10 @@ public final class SearchEvent { // check index-of constraint if ((this.query.constraint != null) && (this.query.constraint.get(Condenser.flag_cat_indexof)) && (!(pagetitle.startsWith("index of")))) { final Iterator wi = this.query.getQueryGoal().getIncludeHashes().iterator(); - while (wi.hasNext()) { - this.query.getSegment().termIndex().removeDelayed(wi.next(), page.hash()); + if (this.query.getSegment().termIndex() != null) { + while (wi.hasNext()) { + this.query.getSegment().termIndex().removeDelayed(wi.next(), page.hash()); + } } if (log.isFine()) log.logFine("dropped RWI: url does not match index-of constraint"); if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet(); diff --git a/source/net/yacy/search/snippet/ResultEntry.java b/source/net/yacy/search/snippet/ResultEntry.java index b49e1eb6b..9fd547d49 100644 --- a/source/net/yacy/search/snippet/ResultEntry.java +++ b/source/net/yacy/search/snippet/ResultEntry.java @@ -86,7 +86,7 @@ public class ResultEntry implements Comparable, Comparator