From 6e2907135ab45e65fd2b62c65337d8b0732a58c7 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 13 Sep 2006 22:19:34 +0000 Subject: [PATCH] bugfixes for remote search server part git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2573 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacy/search.java | 48 ++++++++++--------- source/de/anomic/index/indexRAMCacheRI.java | 7 ++- .../de/anomic/plasma/plasmaSearchEvent.java | 2 +- source/de/anomic/yacy/yacyClient.java | 13 +++-- source/de/anomic/yacy/yacySearch.java | 3 +- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 82446c522..d586e8356 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -127,9 +127,12 @@ public final class search { Map containers = theSearch.localSearchContainers(plasmaSearchQuery.hashes2Set(urls)); // set statistic details of search result and find best result index set - String maxcounthash = null, neardhthash = null; + int joincount = 0; + plasmaSearchResult acc = null; if (containers == null) { - prop.put("indexcount", ""); + prop.put("indexcount", "0"); + prop.put("joincount", "0"); + prop.put("indexabstract",""); } else { Iterator ci = containers.entrySet().iterator(); StringBuffer indexcount = new StringBuffer(); @@ -137,6 +140,7 @@ public final class search { int maxcount = -1; double mindhtdistance = 1.1, d; String wordhash; + String maxcounthash = null, neardhthash = null; while (ci.hasNext()) { entry = (Map.Entry) ci.next(); wordhash = (String) entry.getKey(); @@ -153,27 +157,27 @@ public final class search { indexcount.append("indexcount.").append(container.getWordHash()).append('=').append(Integer.toString(container.size())).append(serverCore.crlfString); } prop.put("indexcount", new String(indexcount)); - } - - // join and order the result - indexContainer localResults = theSearch.localSearchJoin(containers.values()); - int joincount = localResults.size(); - prop.put("joincount", Integer.toString(joincount)); - plasmaSearchResult acc = theSearch.orderFinal(localResults); - - // generate compressed index for maxcounthash - // this is not needed if the search is restricted to specific urls, because it is a re-search - if ((maxcounthash == null) || (urls.length() != 0) || (keyhashes.size() == 1)) { - prop.put("indexabstract",""); - } else { - String indexabstract = "indexabstract." + maxcounthash + "=" + indexURL.compressIndex(((indexContainer) containers.get(maxcounthash)), localResults, 1000).toString() + serverCore.crlfString; - if ((neardhthash != null) && (!(neardhthash.equals(maxcounthash)))) { - indexabstract += "indexabstract." + neardhthash + "=" + indexURL.compressIndex(((indexContainer) containers.get(neardhthash)), localResults, 1000).toString() + serverCore.crlfString; + + // join and order the result + indexContainer localResults = theSearch.localSearchJoin(containers.values()); + joincount = localResults.size(); + prop.put("joincount", Integer.toString(joincount)); + acc = theSearch.orderFinal(localResults); + + // generate compressed index for maxcounthash + // this is not needed if the search is restricted to specific urls, because it is a re-search + if ((maxcounthash == null) || (urls.length() != 0) || (keyhashes.size() == 1)) { + prop.put("indexabstract",""); + } else { + String indexabstract = "indexabstract." + maxcounthash + "=" + indexURL.compressIndex(((indexContainer) containers.get(maxcounthash)), localResults, 1000).toString() + serverCore.crlfString; + if ((neardhthash != null) && (!(neardhthash.equals(maxcounthash)))) { + indexabstract += "indexabstract." + neardhthash + "=" + indexURL.compressIndex(((indexContainer) containers.get(neardhthash)), localResults, 1000).toString() + serverCore.crlfString; + } + System.out.println("DEBUG-ABSTRACTGENERATION: maxcounthash = " + maxcounthash); + System.out.println("DEBUG-ABSTRACTGENERATION: neardhthash = " + neardhthash); + //yacyCore.log.logFine("DEBUG HASH SEARCH: " + indexabstract); + prop.put("indexabstract", indexabstract); } - System.out.println("DEBUG-ABSTRACTGENERATION: maxcounthash = " + maxcounthash); - System.out.println("DEBUG-ABSTRACTGENERATION: neardhthash = " + neardhthash); - //yacyCore.log.logFine("DEBUG HASH SEARCH: " + indexabstract); - prop.put("indexabstract", indexabstract); } // prepare result diff --git a/source/de/anomic/index/indexRAMCacheRI.java b/source/de/anomic/index/indexRAMCacheRI.java index 52df0ca4f..9c241443a 100644 --- a/source/de/anomic/index/indexRAMCacheRI.java +++ b/source/de/anomic/index/indexRAMCacheRI.java @@ -390,8 +390,11 @@ public final class indexRAMCacheRI extends indexAbstractRI implements indexRI { if (urlselection == null) { return (indexContainer) wCache.get(wordHash); } else { - indexContainer ic = ((indexContainer) wCache.get(wordHash)).topLevelClone(); - ic.select(urlselection); + indexContainer ic = (indexContainer) wCache.get(wordHash); + if (ic != null) { + ic = ic.topLevelClone(); + ic.select(urlselection); + } return ic; } } diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index ec125b2de..92cc6285f 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -153,7 +153,7 @@ public final class plasmaSearchEvent extends Thread implements Runnable { if (yacySearch.remainingWaiting(primarySearchThreads) == 0) break; // all threads have finished try {Thread.sleep(100);} catch (InterruptedException e) {} } - prepareSecondarySearch(); + if (query.size() > 1) prepareSecondarySearch(); // catch up global results: // wait until primary timeout passed diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 0c020f0ed..554c55e27 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -437,7 +437,9 @@ public final class yacyClient { final long timestamp = System.currentTimeMillis(); // sending request - final HashMap result = nxTools.table( + HashMap result = null; + try { + result = nxTools.table( httpc.wput( new URL(url), targetPeer.getHexHash() + ".yacyh", @@ -448,10 +450,15 @@ public final class yacyClient { obj, null ) - ); + ); + } catch (IOException e) { + yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (" + e.getMessage() + "), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes)); + yacyCore.peerActions.peerDeparture(targetPeer); + return 0; + } if (result.size() == 0) { - yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + ", score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes)); + yacyCore.log.logFine("SEARCH failed FROM " + targetPeer.hash + ":" + targetPeer.getName() + " (zero response), score=" + targetPeer.selectscore + ", DHTdist=" + yacyDHTAction.dhtDistance(targetPeer.hash, wordhashes)); return 0; } diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index b404f3a3e..27fa4ce07 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -82,6 +82,7 @@ public class yacySearch extends Thread { plasmaURLPattern blacklist, plasmaSnippetCache snippetCache, plasmaSearchTimingProfile timingProfile, plasmaSearchRankingProfile rankingProfile) { super("yacySearch_" + targetPeer.getName()); + System.out.println("DEBUG - yacySearch thread " + this.getName() + " initialized " + ((urlhashes.length() == 0) ? "(primary)" : "(secondary)")); this.wordhashes = wordhashes; this.urlhashes = urlhashes; this.prefer = prefer; @@ -101,8 +102,8 @@ public class yacySearch extends Thread { public void run() { this.links = yacyClient.search(wordhashes, urlhashes, prefer, filter, maxDistance, global, targetPeer, urlManager, containerCache, abstractCache, blacklist, snippetCache, timingProfile, rankingProfile); + yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes); if (links != 0) { - //yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + links + " links for word hash " + wordhashes); yacyCore.seedDB.mySeed.incRI(links); yacyCore.seedDB.mySeed.incRU(links); }