diff --git a/htroot/yacy/user/sidebar_navigation.java b/htroot/yacy/user/sidebar_navigation.java index 4ae133e7f..7fc614d60 100644 --- a/htroot/yacy/user/sidebar_navigation.java +++ b/htroot/yacy/user/sidebar_navigation.java @@ -25,6 +25,7 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import java.util.Iterator; +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -36,6 +37,7 @@ import de.anomic.plasma.plasmaSearchQuery; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyURL; public class sidebar_navigation { @@ -119,15 +121,34 @@ public class sidebar_navigation { prop.put("navigation_topwords_words_" + hintcount + "_resource", theQuery.searchdom()); prop.put("navigation_topwords_words_" + hintcount + "_zonecode", theQuery.zonecode); } - prop.put("navigation_topwords_words", hintcount); - if (hintcount++ > MAX_TOPWORDS) { - break; - } + hintcount++; + if (hintcount >= MAX_TOPWORDS) break; } + prop.put("navigation_topwords_words", hintcount); prop.put("navigation_topwords", "1"); } } + // compose language zone drill-down + int c = 0; + final Iterator> iter = theSearch.getRankingResult().getZoneStatistics().entrySet().iterator(); + Map.Entry entry; + while (iter.hasNext()) { + entry = iter.next(); + if ((theQuery == null) || (theQuery.queryString == null)) break; + prop.putHTML("navigation_languagezone_zones_" + c + "_zone", entry.getKey() + " (" + entry.getValue() + ")"); + prop.putHTML("navigation_languagezone_zones_" + c + "_search", theQuery.queryString.replace(' ', '+')); + prop.put("navigation_languagezone_zones_" + c + "_count", theQuery.displayResults()); + prop.put("navigation_languagezone_zones_" + c + "_offset", "0"); + prop.put("navigation_languagezone_zones_" + c + "_contentdom", theQuery.contentdom()); + prop.put("navigation_languagezone_zones_" + c + "_resource", theQuery.searchdom()); + prop.put("navigation_languagezone_zones_" + c + "_zonecode", yacyURL.zone2map.get(entry.getKey()).intValue()); + prop.put("navigation_languagezone_zones", c); + c++; + } + prop.put("navigation_languagezone", (c > 2) ? "1" : "0"); + + // compose page navigation StringBuffer resnav = new StringBuffer(); int thispage = offset / theQuery.displayResults(); diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 52fdd0041..5c0e82346 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -851,7 +851,17 @@ public final class httpdFileHandler { if (chunkedOut != null) { chunkedOut.finish(); } - } + + // flush all + try {newOut.flush();}catch (Exception e) {} + + // wait a little time until everything closes so that clients can read from the streams/sockets + if ((contentLength >= 0) && ((String)requestHeader.get(httpHeader.CONNECTION, "close")).indexOf("keep-alive") == -1) { + // in case that the client knows the size in advance (contentLength present) the waiting will have no effect on the interface performance + // but if the client waits on a connection interruption this will slow down. + try {Thread.sleep(2000);} catch (InterruptedException e) {} // FIXME: is this necessary? + } + } // check mime type again using the result array: these are 'magics' // if (serverByteBuffer.equals(result, 1, "PNG".getBytes())) mimeType = mimeTable.getProperty("png","text/html"); @@ -921,10 +931,6 @@ public final class httpdFileHandler { } finally { try {out.flush();}catch (Exception e) {} - if (((String)requestHeader.get(httpHeader.CONNECTION, "close")).indexOf("keep-alive") == -1) { - // wait a little time until everything closes so that clients can read from the streams/sockets - try {Thread.sleep(50);} catch (InterruptedException e) {} // FIXME: is this necessary? - } } } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index ded5711f7..4ff3ec163 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -920,7 +920,7 @@ public class kelondroCollectionIndex { array_remove( oldPartitionNumber, serialNumber, this.payloadrow.objectsize, oldrownumber); - index.remove(key, true); + index.remove(key, false); return removed; } diff --git a/source/de/anomic/plasma/plasmaSearchRankingProcess.java b/source/de/anomic/plasma/plasmaSearchRankingProcess.java index 028664cd7..41b6db5d7 100644 --- a/source/de/anomic/plasma/plasmaSearchRankingProcess.java +++ b/source/de/anomic/plasma/plasmaSearchRankingProcess.java @@ -68,6 +68,7 @@ public final class plasmaSearchRankingProcess { private TreeSet misses; // contains url-hashes that could not been found in the LURL-DB private plasmaWordIndex wordIndex; private HashMap[] localSearchContainerMaps; + private int[] domZones; public plasmaSearchRankingProcess(plasmaWordIndex wordIndex, plasmaSearchQuery query, int maxentries, int concurrency) { // we collect the urlhashes and construct a list with urlEntry objects @@ -90,6 +91,8 @@ public final class plasmaSearchRankingProcess { this.wordIndex = wordIndex; this.flagcount = new int[32]; for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;} + this.domZones = new int[8]; + for (int i = 0; i < 8; i++) {this.domZones[i] = 0;} } public long ranking(indexRWIVarEntry word) { @@ -176,6 +179,10 @@ public final class plasmaSearchRankingProcess { continue; } + // count domZones + System.out.println("DEBUG domDomain dom=" + wordIndex.loadedURL.load(iEntry.urlHash, iEntry, 0).comp().url().getHost() + ", zone=" + yacyURL.domDomain(iEntry.urlHash())); + this.domZones[yacyURL.domDomain(iEntry.urlHash())]++; + // insert if ((maxentries < 0) || (stack.size() < maxentries)) { // in case that we don't have enough yet, accept any new entry @@ -327,6 +334,10 @@ public final class plasmaSearchRankingProcess { return this.local_resourceSize; } + public Map getZoneStatistics() { + return yacyURL.zoneStatistics(this.domZones); + } + public indexRWIEntry remove(String urlHash) { kelondroSortStack.stackElement se = stack.remove(urlHash.hashCode()); if (se == null) return null; diff --git a/source/de/anomic/yacy/yacyURL.java b/source/de/anomic/yacy/yacyURL.java index 78ffad860..153f86067 100644 --- a/source/de/anomic/yacy/yacyURL.java +++ b/source/de/anomic/yacy/yacyURL.java @@ -31,6 +31,7 @@ import java.io.File; import java.net.MalformedURLException; import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -382,7 +383,7 @@ public class yacyURL { public static final int language_domain_africa_zone = 128 + 32; //{5, 7}; public static final int language_domain_any_zone = 255; - public static final String[] regions = {"europe", "english", "spanish", "asia", "middleeast", "africa"}; + public static final HashMap zone2map = new HashMap(); static { // create a dummy hash @@ -398,6 +399,14 @@ public class yacyURL { insertTLDProps(TLD_Africa, 5); // africa insertTLDProps(TLD_Generic, 6); // anything else, mixed languages, mainly english // the id=7 is used to flag local addresses + + zone2map.put("europe", language_domain_europe_zone); + zone2map.put("english", language_domain_english_zone); + zone2map.put("spanish", language_domain_spanish_zone); + zone2map.put("asia", language_domain_asia_zone); + zone2map.put("middleeast", language_domain_middleeast_zone); + zone2map.put("africa", language_domain_africa_zone); + zone2map.put("any", language_domain_any_zone); } // class variables @@ -1093,7 +1102,7 @@ public class yacyURL { // returns the ID of the domain of the domain assert (urlHash != null); assert (urlHash.length() == 12) : "urlhash = " + urlHash; - return (kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 12) >> 2; + return (kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 28) >> 2; } public static boolean isLocalDomain(String urlhash) { @@ -1123,6 +1132,26 @@ public class yacyURL { return language; } + public static Map zoneStatistics(int[] domAccumulators) { + assert domAccumulators.length == 8; + HashMap zoneCounter = new HashMap(); + Iterator> j; + Map.Entry entry; + for (int i = 0; i < 8; i++) { + j = zone2map.entrySet().iterator(); + while (j.hasNext()) { + entry = j.next(); + if ((i & entry.getValue().intValue()) != 0) { + if (zoneCounter.containsKey(entry.getKey())) { + zoneCounter.put(entry.getKey(), zoneCounter.get(entry.getKey()) + domAccumulators[i]); + } else { + zoneCounter.put(entry.getKey(), domAccumulators[i]); + } + } + } + } + return zoneCounter; + } public static void main(String[] args) { String[][] test = new String[][]{