- one try to fix the httpd problem

- fix for handling of collection index that appears when removing elements
- added another navigation method (stub, not working yet)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4543 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacy/user/sidebar_navigation.java

@@ -25,6 +25,7 @@
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -36,6 +37,7 @@ import de.anomic.plasma.plasmaSearchQuery;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyURL;
 
 public class sidebar_navigation {
 
@@ -119,15 +121,34 @@ public class sidebar_navigation {
                         prop.put("navigation_topwords_words_" + hintcount + "_resource", theQuery.searchdom());
                         prop.put("navigation_topwords_words_" + hintcount + "_zonecode", theQuery.zonecode);
                     }
-                    prop.put("navigation_topwords_words", hintcount);
+                    hintcount++;
-                    if (hintcount++ > MAX_TOPWORDS) {
+                    if (hintcount >= MAX_TOPWORDS) break;
-                        break;
-                    }
                 }
+                prop.put("navigation_topwords_words", hintcount);
                 prop.put("navigation_topwords", "1");
             }
         }
         
+        // compose language zone drill-down
+        int c = 0;
+        final Iterator<Map.Entry<String, Integer>> iter = theSearch.getRankingResult().getZoneStatistics().entrySet().iterator();
+        Map.Entry<String, Integer> entry;
+        while (iter.hasNext()) {
+            entry = iter.next();
+            if ((theQuery == null) || (theQuery.queryString == null)) break;
+            prop.putHTML("navigation_languagezone_zones_" + c + "_zone", entry.getKey() + " (" + entry.getValue() + ")");
+            prop.putHTML("navigation_languagezone_zones_" + c + "_search", theQuery.queryString.replace(' ', '+'));
+            prop.put("navigation_languagezone_zones_" + c + "_count", theQuery.displayResults());
+            prop.put("navigation_languagezone_zones_" + c + "_offset", "0");
+            prop.put("navigation_languagezone_zones_" + c + "_contentdom", theQuery.contentdom());
+            prop.put("navigation_languagezone_zones_" + c + "_resource", theQuery.searchdom());
+            prop.put("navigation_languagezone_zones_" + c + "_zonecode", yacyURL.zone2map.get(entry.getKey()).intValue());
+            prop.put("navigation_languagezone_zones", c);
+            c++;
+        }
+        prop.put("navigation_languagezone", (c > 2) ? "1" : "0");
+        
+        
         // compose page navigation
         StringBuffer resnav = new StringBuffer();
         int thispage = offset / theQuery.displayResults();

source/de/anomic/http/httpdFileHandler.java

@@ -851,7 +851,17 @@ public final class httpdFileHandler {
                         if (chunkedOut != null) {
                             chunkedOut.finish();
                         }
-                    }   
+
+                        // flush all
+                        try {newOut.flush();}catch (Exception e) {}
+                        
+                        // wait a little time until everything closes so that clients can read from the streams/sockets
+                        if ((contentLength >= 0) && ((String)requestHeader.get(httpHeader.CONNECTION, "close")).indexOf("keep-alive") == -1) {
+                            // in case that the client knows the size in advance (contentLength present) the waiting will have no effect on the interface performance
+                            // but if the client waits on a connection interruption this will slow down.
+                            try {Thread.sleep(2000);} catch (InterruptedException e) {} // FIXME: is this necessary?
+                        }
+                    }
                     
                     // check mime type again using the result array: these are 'magics'
 //                    if (serverByteBuffer.equals(result, 1, "PNG".getBytes())) mimeType = mimeTable.getProperty("png","text/html");
@@ -921,10 +931,6 @@ public final class httpdFileHandler {
             
         } finally {
             try {out.flush();}catch (Exception e) {}
-            if (((String)requestHeader.get(httpHeader.CONNECTION, "close")).indexOf("keep-alive") == -1) {
-                // wait a little time until everything closes so that clients can read from the streams/sockets
-                try {Thread.sleep(50);} catch (InterruptedException e) {} // FIXME: is this necessary?
-            }
         }
     }
     

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -920,7 +920,7 @@ public class kelondroCollectionIndex {
             array_remove(
                     oldPartitionNumber, serialNumber, this.payloadrow.objectsize,
                     oldrownumber);
-            index.remove(key, true);
+            index.remove(key, false);
             return removed;
         }
         

source/de/anomic/plasma/plasmaSearchRankingProcess.java

@@ -68,6 +68,7 @@ public final class plasmaSearchRankingProcess {
     private TreeSet<String> misses; // contains url-hashes that could not been found in the LURL-DB
     private plasmaWordIndex wordIndex;
     private HashMap<String, indexContainer>[] localSearchContainerMaps;
+    private int[] domZones;
     
     public plasmaSearchRankingProcess(plasmaWordIndex wordIndex, plasmaSearchQuery query, int maxentries, int concurrency) {
         // we collect the urlhashes and construct a list with urlEntry objects
@@ -90,6 +91,8 @@ public final class plasmaSearchRankingProcess {
         this.wordIndex = wordIndex;
         this.flagcount = new int[32];
         for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;}
+        this.domZones = new int[8];
+        for (int i = 0; i < 8; i++) {this.domZones[i] = 0;}
     }
     
     public long ranking(indexRWIVarEntry word) {
@@ -176,6 +179,10 @@ public final class plasmaSearchRankingProcess {
                 continue;
             }
             
+            // count domZones
+            System.out.println("DEBUG domDomain dom=" + wordIndex.loadedURL.load(iEntry.urlHash, iEntry, 0).comp().url().getHost() + ", zone=" + yacyURL.domDomain(iEntry.urlHash()));
+            this.domZones[yacyURL.domDomain(iEntry.urlHash())]++;
+            
             // insert
             if ((maxentries < 0) || (stack.size() < maxentries)) {
                 // in case that we don't have enough yet, accept any new entry
@@ -327,6 +334,10 @@ public final class plasmaSearchRankingProcess {
         return this.local_resourceSize;
     }
     
+    public Map<String, Integer> getZoneStatistics() {
+        return yacyURL.zoneStatistics(this.domZones);
+    }
+    
     public indexRWIEntry remove(String urlHash) {
         kelondroSortStack<indexRWIVarEntry>.stackElement se = stack.remove(urlHash.hashCode());
         if (se == null) return null;

source/de/anomic/yacy/yacyURL.java

@@ -31,6 +31,7 @@ import java.io.File;
 import java.net.MalformedURLException;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -382,7 +383,7 @@ public class yacyURL {
     public static final int language_domain_africa_zone     = 128 + 32;      //{5, 7};
     public static final int language_domain_any_zone        = 255;
     
-    public static final String[] regions = {"europe", "english", "spanish", "asia", "middleeast", "africa"};
+    public static final HashMap<String, Integer> zone2map = new HashMap<String, Integer>();
     
     static {
         // create a dummy hash
@@ -398,6 +399,14 @@ public class yacyURL {
         insertTLDProps(TLD_Africa,              5); // africa
         insertTLDProps(TLD_Generic,             6); // anything else, mixed languages, mainly english
         // the id=7 is used to flag local addresses
+        
+        zone2map.put("europe",     language_domain_europe_zone);
+        zone2map.put("english",    language_domain_english_zone);
+        zone2map.put("spanish",    language_domain_spanish_zone);
+        zone2map.put("asia",       language_domain_asia_zone);
+        zone2map.put("middleeast", language_domain_middleeast_zone);
+        zone2map.put("africa",     language_domain_africa_zone);
+        zone2map.put("any",        language_domain_any_zone);
     }
     
     // class variables
@@ -1093,7 +1102,7 @@ public class yacyURL {
         // returns the ID of the domain of the domain
         assert (urlHash != null);
         assert (urlHash.length() == 12) : "urlhash = " + urlHash;
-        return (kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 12) >> 2;
+        return (kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 28) >> 2;
     }
 
     public static boolean isLocalDomain(String urlhash) {
@@ -1123,6 +1132,26 @@ public class yacyURL {
         return language;
     }
     
+    public static Map<String, Integer> zoneStatistics(int[] domAccumulators) {
+        assert domAccumulators.length == 8;
+        HashMap<String, Integer> zoneCounter = new HashMap<String, Integer>();
+        Iterator<Map.Entry<String, Integer>> j;
+        Map.Entry<String, Integer> entry;
+        for (int i = 0; i < 8; i++) {
+            j = zone2map.entrySet().iterator();
+            while (j.hasNext()) {
+                entry = j.next();
+                if ((i & entry.getValue().intValue()) != 0) {
+                    if (zoneCounter.containsKey(entry.getKey())) {
+                        zoneCounter.put(entry.getKey(), zoneCounter.get(entry.getKey()) + domAccumulators[i]);
+                    } else {
+                        zoneCounter.put(entry.getKey(), domAccumulators[i]);
+                    }
+                }
+            }
+        }
+        return zoneCounter;
+    }
     
     public static void main(String[] args) {
         String[][] test = new String[][]{