From 7b80189bda73b7118d996b4ff07681ac06cf5f60 Mon Sep 17 00:00:00 2001
From: reger <reger18@arcor.de>
Date: Mon, 10 Apr 2017 22:42:06 +0200
Subject: [PATCH] Activate hosts navigator plugin. This includes rwi results in
 the navigator count. This might be tangential related to
 http://mantis.tokeek.de/view.php?id=736 as the example includes a local index
 search, while rwi results are not counted.

---
 htroot/yacysearchtrailer.html                 |  8 ----
 htroot/yacysearchtrailer.java                 | 42 -------------------
 htroot/yacysearchtrailer.json                 | 14 +------
 htroot/yacysearchtrailer.xml                  |  7 ----
 .../search/navigator/NavigatorPlugins.java    |  6 +--
 .../net/yacy/search/query/QueryModifier.java  |  4 +-
 source/net/yacy/search/query/SearchEvent.java | 33 +++++++--------
 7 files changed, 22 insertions(+), 92 deletions(-)

diff --git a/htroot/yacysearchtrailer.html b/htroot/yacysearchtrailer.html
index 0057f9b7d..d72c11f2e 100644
--- a/htroot/yacysearchtrailer.html
+++ b/htroot/yacysearchtrailer.html
@@ -156,14 +156,6 @@ function toggleVisibility(name, count) {
 	}
 }
 </script>
-#(nav-domains)#::
-<ul class="nav nav-sidebar menugroup">
-<li style="cursor: pointer; cursor: hand;"><h3 onclick="toggleVisibility('domains', #[count]#);">Provider [#[count]#] <span style="float:right" id="chevron-domains" class="glyphicon glyphicon-chevron-down" title="click to expand facet"></span></h3></li>
-#{element}#
-<li style="display:none" id="#[id]#"><a href="#[url]#" class="MenuItemLink"><input type="checkbox" onchange="window.location.href='#[url]#'"#(on)# checked="checked"::#(/on)#/> #[name]# (#[count]#)</a></li>
-#{/element}#</ul>
-<script>if (#[count]# <= 8) toggleVisibility('domains', #[count]#);</script>
-#(/nav-domains)#
 
 #(nav-languages)#::
 <ul class="nav nav-sidebar menugroup">
diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java
index beb7ce30c..25fdec2c3 100644
--- a/htroot/yacysearchtrailer.java
+++ b/htroot/yacysearchtrailer.java
@@ -103,47 +103,6 @@ public class yacysearchtrailer {
         int count;
         Iterator<String> navigatorIterator;
 
-        // domain navigators
-        final ScoreMap<String> hostNavigator = theSearch.hostNavigator;
-        if (hostNavigator == null || hostNavigator.isEmpty()) {
-            prop.put("nav-domains", 0);
-        } else {
-            prop.put("nav-domains", 1);
-            navigatorIterator = hostNavigator.keys(false);
-            int i = 0, pos = 0, neg = 0;
-            String nav, rawNav;
-            while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
-                name = navigatorIterator.next();
-                count = hostNavigator.get(name);
-                if (count == 0) break;
-                nav = "site%3A" + name;
-                /* Avoid double percent encoding in QueryParams.navurl */
-                rawNav = "site:" + name;
-                if (theSearch.query.modifier.sitehost == null || !theSearch.query.modifier.sitehost.contains(name)) {
-                    pos++;                    
-                    prop.put("nav-domains_element_" + i + "_on", 1);
-                    prop.put(fileType, "nav-domains_element_" + i + "_modifier", nav);
-                } else {
-                    neg++;                    
-                    prop.put("nav-domains_element_" + i + "_on", 0);
-                    prop.put(fileType, "nav-domains_element_" + i + "_modifier", "-" + nav);
-                    nav="";
-                    rawNav = "";
-                }
-                prop.put(fileType, "nav-domains_element_" + i + "_name", name);
-                prop.put(fileType, "nav-domains_element_" + i + "_url", QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false).toString());
-                prop.put(fileType, "nav-domains_element_" + i + "_id", "domains_" + i);
-                prop.put("nav-domains_element_" + i + "_count", count);
-                prop.put("nav-domains_element_" + i + "_nl", 1);
-                i++;
-            }
-            prop.put("nav-domains_element", i);
-            prop.put("nav-domains_count", i);
-            i--;
-            prop.put("nav-domains_element_" + i + "_nl", 0);
-            if (pos == 1 && neg == 0) prop.put("nav-domains", 0); // this navigation is not useful
-        }
-
         // language navigators
         final ScoreMap<String> languageNavigator = theSearch.languageNavigator;
         if (languageNavigator == null || languageNavigator.isEmpty()) {
@@ -213,7 +172,6 @@ public class yacysearchtrailer {
             for (Map.Entry<String, Integer> entry: cloud) {
                 name = entry.getKey();
                 count = entry.getValue();
-                prop.put("nav-topics_element_" + i + "_on", 1);
                 prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);
                 prop.put(fileType, "nav-topics_element_" + i + "_name", name);
                 prop.put(fileType, "nav-topics_element_" + i + "_url", QueryParams.navurl(fileType, 0, theSearch.query, name, false).toString());
diff --git a/htroot/yacysearchtrailer.json b/htroot/yacysearchtrailer.json
index 943286c59..78d6766c7 100644
--- a/htroot/yacysearchtrailer.json
+++ b/htroot/yacysearchtrailer.json
@@ -34,19 +34,7 @@
         {"name": "#[name]#", "count": "#[count]#", "modifier": "#[modifier]#", "url": "#[url]#"}#(nl)#::,#(/nl)#
 #{/element}#
       ]
-    },#(/nav-protocols)##(nav-domains)#::{
-      "facetname": "domains",
-      "displayname": "Domains",
-      "type": "String",
-      "min": "0",
-      "max": "0",
-      "mean": "0",
-      "elements": [
-#{element}#
-        {"name": "#[name]#", "count": "#[count]#", "modifier": "#[modifier]#", "url": "#[url]#"}#(nl)#::,#(/nl)#
-#{/element}#
-      ]
-    },#(/nav-domains)##{navs}#{
+    },#(/nav-protocols)##{navs}#{
       "facetname": "#[name]#",
       "displayname": "#[displayname]#",
       "type": "String",
diff --git a/htroot/yacysearchtrailer.xml b/htroot/yacysearchtrailer.xml
index 909adf43d..386911eac 100644
--- a/htroot/yacysearchtrailer.xml
+++ b/htroot/yacysearchtrailer.xml
@@ -1,11 +1,4 @@
 <yacy:navigation>
-#(nav-domains)#::
-<yacy:facet name="domains" displayname="Domains" type="String" min="0" max="0" mean="0">
-#{element}#
-<yacy:element name="#[name]#" count="#[count]#" modifier="#[modifier]#" url="#[url]#"/>
-#{/element}#
-</yacy:facet>
-#(/nav-domains)#
 #{navs}#
 <yacy:facet name="#[name]#" displayname="#[displayname]#" type="String" min="0" max="0" mean="0">
 #{element}#
diff --git a/source/net/yacy/search/navigator/NavigatorPlugins.java b/source/net/yacy/search/navigator/NavigatorPlugins.java
index 843ea1f59..ae55236c5 100644
--- a/source/net/yacy/search/navigator/NavigatorPlugins.java
+++ b/source/net/yacy/search/navigator/NavigatorPlugins.java
@@ -41,7 +41,7 @@ public class NavigatorPlugins {
     static public Map<String, String> listAvailable() {
         Map<String, String> defaultnavplugins = new TreeMap<String, String>();
         defaultnavplugins.put("filetype", "Filetype");
-        // defaultnavplugins.put("hosts", "Provider");
+        defaultnavplugins.put("hosts", "Provider");
         // defaultnavplugins.put("language", "Language");
         defaultnavplugins.put("authors", "Authors");
         defaultnavplugins.put("collections", "Collection");
@@ -87,11 +87,11 @@ public class NavigatorPlugins {
             if (navname.contains("filetype")) {
                 navigatorPlugins.put("filetype", new FileTypeNavigator("Filetype", CollectionSchema.url_file_ext_s));
             }
-/*
+
             if (navname.contains("hosts")) {
                 navigatorPlugins.put("hosts", new HostNavigator("Provider", CollectionSchema.host_s));
             }
-
+/*
             if (navname.contains("language")) {
                 navigatorPlugins.put("language", new LanguageNavigator("Language"));
             }
diff --git a/source/net/yacy/search/query/QueryModifier.java b/source/net/yacy/search/query/QueryModifier.java
index 6fd8ba623..c2194b565 100644
--- a/source/net/yacy/search/query/QueryModifier.java
+++ b/source/net/yacy/search/query/QueryModifier.java
@@ -378,8 +378,8 @@ public class QueryModifier {
         Date toDate = to == null || to.equals("*") ? null : DateDetection.parseLine(to, timezoneOffset);
         StringBuilder filterQuery = new StringBuilder(20);
         if (fromDate != null && toDate != null) {
-            String dstrFrom = fromDate == null ? "*" : DateFormatUtil.formatExternal(fromDate);
-            String dstrTo = toDate == null ? "*" : DateFormatUtil.formatExternal(toDate);
+            String dstrFrom = DateFormatUtil.formatExternal(fromDate);
+            String dstrTo = DateFormatUtil.formatExternal(toDate);
             filterQuery.append(CollectionSchema.dates_in_content_dts.getSolrFieldName()).append(":[").append(dstrFrom).append(" TO ").append(dstrTo).append(']'); 
         }
         return filterQuery.toString();
diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java
index b43939167..cf53dbab0 100644
--- a/source/net/yacy/search/query/SearchEvent.java
+++ b/source/net/yacy/search/query/SearchEvent.java
@@ -146,7 +146,6 @@ public final class SearchEvent {
     private int localsolroffset;
     private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons
     public final ScoreMap<String> locationNavigator; // a counter for the appearance of location coordinates
-    public final ScoreMap<String> hostNavigator; // a counter for the appearance of host names
     public final ScoreMap<String> protocolNavigator; // a counter for protocol types
     public final ScoreMap<String> dateNavigator; // a counter for file types
     public final ScoreMap<String> languageNavigator; // a counter for appearance of languages
@@ -262,7 +261,6 @@ public final class SearchEvent {
         // prepare configured search navigation
         final String navcfg = Switchboard.getSwitchboard().getConfig("search.navigation", "");
         this.locationNavigator = navcfg.contains("location") ? new ConcurrentScoreMap<String>() : null;
-        this.hostNavigator = navcfg.contains("hosts") ? new ConcurrentScoreMap<String>() : null;
         this.protocolNavigator = navcfg.contains("protocol") ? new ConcurrentScoreMap<String>() : null;
         this.dateNavigator = navcfg.contains("date") ? new ClusteredScoreMap<String>(true) : null;
         this.topicNavigatorCount = navcfg.contains("topics") ? MAX_TOPWORDS : 0;
@@ -842,18 +840,6 @@ public final class SearchEvent {
             }
         }
         
-        if (this.hostNavigator != null) {
-            fcts = facets.get(CollectionSchema.host_s.getSolrFieldName());
-            if (fcts != null) {
-                for (String host: fcts) {
-                    int hc = fcts.get(host);
-                    if (hc == 0) continue;
-                    if (host.startsWith("www.")) host = host.substring(4);
-                    this.hostNavigator.inc(host, hc);
-                }
-            }
-        }
-
         if (this.dateNavigator != null) {
             fcts = facets.get(CollectionSchema.dates_in_content_dts.getSolrFieldName());
             if (fcts != null) this.dateNavigator.inc(fcts);
@@ -1340,6 +1326,11 @@ public final class SearchEvent {
         return null;
     }
 
+    /**
+     * Adds the retrieved results (fulltext & rwi) to the result list and
+     * computes the text snippets
+     * @return true on adding entries to resultlist otherwise false
+     */
     public boolean drainStacksToResult() {
         // we take one entry from both stacks at the same time
         boolean success = false;
@@ -1465,7 +1456,7 @@ public final class SearchEvent {
         if (this.query.getSegment().connectedCitation()) {
             int referencesCount = this.query.getSegment().urlCitation().count(rentry.hash());
             r += (128 * referencesCount / (1 + 2 * rentry.llocal() + rentry.lother())) << this.query.ranking.coeff_citation;
-        } /* else r += 0; */
+        }
         // prefer hit with 'prefer' pattern
         if (this.query.prefer.matcher(rentry.url().toNormalform(true)).matches()) r += 255 << this.query.ranking.coeff_prefer;
         if (this.query.prefer.matcher(rentry.title()).matches()) r += 255 << this.query.ranking.coeff_prefer;
@@ -1482,11 +1473,11 @@ public final class SearchEvent {
         // (example Title="News News News News News News - today is party -- News News News News News News" to add one score instead of 12 * score !)
         for (final String urlcomp : urlcompmap) {
             int tc = topwords.get(urlcomp);
-            if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_urlcompintoplist;
+            if (tc > 0) r += tc << this.query.ranking.coeff_urlcompintoplist;
         }
         for (final String descrcomp : descrcompmap) {
             int tc = topwords.get(descrcomp);
-            if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_descrcompintoplist;
+            if (tc > 0) r += tc << this.query.ranking.coeff_descrcompintoplist;
         }
 
         final Iterator<String> shi = this.query.getQueryGoal().getIncludeWords();
@@ -1553,6 +1544,14 @@ public final class SearchEvent {
         return page.makeResultEntry(this.query.getSegment(), this.peers, null); // result without snippet
     }
     
+    /**
+     * This is the access point for the search interface to retrive ranked results.
+     * for display.
+     *
+     * @param item requested result counting number (starting at 0)
+     * @param timeout
+     * @return
+     */
     public URIMetadataNode oneResult(final int item, final long timeout) {
         // check if we already retrieved this item
         // (happens if a search pages is accessed a second time)