From 8eb6fba59c665f5ece3151f1c4688fda5a5ba2b9 Mon Sep 17 00:00:00 2001
From: reger <reger18@arcor.de>
Date: Wed, 21 Dec 2016 02:04:13 +0100
Subject: [PATCH] activate filetype navigator plugin and restrict config
 (append) of navs to not already actives. Dht results are now included in
 count this might over shoot on redundant dht and solr, while the previous
 solr facet based was always low.

---
 htroot/ConfigSearchPage_p.html                | 13 ++----
 htroot/ConfigSearchPage_p.java                | 24 +++++++++--
 htroot/yacysearchtrailer.html                 |  9 -----
 htroot/yacysearchtrailer.java                 | 40 -------------------
 .../search/navigator/NavigatorPlugins.java    | 30 ++++++++++++++
 source/net/yacy/search/query/SearchEvent.java | 24 -----------
 6 files changed, 54 insertions(+), 86 deletions(-)

diff --git a/htroot/ConfigSearchPage_p.html b/htroot/ConfigSearchPage_p.html
index e2d27267d..ebc51257b 100644
--- a/htroot/ConfigSearchPage_p.html
+++ b/htroot/ConfigSearchPage_p.html
@@ -121,12 +121,6 @@
                             </li>#(/search.navigation.location)#                                
                           </ul></td>
                       </tr>                          
-                      <tr>
-                        <td><input type="checkbox" name="search.navigation.filetype" value="true" #(search.navigation.filetype)#::checked="checked" #(/search.navigation.filetype)# /></td>
-                        <td><ul class="nav nav-sidebar menugroup" id="sidebarFiletypes" style="padding-left:15px; padding-right:10px;">
-                          <li><h3>Filetype</h3></li>
-                          </ul></td>
-                      </tr>
                       <tr>
                         <td><input type="checkbox" name="search.navigation.hosts" value="true" #(search.navigation.hosts)#::checked="checked" #(/search.navigation.hosts)# /></td>
                         <td><ul class="nav nav-sidebar menugroup" id="sidebarDomains" style="padding-left:15px; padding-right:10px;">
@@ -287,10 +281,9 @@ var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=
         <fieldset><legend>Add Navigators</legend>
           <label for="search.navigation.navname">append</label>
           <select type="text" name="search.navigation.navname">
-            <option value="authors">Authors</option>
-            <option value="collections">Collection</option>
-            <option value="namespace">Wiki Name Space</option>
-            <option value="year">Year</option>
+            #{search.navigation.list}#
+            <option value="#[name]#">#[displayname]#</option>
+            #{/search.navigation.list}#
           </select>
           <button type="submit" name="add.nav" class="btn btn-xs btn-default" ><span class="glyphicon glyphicon-plus-sign" style="color: green"/></button><br>
           <label for="search.navigation.maxcount">max. items</label>
diff --git a/htroot/ConfigSearchPage_p.java b/htroot/ConfigSearchPage_p.java
index ff628a30b..c46ac9d93 100644
--- a/htroot/ConfigSearchPage_p.java
+++ b/htroot/ConfigSearchPage_p.java
@@ -32,6 +32,7 @@ import java.io.IOException;
 import java.sql.Date;
 import java.util.Map;
 import java.util.Properties;
+import java.util.TreeMap;
 import net.yacy.cora.date.GenericFormatter;
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.cora.util.ConcurrentLog;
@@ -86,7 +87,7 @@ public class ConfigSearchPage_p {
                 // construct navigation String
                 String nav = "";
                 if (post.getBoolean("search.navigation.location")) nav += "location,";
-                if (post.getBoolean("search.navigation.filetype")) nav += "filetype,";
+                // if (post.getBoolean("search.navigation.filetype")) nav += "filetype,";
                 if (post.getBoolean("search.navigation.protocol")) nav += "protocol,";
                 if (post.getBoolean("search.navigation.hosts")) nav += "hosts,";
                 if (post.getBoolean("search.navigation.language")) nav += "language,";
@@ -112,7 +113,7 @@ public class ConfigSearchPage_p {
 
             if (post.containsKey("add.nav")) { // button: add navigator plugin to ative list
                 String navname = post.get("search.navigation.navname");
-                if (navname != null) {
+                if (navname != null && !navname.isEmpty()) {
                     String naviconf = sb.getConfig("search.navigation", "");
                     naviconf += "," + navname;
                     sb.setConfig("search.navigation", naviconf);
@@ -196,7 +197,7 @@ public class ConfigSearchPage_p {
         prop.put("search.result.show.snapshots", sb.getConfigBool("search.result.show.snapshots", false) ? 1 : 0);
 
         prop.put("search.navigation.location", sb.getConfig("search.navigation", "").indexOf("location",0) >= 0 ? 1 : 0);
-        prop.put("search.navigation.filetype", sb.getConfig("search.navigation", "").indexOf("filetype",0) >= 0 ? 1 : 0);
+        // prop.put("search.navigation.filetype", sb.getConfig("search.navigation", "").indexOf("filetype",0) >= 0 ? 1 : 0);
         prop.put("search.navigation.protocol", sb.getConfig("search.navigation", "").indexOf("protocol",0) >= 0 ? 1 : 0);
         prop.put("search.navigation.hosts", sb.getConfig("search.navigation", "").indexOf("hosts",0) >= 0 ? 1 : 0);
         prop.put("search.navigation.language", sb.getConfig("search.navigation", "").indexOf("language",0) >= 0 ? 1 : 0);
@@ -217,6 +218,23 @@ public class ConfigSearchPage_p {
         }
         prop.put("search.navigation.plugin", i);
 
+        // fill select field options (only navs not already active)
+        Map<String, String> defaultnavplugins = NavigatorPlugins.listAvailable();
+        i=0;
+        for (String navname : defaultnavplugins.keySet()) {
+            if (!navplugins.containsKey(navname)) {
+                prop.put("search.navigation.list_" + i + "_name", navname);
+                prop.put("search.navigation.list_" + i + "_displayname", defaultnavplugins.get(navname));
+                i++;
+            }
+        }
+        if (i == 0) { // on no new nav avail. put in dummy name to indicate empty list
+            prop.put("search.navigation.list_" + i + "_name", "");
+            prop.put("search.navigation.list_" + i + "_displayname", "---");
+            i = 1;
+        }
+        prop.put("search.navigation.list", i);
+
         prop.put("search.navigation.maxcount", sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT));
 
         prop.put("about.headline", sb.getConfig("about.headline", "About"));
diff --git a/htroot/yacysearchtrailer.html b/htroot/yacysearchtrailer.html
index 01fb25499..0057f9b7d 100644
--- a/htroot/yacysearchtrailer.html
+++ b/htroot/yacysearchtrailer.html
@@ -165,15 +165,6 @@ function toggleVisibility(name, count) {
 <script>if (#[count]# <= 8) toggleVisibility('domains', #[count]#);</script>
 #(/nav-domains)#
 
-#(nav-filetypes)#::
-<ul class="nav nav-sidebar menugroup">
-<li style="cursor: pointer; cursor: hand;"><h3 onclick="toggleVisibility('filetypes', #[count]#);">Filetype [#[count]#] <span style="float:right" id="chevron-filetypes" class="glyphicon glyphicon-chevron-down" title="click to expand facet"></span></h3></li>
-#{element}#
-<li style="display:none" id="#[id]#"><a href="#[url]#" class="MenuItemLink"><input type="checkbox" onchange="window.location.href='#[url]#'"#(on)# checked="checked"::#(/on)#/> #[name]# (#[count]#)</a></li>
-#{/element}#</ul>
-<script>if (#[count]# <= 8) toggleVisibility('filetypes', #[count]#);</script>
-#(/nav-filetypes)#
-
 #(nav-languages)#::
 <ul class="nav nav-sidebar menugroup">
 <li style="cursor: pointer; cursor: hand;"><h3 onclick="toggleVisibility('languages', #[count]#);">Language [#[count]#] <span style="float:right" id="chevron-languages" class="glyphicon glyphicon-chevron-down" title="click to expand facet"></span></h3></li>
diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java
index 853989157..beb7ce30c 100644
--- a/htroot/yacysearchtrailer.java
+++ b/htroot/yacysearchtrailer.java
@@ -338,46 +338,6 @@ public class yacysearchtrailer {
             if (pos == 1 && neg == 0) prop.put("nav-dates", 0); // this navigation is not useful
         }
 
-        // filetype navigators
-        if (theSearch.filetypeNavigator == null || theSearch.filetypeNavigator.isEmpty()) {
-            prop.put("nav-filetypes", 0);
-        } else {
-            prop.put("nav-filetypes", 1);
-            navigatorIterator = theSearch.filetypeNavigator.keys(false);
-            int i = 0, pos = 0, neg = 0;
-            String nav, rawNav;
-            while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
-                name = navigatorIterator.next().trim();
-                count = theSearch.filetypeNavigator.get(name);
-                if (count == 0) break;
-                nav = "filetype%3A" + name;
-                /* Avoid double percent encoding in QueryParams.navurl */
-                rawNav = "filetype:" + name;
-                if (theSearch.query.modifier.filetype == null || !theSearch.query.modifier.filetype.contains(name) ) {
-                    pos++;
-                    prop.put("nav-filetypes_element_" + i + "_on", 1);
-                    prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", nav);
-                } else {
-                    neg++;                    
-                    prop.put("nav-filetypes_element_" + i + "_on", 0);                    
-                    prop.put(fileType, "nav-filetypes_element_" + i + "_modifier", "-" + nav);
-                    nav="";
-                    rawNav = "";
-                }
-                prop.put(fileType, "nav-filetypes_element_" + i + "_name", name);
-                prop.put(fileType, "nav-filetypes_element_" + i + "_url", QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false).toString());
-                prop.put(fileType, "nav-filetypes_element_" + i + "_id", "filetypes_" + i);
-                prop.put("nav-filetypes_element_" + i + "_count", count);
-                prop.put("nav-filetypes_element_" + i + "_nl", 1);
-                i++;
-            }
-            prop.put("nav-filetypes_element", i);
-            prop.put("nav-filetypes_count", i);
-            i--;
-            prop.put("nav-filetypes_element_" + i + "_nl", 0);
-            if (pos == 1 && neg == 0) prop.put("nav-filetypes", 0); // this navigation is not useful
-        }
-
         // vocabulary navigators
         final Map<String, ScoreMap<String>> vocabularyNavigators = theSearch.vocabularyNavigator;
         if (vocabularyNavigators != null && !vocabularyNavigators.isEmpty()) {
diff --git a/source/net/yacy/search/navigator/NavigatorPlugins.java b/source/net/yacy/search/navigator/NavigatorPlugins.java
index ad09a63ac..b7547bd48 100644
--- a/source/net/yacy/search/navigator/NavigatorPlugins.java
+++ b/source/net/yacy/search/navigator/NavigatorPlugins.java
@@ -24,6 +24,7 @@ package net.yacy.search.navigator;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.TreeMap;
 import net.yacy.crawler.CrawlSwitchboard;
 import static net.yacy.search.query.SearchEvent.log;
 import net.yacy.search.schema.CollectionSchema;
@@ -33,6 +34,23 @@ import net.yacy.search.schema.CollectionSchema;
  */
 public class NavigatorPlugins {
 
+    /**
+     * List of available navigators
+     * @return Map key=navigatorCfgname, value=std.DisplayName
+     */
+    static public Map<String, String> listAvailable() {
+        Map<String, String> defaultnavplugins = new TreeMap();
+        defaultnavplugins.put("filetype", "Filetype");
+        // defaultnavplugins.put("hosts", "Provider");
+        // defaultnavplugins.put("language", "Language");
+        defaultnavplugins.put("authors", "Authors");
+        defaultnavplugins.put("collections", "Collection");
+        defaultnavplugins.put("namespace", "Wiki Name Space");
+        defaultnavplugins.put("year", "Year");
+        // defaultnavplugins.put("year:dates_in_content_dts:Event","Event");
+        return defaultnavplugins;
+    }
+
     /**
      * Creates map of active search navigators from comma separated config string
      * @param navcfg comma separated string of navigator names
@@ -65,6 +83,18 @@ public class NavigatorPlugins {
                 navigatorPlugins.put("collections", tmpnav);
             }
 
+            if (navname.contains("filetype")) {
+                navigatorPlugins.put("filetype", new FileTypeNavigator("Filetype", CollectionSchema.url_file_ext_s));
+            }
+/*
+            if (navname.contains("hosts")) {
+                navigatorPlugins.put("hosts", new HostNavigator("Provider", CollectionSchema.host_s));
+            }
+
+            if (navname.contains("language")) {
+                navigatorPlugins.put("language", new LanguageNavigator("Language"));
+            }
+*/
             if (navname.contains("namespace")) {
                 navigatorPlugins.put("namespace", new NameSpaceNavigator("Wiki Name Space"));
             }
diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java
index af63bd2e0..1008b26f1 100644
--- a/source/net/yacy/search/query/SearchEvent.java
+++ b/source/net/yacy/search/query/SearchEvent.java
@@ -73,7 +73,6 @@ import net.yacy.data.WorkTables;
 import net.yacy.document.LargeNumberCache;
 import net.yacy.document.LibraryProvider;
 import net.yacy.document.ProbabilisticClassifier;
-import net.yacy.document.TextParser;
 import net.yacy.document.Tokenizer;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
 import net.yacy.kelondro.data.word.Word;
@@ -149,7 +148,6 @@ public final class SearchEvent {
     public final ScoreMap<String> locationNavigator; // a counter for the appearance of location coordinates
     public final ScoreMap<String> hostNavigator; // a counter for the appearance of host names
     public final ScoreMap<String> protocolNavigator; // a counter for protocol types
-    public final ScoreMap<String> filetypeNavigator; // a counter for file types
     public final ScoreMap<String> dateNavigator; // a counter for file types
     public final ScoreMap<String> languageNavigator; // a counter for appearance of languages
     public final Map<String, ScoreMap<String>> vocabularyNavigator; // counters for Vocabularies; key is metatag.getVocabularyName()
@@ -262,7 +260,6 @@ public final class SearchEvent {
         this.locationNavigator = navcfg.contains("location") ? new ConcurrentScoreMap<String>() : null;
         this.hostNavigator = navcfg.contains("hosts") ? new ConcurrentScoreMap<String>() : null;
         this.protocolNavigator = navcfg.contains("protocol") ? new ConcurrentScoreMap<String>() : null;
-        this.filetypeNavigator = navcfg.contains("filetype") ? new ConcurrentScoreMap<String>() : null;
         this.dateNavigator = navcfg.contains("date") ? new ClusteredScoreMap<String>(true) : null;
         this.topicNavigatorCount = navcfg.contains("topics") ? MAX_TOPWORDS : 0;
         this.languageNavigator = navcfg.contains("language") ? new ConcurrentScoreMap<String>() : null;
@@ -853,27 +850,6 @@ public final class SearchEvent {
             }
         }
 
-        if (this.filetypeNavigator != null) {
-            fcts = facets.get(CollectionSchema.url_file_ext_s.getSolrFieldName());
-            if (fcts != null) {
-                // remove all filetypes that we don't know
-                Iterator<String> i = fcts.iterator();
-                while (i.hasNext()) {
-                    String ext = i.next();
-                    if (this.query.contentdom == ContentDomain.TEXT) {
-                        if ((Classification.isImageExtension(ext) && this.excludeintext_image) ||
-                            (TextParser.supportsExtension(ext) != null && !Classification.isAnyKnownExtension(ext))) {
-                            //Log.logInfo("SearchEvent", "removed unknown extension " + ext + " from navigation.");
-                            i.remove();
-                        }
-                    } else { // for image and media search also limit to known extension
-                        if (!Classification.isAnyKnownExtension(ext)) i.remove();
-                    }
-                }
-                this.filetypeNavigator.inc(fcts);
-            }
-        }
-
         if (this.dateNavigator != null) {
             fcts = facets.get(CollectionSchema.dates_in_content_dts.getSolrFieldName());
             if (fcts != null) this.dateNavigator.inc(fcts);