Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

htroot/Load_RSS_p.html

@@ -59,11 +59,11 @@
           </dl>
           <input type="submit" name="indexAllItemContent" value="Add All Items to Index (full content of url)" />
           #(/showload)#</dd>
-	      
+          #(showerrmsg)#::<dd class="error">#[msgtxt]#</dd>#(/showerrmsg)#   
         </dl>
       </fieldset>
     </form>
-
+    
 	#(showscheduledfeeds)#::
     <form id="scheduledfeeds" action="Load_RSS_p.html" method="post" enctype="multipart/form-data"><fieldset>
 	  <legend>List of Scheduled RSS Feed Load Targets</legend>

htroot/Load_RSS_p.java

@@ -73,7 +73,8 @@ public class Load_RSS_p {
         prop.put("shownewfeeds", 0);
         prop.put("showscheduledfeeds", 0);
         prop.put("url", "");
-
+        prop.put("showerrmsg", 0);
+        
         if (post != null && post.containsKey("removeSelectedFeedsNewList")) {
             for (final Map.Entry<String, String> entry: post.entrySet()) {
                 if (entry.getValue().startsWith("mark_")) try {
@@ -262,7 +263,7 @@ public class Load_RSS_p {
         try {
             url = post.containsKey("url") ? new DigestURL(post.get("url", "")) : null;
         } catch (final MalformedURLException e) {
-            ConcurrentLog.warn("Load_RSS_p", "url not well-formed: '" + post.get("url", "") + "'");
+            ConcurrentLog.warn("Load_RSS", "url not well-formed: '" + post.get("url", "") + "'");
         }
 
         ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
@@ -275,7 +276,10 @@ public class Load_RSS_p {
             final byte[] resource = response == null ? null : response.getContent();
             rss = resource == null ? null : RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, resource);
         } catch (final IOException e) {
-            ConcurrentLog.logException(e);
+            ConcurrentLog.warn("Load_RSS", e.getMessage());
+            prop.put("showerrmsg", 1);
+            prop.put("showerrmsg_msgtxt", "no valid response from given url");
+            return prop; // if no response nothing to process further
         }
 
         // index all selected items: description only

htroot/ViewFile.java

@@ -376,8 +376,8 @@ public class ViewFile {
                 prop.put("showSnippet_teasertext", desc);
                 prop.put("showSnippet", 1);
             }
-            // update index with parsed resouce if index entry is older
-            if (urlEntry.loaddate().before(response.lastModified())) {
+            // update index with parsed resouce if index entry is older or missing
+            if (urlEntry == null || urlEntry.loaddate().before(response.lastModified())) {
                 Switchboard.getSwitchboard().toIndexer(response);
             }
             if (document != null) document.close();

htroot/yacysearch_location.java

@@ -113,7 +113,7 @@ public class yacysearch_location {
                     prop.put("kml_placemark_" + placemarkCounter + "_copyright", message.getCopyright());
                     prop.put("kml_placemark_" + placemarkCounter + "_subject", message.getSubject());
                     prop.put("kml_placemark_" + placemarkCounter + "_description", message.getDescriptions().size() > 0 ? message.getDescriptions().get(0) : "");
-                    prop.put("kml_placemark_" + placemarkCounter + "_date", message.getPubDate());
+                    prop.put("kml_placemark_" + placemarkCounter + "_date", (message.getPubDate() != null) ? message.getPubDate() : new Date());
                     prop.putXML("kml_placemark_" + placemarkCounter + "_url", message.getLink());
                     prop.put("kml_placemark_" + placemarkCounter + "_pointname", message.getTitle());
                     prop.put("kml_placemark_" + placemarkCounter + "_lon", lo);

source/net/yacy/cora/document/feed/RSSMessage.java

@@ -212,20 +212,26 @@ public class RSSMessage implements Hit, Comparable<RSSMessage>, Comparator<RSSMe
         return Token.language.valueFrom(this.map, "");
     }
 
+    /**
+     * @return publishDate or null
+     */
     @Override
     public Date getPubDate() {
         final String dateString = Token.pubDate.valueFrom(this.map, "");
-        Date date;
-        try {
-            date = HeaderFramework.FORMAT_RFC1123.parse(dateString);
-        } catch (final ParseException e) {
+        if (!dateString.isEmpty()) { // skip parse exception on empty string
+            Date date;
             try {
-                date = GenericFormatter.SHORT_SECOND_FORMATTER.parse(dateString);
-            } catch (final ParseException e1) {
-                date = HeaderFramework.parseHTTPDate(dateString);
+                date = HeaderFramework.FORMAT_RFC1123.parse(dateString);
+            } catch (final ParseException e) {
+                try {
+                    date = GenericFormatter.SHORT_SECOND_FORMATTER.parse(dateString);
+                } catch (final ParseException e1) {
+                    date = HeaderFramework.parseHTTPDate(dateString); // returns null on parse error
+                }
             }
+            return date;
         }
-        return date;
+        return null;
     }
 
     @Override

source/net/yacy/cora/document/id/MultiProtocolURL.java

@@ -599,11 +599,12 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
             final int ch = s.charAt(i);
             if (ch == ' ') {                 // space
                 sbuf.append("%20");
-            } else if (ch == '%') {
+            } else if (ch == '%') {  
                 if (i < len - 2 && s.charAt(i + 1) >= '0' && s.charAt(i + 1) <= '9' && s.charAt(i + 2) >= '0' && s.charAt(i + 2) <= '9') {
+                    // TODO: actually 0..9 A..F a..f is allowed (or any of hex[] sequence)
                     sbuf.append((char)ch);   // lets consider this is used for encoding, leave it that way
                 } else {
-                    sbuf.append("%23");      // RFC 1738 2.2 unsafe char shall be encoded
+                    sbuf.append("%25");      // '%' RFC 1738 2.2 unsafe char shall be encoded
                 }
             } else if (ch == '&') { 
                 if (i < len - 6 && "amp;".equals(s.substring(i + 1, i + 5).toLowerCase())) {

source/net/yacy/search/query/QueryParams.java

@@ -93,7 +93,7 @@ public final class QueryParams {
     private static final Map<String, CollectionSchema> defaultfacetfields = new HashMap<String, CollectionSchema>();
     static {
         // the key shall match with configuration property search.navigation
-        defaultfacetfields.put("location", CollectionSchema.coordinate_p);
+        defaultfacetfields.put("location", CollectionSchema.coordinate_p_0_coordinate); // coordinate_p can't be used for facet (subfields), as value isn't used subfield can be used
         defaultfacetfields.put("hosts", CollectionSchema.host_s);
         defaultfacetfields.put("protocol", CollectionSchema.url_protocol_s);
         defaultfacetfields.put("filetype", CollectionSchema.url_file_ext_s);
@@ -249,7 +249,9 @@ public final class QueryParams {
         for (String navkey: search_navigation) {
             CollectionSchema f = defaultfacetfields.get(navkey);
             // handle special field, authors_sxt (add to facet w/o contains check, as authors_sxt is not enabled (is copyfield))
-            if (f != null && (solrSchema.contains(f) || f.name().equals("author_sxt"))) this.facetfields.add(f.getSolrFieldName());
+            // dto. for coordinate_p_0_coordinate is not enabled but used for location facet (because coordinate_p not valid for facet field)
+            if (f != null && (solrSchema.contains(f) || f.name().equals("author_sxt") || f.name().equals("coordinate_p_0_coordinate") ))
+                this.facetfields.add(f.getSolrFieldName());
         }
         if (LibraryProvider.autotagging != null) for (Tagging v: LibraryProvider.autotagging.getVocabularies()) {
             if (v.isFacet()) {

source/net/yacy/search/query/SearchEvent.java

@@ -795,7 +795,7 @@ public final class SearchEvent {
         // collect navigation information
         ReversibleScoreMap<String> fcts;
         if (this.locationNavigator != null) {
-            fcts = facets.get(CollectionSchema.coordinate_p.getSolrFieldName());
+            fcts = facets.get(CollectionSchema.coordinate_p_0_coordinate.getSolrFieldName());
             if (fcts != null) {
                 for (String coordinate: fcts) {
                     int hc = fcts.get(coordinate);

source/net/yacy/yacy.java

@@ -63,6 +63,7 @@ import com.google.common.io.Files;
 import net.yacy.cora.document.id.DigestURL;
 import net.yacy.cora.federate.yacy.CacheStrategy;
 import net.yacy.cora.order.Digest;
+import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.crawler.retrieval.Response;
 import net.yacy.server.serverSwitch;
 
@@ -299,6 +300,9 @@ public final class yacy {
                 httpServer = new Jetty9HttpServerImpl(port);
                 httpServer.startupServer();
                 sb.setHttpServer(httpServer);
+                // TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used)
+                ConnectionInfo.setServerMaxcount(sb.getConfigInt("connectionsMax", ConnectionInfo.getMaxcount()));
+
                 ConcurrentLog.info("STARTUP",httpServer.getVersion());
                 
                 // open the browser window