From e37a4f0b3da605e68b5256b244ea1aa5bf198a68 Mon Sep 17 00:00:00 2001
From: reger <reger18@arcor.de>
Date: Sun, 6 Sep 2015 22:19:05 +0200
Subject: [PATCH] prevent metadata records in index w/o valid url by throwing
 MalformedURL exception on URIMetadataNode creation

---
 .../AbstractFederateSearchConnector.java      |  6 ++--
 .../federate/SolrFederateSearchConnector.java |  7 ++--
 .../kelondro/data/meta/URIMetadataNode.java   | 33 ++++---------------
 source/net/yacy/peers/Protocol.java           |  8 ++++-
 source/net/yacy/search/Switchboard.java       | 10 ++++--
 5 files changed, 29 insertions(+), 35 deletions(-)

diff --git a/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java b/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java
index b9e7c297a..932b128ca 100644
--- a/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java
+++ b/source/net/yacy/cora/federate/AbstractFederateSearchConnector.java
@@ -21,6 +21,7 @@ package net.yacy.cora.federate;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -143,7 +144,7 @@ abstract public class AbstractFederateSearchConnector implements FederateSearchC
      * @param remote result (with remote fieldnames)
      * @return SolrDocument with field names according to the YaCy schema
      */
-    protected URIMetadataNode toYaCySchema(final SolrDocument doc) {
+    protected URIMetadataNode toYaCySchema(final SolrDocument doc) throws MalformedURLException {
         // set YaCy id
         String urlstr;
         if (localcfg.contains("sku"))  {
@@ -156,7 +157,8 @@ abstract public class AbstractFederateSearchConnector implements FederateSearchC
             }
         }
 
-        URIMetadataNode newdoc = new URIMetadataNode(urlstr);
+        final DigestURL url = new DigestURL(urlstr);
+        URIMetadataNode newdoc = new URIMetadataNode(url);
         Iterator<Configuration.Entry> it = localcfg.entryIterator();
         while (it.hasNext()) {
             Configuration.Entry et = it.next();
diff --git a/source/net/yacy/cora/federate/SolrFederateSearchConnector.java b/source/net/yacy/cora/federate/SolrFederateSearchConnector.java
index 7e9fceaaa..1a134a0fb 100644
--- a/source/net/yacy/cora/federate/SolrFederateSearchConnector.java
+++ b/source/net/yacy/cora/federate/SolrFederateSearchConnector.java
@@ -20,6 +20,7 @@
 package net.yacy.cora.federate;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -101,8 +102,10 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector
                     SolrDocumentList docList = solrConnector.getDocumentListByParams(msp);
                     // convert to YaCy schema documentlist
                     for (SolrDocument doc : docList) {
-                        URIMetadataNode anew = toYaCySchema(doc);
-                        docs.add(anew);
+                        try {
+                            URIMetadataNode anew = toYaCySchema(doc);
+                            docs.add(anew);
+                        } catch (MalformedURLException ex) { }
                     }
                 } catch (IOException | SolrException e) {
                 } finally {
diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java
index 511e571ee..928ae92d2 100644
--- a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java
+++ b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java
@@ -90,18 +90,13 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
     private String alternative_urlname;
     private TextSnippet textSnippet = null;
 
-    public URIMetadataNode(final Properties prop, String collection) {
+    public URIMetadataNode(final Properties prop, String collection) throws MalformedURLException {
         // generates an plasmaLURLEntry using the properties from the argument
         // the property names must correspond to the one from toString
         //System.out.println("DEBUG-ENTRY: prop=" + prop.toString());
         super();
         final String urlRaw = crypt.simpleDecode(prop.getProperty("url", ""));
-        try {
-            url = new DigestURL(urlRaw);
-        } catch (final MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            this.url = null;
-        }
+        url = new DigestURL(urlRaw);
         String descr = crypt.simpleDecode(prop.getProperty("descr", "")); if (descr == null) descr = "";
         String dc_creator = crypt.simpleDecode(prop.getProperty("author", "")); if (dc_creator == null) dc_creator = "";
         String tags = crypt.simpleDecode(prop.getProperty("tags", "")); if (tags == null) tags = "";
@@ -161,7 +156,7 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
         }
     }
 
-    public URIMetadataNode(final SolrDocument doc) {
+    public URIMetadataNode(final SolrDocument doc) throws MalformedURLException {
         super();
         for (String name : doc.getFieldNames()) {
             this.addField(name, doc.getFieldValue(name));
@@ -170,31 +165,15 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
         this.score = scorex == null ? 0.0f : scorex.floatValue();
         final byte[] hash = ASCII.getBytes(getString(CollectionSchema.id)); // TODO: can we trust this id ?
         final String urlRaw = getString(CollectionSchema.sku);
-        try {
-            this.url = new DigestURL(urlRaw, hash);
-        } catch (final MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            this.url = null;
-        }
+        this.url = new DigestURL(urlRaw, hash);
     }
 
-    public URIMetadataNode(final SolrDocument doc, final WordReferenceVars searchedWord, final float scorex) {
+    public URIMetadataNode(final SolrDocument doc, final WordReferenceVars searchedWord, final float scorex) throws MalformedURLException {
         this(doc);
         this.word = searchedWord;
         this.score = scorex;
     }
 
-    public URIMetadataNode (final String urlstr) {
-        super();
-        try {
-            url = new DigestURL(urlstr);
-            this.setField(CollectionSchema.sku.name(), url.toNormalform(true));
-            this.setField(CollectionSchema.id.name(), ASCII.String(url.hash()));
-        } catch (final MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            this.url = null;
-        }
-    }
     public URIMetadataNode(DigestURL theurl) {
         super();
         url = theurl;
@@ -520,7 +499,7 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
         }
         try {
             return new URIMetadataNode(MapTools.s2p(propStr.substring(1, propStr.length() - 1)), collection);
-        } catch (final kelondroException e) {
+        } catch (final kelondroException | MalformedURLException e) {
             // wrong format
             ConcurrentLog.severe("URIMetadataNode", e.getMessage());
             return null;
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java
index bc7463c90..59198b53f 100644
--- a/source/net/yacy/peers/Protocol.java
+++ b/source/net/yacy/peers/Protocol.java
@@ -48,6 +48,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.InetAddress;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -1089,7 +1090,12 @@ public final class Protocol {
             if ( doc == null ) {
                 continue;
             }
-            URIMetadataNode urlEntry = new URIMetadataNode(doc);
+            URIMetadataNode urlEntry;
+            try {
+                urlEntry = new URIMetadataNode(doc);
+            } catch (MalformedURLException ex) {
+                continue;
+            }
 
             if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry.url()) ) {
                 if ( Network.log.isInfo() ) {
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index a61bc4ae1..ad269ff2c 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -165,6 +165,7 @@ import net.yacy.document.parser.pdfParser;
 import net.yacy.document.parser.html.Evaluation;
 import net.yacy.gui.Audio;
 import net.yacy.gui.Tray;
+import net.yacy.http.YaCyHttpServer;
 import net.yacy.kelondro.blob.BEncodedHeap;
 import net.yacy.kelondro.blob.Tables;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
@@ -215,7 +216,6 @@ import net.yacy.visualization.CircleTool;
 
 import com.google.common.io.Files;
 
-import net.yacy.http.YaCyHttpServer;
 
 
 public final class Switchboard extends serverSwitch {
@@ -2998,8 +2998,12 @@ public final class Switchboard extends serverSwitch {
             final Seed initiatorPeer = this.peers.getConnected(queueEntry.initiator());
             if ( initiatorPeer != null ) {
                 // start a thread for receipt sending to avoid a blocking here
-                SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry);
-                new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start();
+                try {
+                    SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry);
+                    new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start();
+                } catch (MalformedURLException ex) {
+                    this.log.info("malformed url: "+ex.getMessage());
+                }
             }
         }
     }