Merge branch 'master' of https://github.com/yacy/yacy_search_server

htroot/Crawler_p.java

@@ -219,8 +219,8 @@ public class Crawler_p {
                 String newcrawlingMustMatch = post.get("mustmatch", CrawlProfile.MATCH_ALL_STRING);
                 String newcrawlingMustNotMatch = post.get("mustnotmatch", CrawlProfile.MATCH_NEVER_STRING);
                 if (newcrawlingMustMatch.length() < 2) newcrawlingMustMatch = CrawlProfile.MATCH_ALL_STRING; // avoid that all urls are filtered out if bad value was submitted
-                final boolean fullDomain = "domain".equals(post.get("range", "wide")); // special property in simple crawl start
+                boolean fullDomain = "domain".equals(post.get("range", "wide")); // special property in simple crawl start
-                final boolean subPath    = "subpath".equals(post.get("range", "wide")); // special property in simple crawl start
+                boolean subPath    = "subpath".equals(post.get("range", "wide")); // special property in simple crawl start
 
                 final boolean restrictedcrawl = fullDomain || subPath || !CrawlProfile.MATCH_ALL_STRING.equals(newcrawlingMustMatch);
                 final boolean deleteage = restrictedcrawl && "age".equals(post.get("deleteold","off"));
@@ -261,6 +261,10 @@ public class Crawler_p {
                     if (p >= 8) crawlName = crawlName.substring(0, p);
                 }
                 if (crawlName.length() == 0 && sitemapURLStr.length() > 0) crawlName = "sitemap loader for " + sitemapURLStr;
+                // in case that a root url has a file protocol, then the site filter does not work, patch that:
+                if (fullDomain) {
+                    for (DigestURL u: rootURLs) if (u.isFile()) {fullDomain = false; subPath = true; break;}
+                }
                 
                 // delete old robots entries
                 for (DigestURL ru : rootURLs) {

source/net/yacy/cora/federate/AbstractFederateSearchConnector.java

@@ -21,6 +21,7 @@ package net.yacy.cora.federate;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -143,7 +144,7 @@ abstract public class AbstractFederateSearchConnector implements FederateSearchC
      * @param remote result (with remote fieldnames)
      * @return SolrDocument with field names according to the YaCy schema
      */
-    protected URIMetadataNode toYaCySchema(final SolrDocument doc) {
+    protected URIMetadataNode toYaCySchema(final SolrDocument doc) throws MalformedURLException {
         // set YaCy id
         String urlstr;
         if (localcfg.contains("sku"))  {
@@ -156,7 +157,8 @@ abstract public class AbstractFederateSearchConnector implements FederateSearchC
             }
         }
 
-        URIMetadataNode newdoc = new URIMetadataNode(urlstr);
+        final DigestURL url = new DigestURL(urlstr);
+        URIMetadataNode newdoc = new URIMetadataNode(url);
         Iterator<Configuration.Entry> it = localcfg.entryIterator();
         while (it.hasNext()) {
             Configuration.Entry et = it.next();

source/net/yacy/cora/federate/SolrFederateSearchConnector.java

@@ -20,6 +20,7 @@
 package net.yacy.cora.federate;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -101,8 +102,10 @@ public class SolrFederateSearchConnector extends AbstractFederateSearchConnector
                     SolrDocumentList docList = solrConnector.getDocumentListByParams(msp);
                     // convert to YaCy schema documentlist
                     for (SolrDocument doc : docList) {
-                        URIMetadataNode anew = toYaCySchema(doc);
+                        try {
-                        docs.add(anew);
+                            URIMetadataNode anew = toYaCySchema(doc);
+                            docs.add(anew);
+                        } catch (MalformedURLException ex) { }
                     }
                 } catch (IOException | SolrException e) {
                 } finally {

source/net/yacy/document/parser/docParser.java

@@ -29,7 +29,6 @@ package net.yacy.document.parser;
 
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 
 import net.yacy.cora.document.id.AnchorURL;
@@ -110,24 +109,25 @@ public class docParser extends AbstractParser implements Parser {
 
         Document[] docs;
         docs = new Document[]{new Document(
-                  location,
+            location,
-                  mimeType,
+            mimeType,
-                  "UTF-8",
+            "UTF-8",
-                  this,
+            this,
-                  null,
+            null,
-                  keywlist,
+            keywlist,
-                  singleList(title),
+            singleList(title),
-                  extractor.getSummaryInformation().getAuthor(), // constuctor can handle null
+            extractor.getSummaryInformation().getAuthor(), // constuctor can handle null
-                  extractor.getDocSummaryInformation().getCompany(), // publisher
+            extractor.getDocSummaryInformation().getCompany(), // publisher
-                  null,
+            null,
-                  descriptions,
+            descriptions,
-                  0.0f, 0.0f,
+            0.0f, 0.0f,
-                  contents.toString(),
+            contents.toString(),
-                  null,
+            null,
-                  null,
+            null,
-                  null,
+            null,
-                  false,
+            false,
-                  new Date())};
+            extractor.getSummaryInformation().getLastSaveDateTime() // maybe null
+            )};
 
         return docs;
     }

source/net/yacy/document/parser/odtParser.java

@@ -120,6 +120,7 @@ public class odtParser extends AbstractParser implements Parser {
             String docLongTitle   = null;
             String docAuthor      = null;
             String docLanguage    = null;
+            Date docModified    = null;
 
             // opening the file as zip file
             final ZipFile zipFile = new ZipFile(dest);
@@ -160,6 +161,7 @@ public class odtParser extends AbstractParser implements Parser {
                     docLongTitle   = metaData.getSubject();
                     docAuthor      = metaData.getCreator();
                     docLanguage    = metaData.getLanguage();
+                    docModified    = metaData.getLastModified(); // maybe null
                 }
             }
 
@@ -201,7 +203,7 @@ public class odtParser extends AbstractParser implements Parser {
                     null,
                     null,
                     false,
-                    new Date()
+                    docModified
                     )};
             return docs;
         } catch (final Exception e) {

source/net/yacy/document/parser/ooxmlParser.java

@@ -102,6 +102,7 @@ public class ooxmlParser extends AbstractParser implements Parser {
             String docLongTitle   = null;
             String docAuthor      = null;
             String docLanguage    = null;
+            Date docModified      = null;
 
             // opening the file as zip file
             final ZipFile zipFile= new ZipFile(dest);
@@ -145,6 +146,7 @@ public class ooxmlParser extends AbstractParser implements Parser {
                     docLongTitle   = metaData.getSubject();
                     docAuthor      = metaData.getCreator();
                     docLanguage    = metaData.getLanguage();
+                    docModified    = metaData.getLastModified();
                 }
             }
 
@@ -185,7 +187,7 @@ public class ooxmlParser extends AbstractParser implements Parser {
                     null,
                     null,
                     false,
-                    new Date())};
+                    docModified)};
             return docs;
         } catch (final Exception e) {
             if (e instanceof InterruptedException) throw (InterruptedException) e;

source/net/yacy/document/parser/pptParser.java

@@ -30,7 +30,6 @@ package net.yacy.document.parser;
 import java.io.BufferedInputStream;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 
 import net.yacy.cora.document.id.AnchorURL;
@@ -103,24 +102,25 @@ public class pptParser extends AbstractParser implements Parser {
              * and set shortText and bodyText properly
              */
             final Document[] docs = new Document[]{new Document(
-                    location,
+                location,
-                    mimeType,
+                mimeType,
-                    "UTF-8",
+                "UTF-8",
-                    this,
+                this,
-                    null,
+                null,
-                    keywlist,
+                keywlist,
-                    singleList(title),
+                singleList(title),
-                    pptExtractor.getSummaryInformation().getAuthor(), // may be null
+                pptExtractor.getSummaryInformation().getAuthor(), // may be null
-                    pptExtractor.getDocSummaryInformation().getCompany(),
+                pptExtractor.getDocSummaryInformation().getCompany(),
-                    null,
+                null,
-                    descriptions,
+                descriptions,
-                    0.0f, 0.0f,
+                0.0f, 0.0f,
-                    contents,
+                contents,
-                    null,
+                null,
-                    null,
+                null,
-                    null,
+                null,
-                    false,
+                false,
-                    new Date())};
+                pptExtractor.getSummaryInformation().getLastSaveDateTime() // may be null
+                )};
             return docs;
         } catch (final Exception e) {
             if (e instanceof InterruptedException) throw (InterruptedException) e;

source/net/yacy/document/parser/vcfParser.java

@@ -32,6 +32,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.MalformedURLException;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -39,6 +40,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 
+import net.yacy.cora.date.ISO8601Formatter;
 import net.yacy.cora.document.encoding.UTF8;
 import net.yacy.cora.document.id.AnchorURL;
 import net.yacy.cora.order.Base64Order;
@@ -81,6 +83,7 @@ public class vcfParser extends AbstractParser implements Parser {
             final HashMap<String, String> parsedData = new HashMap<String, String>();
             final List<AnchorURL> anchors = new ArrayList<AnchorURL>();
             final LinkedList<String> parsedNames = new LinkedList<String>();
+            Date revDate = null; // rev=modified date
 
             boolean useLastLine = false;
             int lineNr = 0;
@@ -183,6 +186,13 @@ public class vcfParser extends AbstractParser implements Parser {
                             parsedDataText.append(value).append("\r\n");
                         }
                         parsedDataText.append("\r\n");
+                        // get specific meta data from parsed key-value
+                        value = parsedData.get("REV"); // modified date
+                        if (value != null && !value.isEmpty()) {
+                            try {
+                                revDate = ISO8601Formatter.FORMATTER.parse(value, 0).getTime();
+                            } catch(ParseException ex){ }
+                        }
                         parsedData.clear();
                     } else if (key.toUpperCase().startsWith("URL")) {
                         try {
@@ -235,7 +245,7 @@ public class vcfParser extends AbstractParser implements Parser {
                     null,
                     null,                       // a treeset of image URLs
                     false,
-                    new Date())};
+                    revDate)};                   // modified date
         } catch (final Exception e) {
             if (e instanceof InterruptedException) throw (InterruptedException) e;
             if (e instanceof Parser.Failure) throw (Parser.Failure) e;

source/net/yacy/document/parser/xml/ODMetaHandler.java

@@ -26,6 +26,9 @@
 
 package net.yacy.document.parser.xml;
 
+import java.text.ParseException;
+import java.util.Date;
+import net.yacy.cora.date.ISO8601Formatter;
 import org.xml.sax.Attributes;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
@@ -39,6 +42,7 @@ public class ODMetaHandler extends DefaultHandler {
 	private String docSubject = null;
 	private String docTitle = null;
 	private String docDescription = null;
+        private String docLastmodified = null;
 	
 	public ODMetaHandler() {
 	}
@@ -67,7 +71,9 @@ public class ODMetaHandler extends DefaultHandler {
 		this.docTitle  = buffer.toString();
 	    } else if ("dc:description".equals(tag)) {
 		this.docDescription  = buffer.toString();
-	    }
+	    } else if ("dcterms:modified".equals(tag) || "dc:date".equals(tag)) { // Microsoft uses <dcterms:modified>, OpenOffice <dc:date>
+                this.docLastmodified = buffer.toString();
+            }
 	}
 
 	public String getCreator() {
@@ -89,5 +95,24 @@ public class ODMetaHandler extends DefaultHandler {
 	public String getDescription() {
 	    return docDescription;
 	}
+
+    /**
+     * get the last modification date of the document
+     *
+     * @return date or null
+     */
+    public Date getLastModified() {
+        Date d;
+        if (docLastmodified != null && !docLastmodified.isEmpty()) {
+            try {
+                d = ISO8601Formatter.FORMATTER.parse(this.docLastmodified, 0).getTime();
+            } catch (ParseException ex) {
+                d = null;
+            }
+        } else {
+            d = null;
+        }
+        return d;
+    }
 }
 

source/net/yacy/kelondro/data/meta/URIMetadataNode.java

@@ -90,18 +90,13 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
     private String alternative_urlname;
     private TextSnippet textSnippet = null;
 
-    public URIMetadataNode(final Properties prop, String collection) {
+    public URIMetadataNode(final Properties prop, String collection) throws MalformedURLException {
         // generates an plasmaLURLEntry using the properties from the argument
         // the property names must correspond to the one from toString
         //System.out.println("DEBUG-ENTRY: prop=" + prop.toString());
         super();
         final String urlRaw = crypt.simpleDecode(prop.getProperty("url", ""));
-        try {
+        url = new DigestURL(urlRaw);
-            url = new DigestURL(urlRaw);
-        } catch (final MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            this.url = null;
-        }
         String descr = crypt.simpleDecode(prop.getProperty("descr", "")); if (descr == null) descr = "";
         String dc_creator = crypt.simpleDecode(prop.getProperty("author", "")); if (dc_creator == null) dc_creator = "";
         String tags = crypt.simpleDecode(prop.getProperty("tags", "")); if (tags == null) tags = "";
@@ -161,7 +156,7 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
         }
     }
 
-    public URIMetadataNode(final SolrDocument doc) {
+    public URIMetadataNode(final SolrDocument doc) throws MalformedURLException {
         super();
         for (String name : doc.getFieldNames()) {
             this.addField(name, doc.getFieldValue(name));
@@ -170,31 +165,15 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
         this.score = scorex == null ? 0.0f : scorex.floatValue();
         final byte[] hash = ASCII.getBytes(getString(CollectionSchema.id)); // TODO: can we trust this id ?
         final String urlRaw = getString(CollectionSchema.sku);
-        try {
+        this.url = new DigestURL(urlRaw, hash);
-            this.url = new DigestURL(urlRaw, hash);
-        } catch (final MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            this.url = null;
-        }
     }
 
-    public URIMetadataNode(final SolrDocument doc, final WordReferenceVars searchedWord, final float scorex) {
+    public URIMetadataNode(final SolrDocument doc, final WordReferenceVars searchedWord, final float scorex) throws MalformedURLException {
         this(doc);
         this.word = searchedWord;
         this.score = scorex;
     }
 
-    public URIMetadataNode (final String urlstr) {
-        super();
-        try {
-            url = new DigestURL(urlstr);
-            this.setField(CollectionSchema.sku.name(), url.toNormalform(true));
-            this.setField(CollectionSchema.id.name(), ASCII.String(url.hash()));
-        } catch (final MalformedURLException e) {
-            ConcurrentLog.logException(e);
-            this.url = null;
-        }
-    }
     public URIMetadataNode(DigestURL theurl) {
         super();
         url = theurl;
@@ -520,7 +499,7 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
         }
         try {
             return new URIMetadataNode(MapTools.s2p(propStr.substring(1, propStr.length() - 1)), collection);
-        } catch (final kelondroException e) {
+        } catch (final kelondroException | MalformedURLException e) {
             // wrong format
             ConcurrentLog.severe("URIMetadataNode", e.getMessage());
             return null;

source/net/yacy/kelondro/table/Table.java

@@ -232,7 +232,7 @@ public class Table implements Index, Iterable<Row.Entry> {
         }
 
         // track this table
-        tableTracker.put(tablefile.toString(), this);
+        synchronized (tableTracker) {tableTracker.put(tablefile.toString(), this);}
     }
 
     public synchronized void warmUp() {

source/net/yacy/peers/Protocol.java

@@ -48,6 +48,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.InetAddress;
+import java.net.MalformedURLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -1089,7 +1090,12 @@ public final class Protocol {
             if ( doc == null ) {
                 continue;
             }
-            URIMetadataNode urlEntry = new URIMetadataNode(doc);
+            URIMetadataNode urlEntry;
+            try {
+                urlEntry = new URIMetadataNode(doc);
+            } catch (MalformedURLException ex) {
+                continue;
+            }
 
             if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry.url()) ) {
                 if ( Network.log.isInfo() ) {

source/net/yacy/search/Switchboard.java

@@ -165,6 +165,7 @@ import net.yacy.document.parser.pdfParser;
 import net.yacy.document.parser.html.Evaluation;
 import net.yacy.gui.Audio;
 import net.yacy.gui.Tray;
+import net.yacy.http.YaCyHttpServer;
 import net.yacy.kelondro.blob.BEncodedHeap;
 import net.yacy.kelondro.blob.Tables;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
@@ -215,7 +216,6 @@ import net.yacy.visualization.CircleTool;
 
 import com.google.common.io.Files;
 
-import net.yacy.http.YaCyHttpServer;
 
 
 public final class Switchboard extends serverSwitch {
@@ -2998,8 +2998,12 @@ public final class Switchboard extends serverSwitch {
             final Seed initiatorPeer = this.peers.getConnected(queueEntry.initiator());
             if ( initiatorPeer != null ) {
                 // start a thread for receipt sending to avoid a blocking here
-                SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry);
+                try {
-                new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start();
+                    SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry);
+                    new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start();
+                } catch (MalformedURLException ex) {
+                    this.log.info("malformed url: "+ex.getMessage());
+                }
             }
         }
     }

source/net/yacy/search/query/QueryParams.java

@@ -205,7 +205,7 @@ public final class QueryParams {
             String protocolfilter = modifier.protocol == null ? ".*" : modifier.protocol;
             String defaulthostprefix = modifier.protocol == null ? "www" : modifier.protocol;
             String hostfilter = modifier.sitehost == null && tld == null ? ".*" : modifier.sitehost == null ? ".*\\." + tld : modifier.sitehost.startsWith(defaulthostprefix + ".") ? "(" + defaulthostprefix + "\\.)?" + modifier.sitehost.substring(4) : "(" + defaulthostprefix + "\\.)?" + modifier.sitehost;
-            String filefilter = modifier.filetype == null ? ".*" : ".*" + modifier.filetype + ".*";
+            String filefilter = modifier.filetype == null ? ".*" : ".*" + modifier.filetype + ".*"; // TODO: should be ".ext" but while/comment above suggests not -> add filetype contrain pullOneFilteredFromRWI()
             String filter = protocolfilter + "..." + hostfilter + "." + filefilter;
             if (!filter.equals(".*....*..*")) {
                 Pattern r = Pattern.compile("(\\.|(\\.\\*))\\.\\*");

source/net/yacy/search/query/SearchEvent.java

@@ -1159,6 +1159,13 @@ public final class SearchEvent {
                 continue;
             }
 
+            // check modifier constraint filetype (using fileextension)
+            if (this.query.modifier.filetype != null && !this.query.modifier.filetype.equals(ext)) {
+                if (log.isFine()) log.fine("dropped RWI: file type constraint = " + this.query.modifier.filetype);
+                if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
+                continue;
+            }
+
             // check modifier constraint (language)
             // TODO: : page.language() never null but defaults to "en" (may cause false drop of result)
             if (this.query.modifier.language != null && !this.query.modifier.language.equals(page.language())) {