eleminate dependency on file-extension in storeDocument but use supported mime-type

to also support handling of urls w/o corresponding file-extension.
For this refactor use of document.getParserObject() to alway return a Parser (for clean logic)
and define/move the scraperObject as local var of AbstractParser.
Adjust related calls to getParserObject (where actually a scraperObject is wanted).
Addionally skip appending url token to parsed text for dht metadata entries 
(by default returned as result by rwi index).

source/net/yacy/document/AbstractParser.java

@@ -39,7 +39,8 @@ public abstract class AbstractParser implements Parser {
     protected final Set<String> SUPPORTED_MIME_TYPES = new LinkedHashSet<String>(); 
     protected final Set<String> SUPPORTED_EXTENSIONS = new HashSet<String>();
     private   final String name;
-
+    protected Object scraperObject; // used scraper or source object if any, otherwise null
+    
     /**
      * initialize a parser with a name
      * @param name

source/net/yacy/document/Document.java

@@ -92,13 +92,13 @@ public class Document {
     private final Set<String> languages;
     private boolean indexingDenied;
     private final double lon, lat;
-    private final Object parserObject; // the source object that was used to create the Document
+    private final Parser parserObject; // the source object that was used to create the Document
     private final Map<String, Set<String>> generic_facets; // a map from vocabulary names to the set of tags for that vocabulary which apply for this document
     private final Date lastModified;
     private int crawldepth;
 
     public Document(final DigestURL location, final String mimeType, final String charset,
-                    final Object parserObject,
+                    final Parser parserObject,
                     final Set<String> languages,
                     final String[] keywords,
                     final List<String> titles,
@@ -160,11 +160,29 @@ public class Document {
         if (contentDomain != ContentDomain.ALL) return contentDomain;
         return this.dc_source().getContentDomainFromExt();
     }
-    
+
-    public Object getParserObject() {
+    /**
+     * The parser used to generate the document
+     * @return Parser
+     */
+    public Parser getParserObject() {
         return this.parserObject;
     }
 
+    /**
+     * Confinient call to get the source/scraper object of the underlaying parser
+     * if the parser uses a scraper, like htmlParser
+     * @return scraper object typically of type ContentScraper but may also of type DCEntry
+     */
+    public Object getScraperObject() {
+        if (this.parserObject instanceof AbstractParser) {
+            if (((AbstractParser) this.parserObject).scraperObject != null) {
+                return ((AbstractParser) this.parserObject).scraperObject;
+            }
+        }
+        return null;
+    }
+    
     public Set<String> getContentLanguages() {
         return this.languages;
     }
@@ -931,9 +949,9 @@ dc_rights
 
         // clean up parser data
         for (final Document doc: docs) {
-            Object parserObject = doc.getParserObject();
+            Object scraper = doc.getScraperObject();
-            if (parserObject instanceof ContentScraper) {
+            if (scraper instanceof ContentScraper) {
-                final ContentScraper html = (ContentScraper) parserObject;
+                final ContentScraper html = (ContentScraper) scraper;
                 html.close();
             }
         }
@@ -979,9 +997,9 @@ dc_rights
                     if (!entry.getKey().attachedNofollow()) result.put(entry.getKey(), entry.getValue());
                 }
             }
-            final Object parser = d.getParserObject();
+            final Object scraper = d.getScraperObject();
-            if (parser instanceof ContentScraper) {
+            if (scraper instanceof ContentScraper) {
-                final ContentScraper html = (ContentScraper) parser;
+                final ContentScraper html = (ContentScraper) scraper;
                 String refresh = html.getRefreshPath();
                 if (refresh != null && refresh.length() > 0) try {result.put(new AnchorURL(refresh), "refresh");} catch (final MalformedURLException e) {}
                 AnchorURL canonical = html.getCanonical();

source/net/yacy/document/content/DCEntry.java

@@ -45,6 +45,8 @@ import net.yacy.cora.document.id.DigestURL;
 import net.yacy.cora.util.CommonPattern;
 import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.document.Document;
+import net.yacy.document.parser.genericParser;
+import net.yacy.search.schema.CollectionSchema;
 
 public class DCEntry extends MultiMapSolrParams {
 
@@ -330,11 +332,15 @@ public class DCEntry extends MultiMapSolrParams {
         languages.add(getLanguage());
         List<String> t = new ArrayList<String>(1);
         t.add(getTitle());
+        
+        // for processing during indexing, embed entry as source scraperObject in a standard parserobj object
+        genericParser parserobj = new genericParser(this); // init the simplest parser with DCEntry as source/scraperObject used during indexing
+
         return new Document(
             getIdentifier(true),
             "text/html",
             StandardCharsets.UTF_8.name(),
-            this,
+            parserobj,
             languages,
             getSubject(), // might be null
             t,
@@ -343,7 +349,7 @@ public class DCEntry extends MultiMapSolrParams {
             null,
             getDescriptions(),
             getLon(), getLat(),
-            get("text_t", ""),
+            get(CollectionSchema.text_t.name(), ""),
             null,
             null,
             null,

source/net/yacy/document/parser/genericParser.java

@@ -44,6 +44,17 @@ public class genericParser extends AbstractParser implements Parser {
         // this parser is used if no other fits. This parser fits all
     }
 
+    /**
+     * Constructor to allow to set a scraperObject
+     * because it is desired to keep the scraper/source object protected
+     * This is used for surrogate parsers to set a other source/scraper then ContentScraper
+     * @param scraper
+     */
+    public genericParser(Object scraper) {
+        super("Generic Parser");
+        this.scraperObject = scraper;
+    }
+
     @Override
     public Document[] parse(
             final DigestURL location,

source/net/yacy/document/parser/htmlParser.java

@@ -34,9 +34,7 @@ import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.StandardCharsets;
 import java.nio.charset.UnsupportedCharsetException;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
-import java.util.Set;
 
 import net.yacy.cora.document.encoding.UTF8;
 import net.yacy.cora.document.id.DigestURL;
@@ -60,21 +58,29 @@ public class htmlParser extends AbstractParser implements Parser {
 
     private static final int maxLinks = 10000;
 
-    public final static String[] htmlExtensions = new String[]{
-        "htm","html","shtml","shtm","stm","xhtml","phtml","phtm",
-        "tpl","php","php2","php3","php4","php5","cfm","asp","aspx","tex","txt","msg"
-        }; 
-    
-    public final static Set<String> htmlExtensionsSet;
-    
-    static {
-        htmlExtensionsSet = new HashSet<>(htmlExtensions.length);
-        for (String ext: htmlExtensions) htmlExtensionsSet.add(ext);
-    }
-    
     public htmlParser() {
         super("Streaming HTML Parser");
-        this.SUPPORTED_EXTENSIONS.addAll(htmlExtensionsSet);
+        this.SUPPORTED_EXTENSIONS.add("htm");
+        this.SUPPORTED_EXTENSIONS.add("html");
+        this.SUPPORTED_EXTENSIONS.add("shtml");
+        this.SUPPORTED_EXTENSIONS.add("shtm");
+        this.SUPPORTED_EXTENSIONS.add("stm");
+        this.SUPPORTED_EXTENSIONS.add("xhtml");
+        this.SUPPORTED_EXTENSIONS.add("phtml");
+        this.SUPPORTED_EXTENSIONS.add("phtm");
+        this.SUPPORTED_EXTENSIONS.add("tpl");
+        this.SUPPORTED_EXTENSIONS.add("php");
+        this.SUPPORTED_EXTENSIONS.add("php2");
+        this.SUPPORTED_EXTENSIONS.add("php3");
+        this.SUPPORTED_EXTENSIONS.add("php4");
+        this.SUPPORTED_EXTENSIONS.add("php5");
+        this.SUPPORTED_EXTENSIONS.add("cfm");
+        this.SUPPORTED_EXTENSIONS.add("asp");
+        this.SUPPORTED_EXTENSIONS.add("aspx");
+        this.SUPPORTED_EXTENSIONS.add("tex");
+        this.SUPPORTED_EXTENSIONS.add("txt");
+        this.SUPPORTED_EXTENSIONS.add("msg");
+
         this.SUPPORTED_MIME_TYPES.add("text/html");
         this.SUPPORTED_MIME_TYPES.add("text/xhtml+xml");
         this.SUPPORTED_MIME_TYPES.add("application/xhtml+xml");
@@ -97,7 +103,8 @@ public class htmlParser extends AbstractParser implements Parser {
         try {
             // first get a document from the parsed html
             Charset[] detectedcharsetcontainer = new Charset[]{null};
-            final ContentScraper scraper = parseToScraper(location, documentCharset, vocscraper, detectedcharsetcontainer, timezoneOffset, sourceStream, maxLinks);
+            scraperObject = parseToScraper(location, documentCharset, vocscraper, detectedcharsetcontainer, timezoneOffset, sourceStream, maxLinks);
+            ContentScraper scraper = (ContentScraper)scraperObject; // shortcut to access ContentScraper methodes
             // parseToScraper also detects/corrects/sets charset from html content tag
             final Document document = transformScraper(location, mimeType, detectedcharsetcontainer[0].name(), scraper);
             Document documentSnapshot = null;
@@ -130,7 +137,7 @@ public class htmlParser extends AbstractParser implements Parser {
      * @param scraper
      * @return
      */
-    private static Document transformScraper(final DigestURL location, final String mimeType, final String charSet, final ContentScraper scraper) {
+    private Document transformScraper(final DigestURL location, final String mimeType, final String charSet, final ContentScraper scraper) {
         final String[] sections = new String[
                  scraper.getHeadlines(1).length +
                  scraper.getHeadlines(2).length +
@@ -150,7 +157,7 @@ public class htmlParser extends AbstractParser implements Parser {
                 location,
                 mimeType,
                 charSet,
-                scraper,
+                this,
                 scraper.getContentLanguages(),
                 scraper.getKeywords(),
                 scraper.getTitles(),
@@ -178,7 +185,7 @@ public class htmlParser extends AbstractParser implements Parser {
         } catch (UnsupportedEncodingException e) {
             sourceStream = new ByteArrayInputStream(UTF8.getBytes(input));
         }
-        ContentScraper scraper;
+        ContentScraper scraper; // for this static methode no need to init local this.scraperObject
         try {
             scraper = parseToScraper(location, documentCharset, vocabularyScraper, detectedcharsetcontainer, timezoneOffset, sourceStream, maxLinks);
         } catch (Failure e) {
@@ -242,6 +249,7 @@ public class htmlParser extends AbstractParser implements Parser {
         }
         
         // parsing the content
+        // for this static methode no need to init local this.scraperObject here
         final ContentScraper scraper = new ContentScraper(location, maxLinks, vocabularyScraper, timezoneOffset);
         final TransformerWriter writer = new TransformerWriter(null,null,scraper,null,false, Math.max(64, Math.min(4096, sourceStream.available())));
         try {

source/net/yacy/document/parser/swfParser.java

@@ -68,10 +68,9 @@ public class swfParser extends AbstractParser implements Parser {
         try {
             final SWF2HTML swf2html = new SWF2HTML();
             String contents = "";
-            ContentScraper htmlscraper=null;
             try {
-            	contents = swf2html.convertSWFToHTML(source);
+                contents = swf2html.convertSWFToHTML(source);
-                htmlscraper =  htmlParser.parseToScraper(location, charset, scraper, timezoneOffset, contents, 100);
+                scraperObject = htmlParser.parseToScraper(location, charset, scraper, timezoneOffset, contents, 100);
             } catch (final NegativeArraySizeException e) {
                 throw new Parser.Failure(e.getMessage(), location);
             } catch (final IOException e) {
@@ -79,29 +78,9 @@ public class swfParser extends AbstractParser implements Parser {
             } catch (final Exception e) {
                 throw new Parser.Failure(e.getMessage(), location);
             }
-            /*
-            String url = null;
-            String urlnr = null;
-            final String linebreak = System.getProperty("line.separator");
-            final List<AnchorURL> anchors = new ArrayList<AnchorURL>();
-            int urls = 0;
-            int urlStart = -1;
-            int urlEnd = 0;
-            int p0 = 0;
 
-            //extracting urls
+            // As the result of parsing this function must return a plasmaParserDocument object
-            while ((urlStart = contents.indexOf("http://",urlEnd)) >= 0){
+            ContentScraper htmlscraper = (ContentScraper) this.scraperObject; // shortcut to access ContentScraper methodes
-                urlEnd = contents.indexOf(linebreak,urlStart);
-                url = contents.substring(urlStart,urlEnd);
-                urlnr = Integer.toString(++urls);
-                AnchorURL u = new AnchorURL(url);
-                u.setNameProperty(urlnr);
-                anchors.add(u);
-                contents = contents.substring(0,urlStart)+contents.substring(urlEnd);
-            }
-            */
-
-           // As the result of parsing this function must return a plasmaParserDocument object
             return new Document[]{new Document(
                 location, // url of the source document
                 mimeType, // the documents mime type

source/net/yacy/search/index/Segment.java

@@ -601,11 +601,14 @@ public class Segment {
                 crawlProfile != null && document.getDepth() <= crawlProfile.snapshotMaxdepth() &&
                 !crawlProfile.snapshotsMustnotmatch().matcher(urlNormalform).matches()) {
             // load pdf in case that is wanted. This can later be used to compute a web page preview in the search results
-            String ext = MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase();
+            Parser p = document.getParserObject();
-            if (ext.length() == 0 || url.getFile().length() <= 1 || htmlParser.htmlExtensionsSet.contains(ext)) {
+            boolean mimesupported = false;
+            if (p instanceof htmlParser)
+                    mimesupported = ((htmlParser)p).supportedMimeTypes().contains(document.dc_format());
+
+            if (mimesupported)
                 // STORE IMAGE AND METADATA
                 Transactions.store(vector, true, crawlProfile.snapshotLoadImage(), crawlProfile.snapshotReplaceold(), proxy, acceptLanguage);
-            }
         }
         
         // STORE TO SOLR

source/net/yacy/search/schema/CollectionConfiguration.java

@@ -335,29 +335,10 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
         if (allAttr || contains(CollectionSchema.audiolinkscount_i)) add(doc, CollectionSchema.audiolinkscount_i, md.laudio());
         if (allAttr || contains(CollectionSchema.videolinkscount_i)) add(doc, CollectionSchema.videolinkscount_i, md.lvideo());
         if (allAttr || contains(CollectionSchema.applinkscount_i)) add(doc, CollectionSchema.applinkscount_i, md.lapp());
-        if (allAttr || contains(CollectionSchema.text_t)) {
-        	// construct the text from other metadata parts.
-        	// This is necessary here since that is used to search the link when no other data (parsed text body) is available
-        	StringBuilder sb = new StringBuilder(120);
-        	// accText(sb, md.dc_title()); // default search field via getQueryFields(), not needed for snippet (always displayed)
-        	// accText(sb, md.dc_creator()); // author is in Default ranking/getQueryFields
-        	// accText(sb, md.dc_publisher()); // has it's own metadata field publisher_t (not part of default queryfields) and mostly N/A
-        	// accText(sb, md.snippet()); // above added to description_txt, default search field via getQueryFields(), description_txt incl. in snippet calculation
-        	accText(sb, md.url().toTokens());
-        	// accText(sb, keywords); // default search field via getQueryFields(), keywords not incl. in snippet calculation
-        	add(doc, CollectionSchema.text_t, sb.toString());
-        }
 
         return doc;
     }
 
-    private static void accText(final StringBuilder sb, String text) {
-    	if (text == null || text.length() == 0) return;
-    	if (sb.length() != 0) sb.append(' ');
-    	text = text.trim();
-    	if (!text.isEmpty() && text.charAt(text.length() - 1) == '.') sb.append(text); else sb.append(text).append('.');
-    }
-
     public static class Subgraph {
         public final ArrayList<String>[] urlProtocols, urlStubs, urlAnchorTexts;
         @SuppressWarnings("unchecked")
@@ -541,11 +522,11 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
 
         Subgraph subgraph = new Subgraph(inboundLinks.size(), outboundLinks.size());
         int c = 0;
-        final Object parser = document.getParserObject();
+        final Object scraper = document.getScraperObject();
         boolean containsCanonical = false;
         DigestURL canonical = null;
-        if (parser instanceof ContentScraper) {
+        if (scraper instanceof ContentScraper) {
-            final ContentScraper html = (ContentScraper) parser;
+            final ContentScraper html = (ContentScraper) scraper;
             List<ImageEntry> images = html.getImages();
 
             // header tags
@@ -885,9 +866,9 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
             }
         }
 
-        if (parser instanceof DCEntry) {
+        if (scraper instanceof DCEntry) {
             // the document was created with a surrogate parsing; overwrite all md: -entries to Solr
-            DCEntry dcentry = (DCEntry) parser;
+            DCEntry dcentry = (DCEntry) scraper;
             for (Map.Entry<String, String[]> entry: dcentry.getMap().entrySet()) {
                 String tag = entry.getKey();
                 if (!tag.startsWith("md:") || tag.length() < 4) continue;