bugfix to http://bugs.yacy.net/view.php?id=181

tried to make a bit less 'noise' to dns server

also included: less processes in snippet fetch to reduce load during
search on small computers

htroot/IndexControlURLs_p.java

@@ -131,7 +131,7 @@ public class IndexControlURLs_p {
         String urlhash = post.get("urlhash", "").trim();
         if (urlhash.length() == 0 && urlstring.length() > 0) {
             try {
-                urlhash = ASCII.String(new DigestURI(urlstring, null).hash());
+                urlhash = ASCII.String(new DigestURI(urlstring).hash());
             } catch (final MalformedURLException e) {
             }
         }

htroot/yacysearchitem.java

@@ -115,7 +115,7 @@ public class yacysearchitem {
 
             final int port = resultURL.getPort();
             DigestURI faviconURL = null;
-            if ((fileType == FileType.HTML || fileType == FileType.JSON) && !sb.isIntranetMode() && !resultURL.isLocal()) try {
+            if ((fileType == FileType.HTML || fileType == FileType.JSON) && !sb.isIntranetMode()) try {
                 faviconURL = new DigestURI(resultURL.getProtocol() + "://" + resultURL.getHost() + ((port != -1) ? (":" + port) : "") + "/favicon.ico");
             } catch (final MalformedURLException e1) {
                 Log.logException(e1);

source/de/anomic/crawler/CrawlStacker.java

@@ -211,7 +211,7 @@ public final class CrawlStacker {
                     u = u + "/index.html";
                 }
                 try {
-                    final byte[] uh = new DigestURI(u, null).hash();
+                    final byte[] uh = new DigestURI(u).hash();
                     this.indexSegment.urlMetadata().remove(uh);
                     this.nextQueue.noticeURL.removeByURLHash(uh);
                     this.nextQueue.errorURL.remove(uh);

source/de/anomic/data/ymark/YMarkUtil.java

@@ -46,7 +46,7 @@ public class YMarkUtil {
      * @see net.yacy.kelondro.data.meta.DigestURI.DigestURI(String url, byte[] hash).hash()
      */
     public final static byte[] getBookmarkId(String url) throws MalformedURLException {
-		return (new DigestURI(url, null)).hash();
+		return (new DigestURI(url)).hash();
     }
 
     /**

source/net/yacy/kelondro/data/meta/DigestURI.java

@@ -116,7 +116,8 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
      * DigestURI from URI string
      */
     public DigestURI(final String url) throws MalformedURLException {
-        this(url, null);
+        super(url);
+        this.hash = null;
     }
 
     /**

source/net/yacy/peers/graphics/WebStructureGraph.java

@@ -157,9 +157,6 @@ public class WebStructureGraph
         final Document document,
         final Condenser condenser) {
         // generate citation reference
-        if ( url.isLocal() ) {
-            return; // we do this only for global urls
-        }
         final Map<MultiProtocolURI, String> hl = document.getHyperlinks();
         final Iterator<MultiProtocolURI> it = hl.keySet().iterator();
         final HashSet<MultiProtocolURI> globalRefURLs = new HashSet<MultiProtocolURI>();

source/net/yacy/search/Switchboard.java

@@ -2422,7 +2422,7 @@ public final class Switchboard extends serverSwitch
             // send the documents to solr
             for ( final Document doc : in.documents ) {
                 try {
-                    final String id = UTF8.String(new DigestURI(doc.dc_identifier(), null).hash());
+                    final String id = UTF8.String(new DigestURI(doc.dc_identifier()).hash());
                     final String iquh = UTF8.String(in.queueEntry.url().hash());
                     if ( !id.equals(iquh) ) {
                         this.log.logWarning("condenseDocument consistency check doc="

source/net/yacy/search/query/SnippetProcess.java

@@ -53,6 +53,8 @@ import net.yacy.peers.graphics.ProfilingGraph;
 import net.yacy.repository.LoaderDispatcher;
 import net.yacy.search.EventTracker;
 import net.yacy.search.Switchboard;
+import net.yacy.search.index.Segment;
+import net.yacy.search.index.SolrField;
 import net.yacy.search.snippet.MediaSnippet;
 import net.yacy.search.snippet.ResultEntry;
 import net.yacy.search.snippet.TextSnippet;
@@ -62,11 +64,11 @@ import org.apache.solr.common.SolrDocumentList;
 
 import de.anomic.crawler.Cache;
 import de.anomic.data.WorkTables;
-import net.yacy.search.index.Segment;
-import net.yacy.search.index.SolrField;
 
 public class SnippetProcess {
 
+    private final static int SNIPPET_WORKER_THREADS = Math.max(4, Runtime.getRuntime().availableProcessors() * 2);
+
     // input values
     final RWIProcess  rankingProcess; // ordered search results, grows dynamically as all the query threads enrich this container
     QueryParams     query;
@@ -126,7 +128,7 @@ public class SnippetProcess {
 
         // start worker threads to fetch urls and snippets
         this.workerThreads = null;
-        deployWorker(Math.min(10, query.itemsPerPage), query.neededResults());
+        deployWorker(Math.min(SNIPPET_WORKER_THREADS, query.itemsPerPage), query.neededResults());
         EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(query.id(true), SearchEvent.Type.SNIPPETFETCH_START, ((this.workerThreads == null) ? "no" : this.workerThreads.length) + " online snippet fetch threads started", 0, 0), false);
     }
 
@@ -193,8 +195,8 @@ public class SnippetProcess {
 
             // deploy worker to get more results
             if (!anyWorkerAlive()) {
-                final int neededInclPrefetch = this.query.neededResults() + ((MemoryControl.available() > 100 * 1024 * 1024) ? this.query.itemsPerPage : 0);
-                deployWorker(Math.min(20, this.query.itemsPerPage), neededInclPrefetch);
+                final int neededInclPrefetch = this.query.neededResults() + ((MemoryControl.available() > 100 * 1024 * 1024 && SNIPPET_WORKER_THREADS >= 8) ? this.query.itemsPerPage : 0);
+                deployWorker(Math.min(SNIPPET_WORKER_THREADS, this.query.itemsPerPage), neededInclPrefetch);
             }
 
             try {entry = this.result.element(item, 50);} catch (final InterruptedException e) {break;}