Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Conflicts:
	htroot/PerformanceMemory_p.java

htroot/ConfigUpdate_p.html

@@ -114,8 +114,9 @@
       </dl>
       </fieldset>
       ::
-      You installed YaCy with a package manager. To update YaCy, use the package manager:<p>
-      Debian: apt-get update &amp;&amp; apt-get install yacy</p>
+      <p>You installed YaCy with a package manager. To update YaCy, use the package manager:</p>
+      <p>manual update:<br/>apt-get update &amp;&amp; apt-get install yacy</p>
+      <p>automatic update: add the following line to /etc/crontab<br/>0 6 * * * root apt-get update &amp;&amp; apt-get -y --force-yes install yacy</p>
       ::
     #(/candeploy)#
     

htroot/PerformanceMemory_p.java

@@ -114,7 +114,7 @@ public class PerformanceMemory_p {
             totalmem += mem;
             prop.put("EcoList_" + c + "_tableKeyMem", Formatter.bytesToString(mem));
             prop.put("EcoList_" + c + "_tableKeyChunkSize", mapx.get(Table.StatKeys.tableKeyChunkSize));
-
+
             v = mapx.get(Table.StatKeys.tableValueMem);
             mem = v == null ? 0 : Long.parseLong(v);
             totalmem += mem;

htroot/sharedBlacklist_p.java

@@ -210,7 +210,6 @@ public class sharedBlacklist_p {
                 prop.put("page", "1"); //result page
                 prop.put("status", STATUS_ENTRIES_ADDED); //list of added Entries
 
-                int count = 0;//couter of added entries
                 PrintWriter pw = null;
                 try {
                     // open the blacklist file
@@ -238,7 +237,6 @@ public class sharedBlacklist_p {
                             // append the item to the file
                             pw.println(newItem);
 
-                            count++;
                             if (Switchboard.urlBlacklist != null) {
                                 final String supportedBlacklistTypesStr = Blacklist.BLACKLIST_TYPES_STRING;
                                 final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");

htroot/yacyinteractive.java

@@ -27,6 +27,7 @@
 import net.yacy.cora.protocol.RequestHeader;
 import net.yacy.search.Switchboard;
 import net.yacy.search.SwitchboardConstants;
+import net.yacy.search.ranking.BlockRank;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
@@ -37,6 +38,8 @@ public class yacyinteractive {
     public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
         final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
+        BlockRank.ensureLoaded();
+        
         prop.put("topmenu", sb.getConfigBool("publicTopmenu", true) ? 1 : 0);
         final String promoteSearchPageGreeting =
                 (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ?

htroot/yacysearch.java

@@ -78,6 +78,7 @@ import net.yacy.search.query.AccessTracker;
 import net.yacy.search.query.QueryParams;
 import net.yacy.search.query.SearchEvent;
 import net.yacy.search.query.SearchEventCache;
+import net.yacy.search.ranking.BlockRank;
 import net.yacy.search.ranking.RankingProfile;
 import de.anomic.data.DidYouMean;
 import de.anomic.data.UserDB;
@@ -96,6 +97,7 @@ public class yacysearch {
         final Switchboard sb = (Switchboard) env;
         sb.localSearchLastAccess = System.currentTimeMillis();
 
+        BlockRank.ensureLoaded();
         final boolean searchAllowed =
             sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
 

htroot/yacysearch_location.java

@@ -31,6 +31,7 @@ import net.yacy.document.LibraryProvider;
 import net.yacy.document.geolocalization.GeoLocation;
 import net.yacy.search.Switchboard;
 import net.yacy.search.SwitchboardConstants;
+import net.yacy.search.ranking.BlockRank;
 import de.anomic.server.serverCore;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -40,7 +41,8 @@ public class yacysearch_location {
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
         final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
-
+        BlockRank.ensureLoaded();
+        
         prop.put("kml", 0);
 
         if (header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("kml") ||

source/de/anomic/crawler/Cache.java

@@ -143,6 +143,10 @@ public final class Cache {
         }
     }
 
+    public static void commit() {
+    	fileDB.flushAll();
+    }
+    
     /**
      * clear the cache
      */

source/de/anomic/crawler/CrawlProfile.java

@@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 
 import de.anomic.server.serverObjects;
-import de.anomic.server.servletProperties;
 
 import net.yacy.cora.document.ASCII;
 import net.yacy.cora.document.MultiProtocolURI;

source/de/anomic/crawler/CrawlStacker.java

@@ -67,7 +67,6 @@ public final class CrawlStacker {
     private final Log log = new Log("STACKCRAWL");
 
     private final WorkflowProcessor<Request>  fastQueue, slowQueue;
-    private long                    dnsMiss;
     private final CrawlQueues       nextQueue;
     private final CrawlSwitchboard  crawler;
     private final Segment           indexSegment;
@@ -89,8 +88,6 @@ public final class CrawlStacker {
         this.crawler = cs;
         this.indexSegment = indexSegment;
         this.peers = peers;
-        //this.dnsHit = 0;
-        this.dnsMiss = 0;
         this.acceptLocalURLs = acceptLocalURLs;
         this.acceptGlobalURLs = acceptGlobalURLs;
         this.domainList = domainList;
@@ -179,7 +176,6 @@ public final class CrawlStacker {
         } else {
             try {
                 this.slowQueue.enQueue(entry);
-                this.dnsMiss++;
             } catch (final InterruptedException e) {
                 Log.logException(e);
             }

source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java

@@ -25,6 +25,7 @@
 
 package net.yacy.cora.sorting;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -39,10 +40,11 @@ import java.util.concurrent.TimeUnit;
  * If the queue gets larger that the given maxsize, then elements from the tail of the queue
  * are drained (deleted).
  */
-public class WeakPriorityBlockingQueue<E> {
+public class WeakPriorityBlockingQueue<E> implements Serializable {
 
+	private static final long serialVersionUID = 4573442576760691887L;
 
-    private final TreeSet<Element<E>>   queue;    // object within the stack, ordered using a TreeSet
+	private final TreeSet<Element<E>>   queue;    // object within the stack, ordered using a TreeSet
     private final Semaphore    enqueued; // semaphore for elements in the stack
     private final ArrayList<Element<E>> drained;  // objects that had been on the stack but had been removed
     protected int maxsize;
@@ -281,7 +283,7 @@ public class WeakPriorityBlockingQueue<E> {
         return this.drained.iterator();
     }
 
-    public interface Element<E> {
+    public interface Element<E> extends Serializable {
         public long getWeight();
         public E getElement();
         public boolean equals(Element<E> o);
@@ -291,9 +293,11 @@ public class WeakPriorityBlockingQueue<E> {
         public String toString();
     }
 
-    protected abstract static class AbstractElement<E> implements Element<E> {
+    protected abstract static class AbstractElement<E> implements Element<E>, Serializable {
 
-        public long weight;
+		private static final long serialVersionUID = -7026597258248026566L;
+
+		public long weight;
         public E element;
 
         @Override
@@ -328,7 +332,9 @@ public class WeakPriorityBlockingQueue<E> {
      */
     public static class NaturalElement<E> extends AbstractElement<E> implements Element<E>, Comparable<NaturalElement<E>>, Comparator<NaturalElement<E>> {
 
-        public NaturalElement(final E element, final long weight) {
+		private static final long serialVersionUID = 6816543012966928794L;
+
+		public NaturalElement(final E element, final long weight) {
             this.element = element;
             this.weight = weight;
         }
@@ -359,7 +365,9 @@ public class WeakPriorityBlockingQueue<E> {
      */
     public static class ReverseElement<E> extends AbstractElement<E> implements Element<E>, Comparable<ReverseElement<E>>, Comparator<ReverseElement<E>> {
 
-        public ReverseElement(final E element, final long weight) {
+		private static final long serialVersionUID = -8166724491837508921L;
+
+		public ReverseElement(final E element, final long weight) {
             this.element = element;
             this.weight = weight;
         }

source/net/yacy/document/content/dao/ImportDump.java

@@ -59,7 +59,6 @@ public class ImportDump {
         	FileUtils.copy(dump, baos);
         	
         	String s = UTF8.String(baos.toByteArray());
-        	int batchSize = 0;
         	int p, q;
         	String t;
         	loop: while (s.length() > 0) {
@@ -78,11 +77,9 @@ public class ImportDump {
         		s = s.substring(p);
         		//if (batchSize + t.length() >= maxBatch) {
         			statement.executeBatch();
-        			batchSize = 0;
         		//}
     			System.out.println(t);
         		statement.addBatch(t);
-        		batchSize += t.length();
         	}
         	statement.executeBatch();
         } catch (SQLException e) {

source/net/yacy/document/importer/OAIListFriendsLoader.java

@@ -27,6 +27,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Serializable;
 import java.net.MalformedURLException;
 import java.util.HashMap;
 import java.util.Map;
@@ -51,7 +52,7 @@ import org.xml.sax.helpers.DefaultHandler;
 
 import de.anomic.crawler.retrieval.Response;
 
-public class OAIListFriendsLoader {
+public class OAIListFriendsLoader implements Serializable {
 
     private static final long serialVersionUID = -8705115274655024604L;
 
@@ -117,7 +118,7 @@ public class OAIListFriendsLoader {
     }
     
     // get a resumption token using a SAX xml parser from am input stream
-    private static class Parser extends DefaultHandler {
+    public static class Parser extends DefaultHandler {
 
         // class variables
         private final StringBuilder buffer;
@@ -161,6 +162,10 @@ public class OAIListFriendsLoader {
          <baseURL id="http://roar.eprints.org/id/eprint/1064">http://oai.repec.openlib.org/</baseURL>
          </BaseURLs>
          */
+        
+        public int getCounter() {
+        	return this.recordCounter;
+        }
 
         public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
             if ("baseURL".equals(tag)) {

source/net/yacy/kelondro/blob/Compressor.java

@@ -335,7 +335,7 @@ public class Compressor implements BLOB, Iterable<byte[]> {
         }
     }
 
-    private void flushAll() {
+    public void flushAll() {
         while (!this.buffer.isEmpty()) {
             if (!flushOne()) break;
         }

source/net/yacy/kelondro/blob/MapColumnIndex.java

@@ -25,6 +25,7 @@
 
 package net.yacy.kelondro.blob;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -38,7 +39,7 @@ import net.yacy.kelondro.order.NaturalOrder;
 /**
  * a mapping from a column name to maps with the value of the columns to the primary keys where the entry exist in the table
  */
-public class MapColumnIndex {
+public class MapColumnIndex implements Serializable {
 
     private static final long serialVersionUID=-424741536889467566L;
 

source/net/yacy/kelondro/util/ByteArray.java

@@ -44,12 +44,10 @@ import net.yacy.cora.order.ByteOrder;
 public class ByteArray {
 
     private final byte[] buffer;
-    private final int hash;
 
 
     public ByteArray(final byte[] bb) {
         this.buffer = bb;
-        this.hash = 0;
     }
 
     public int length() {

source/net/yacy/search/Switchboard.java

@@ -1892,7 +1892,7 @@ public final class Switchboard extends serverSwitch
     }
 
     public int cleanupJobSize() {
-        int c = 0;
+        int c = 1; // "es gibt immer was zu tun"
         if ( (this.crawlQueues.delegatedURL.stackSize() > 1000) ) {
             c++;
         }
@@ -1909,6 +1909,9 @@ public final class Switchboard extends serverSwitch
 
     public boolean cleanupJob() {
         try {
+        	// flush the document compressor cache
+        	Cache.commit();
+        	
             // clear caches if necessary
             if ( !MemoryControl.request(8000000L, false) ) {
                 for ( final Segment indexSegment : this.indexSegments ) {

source/net/yacy/search/query/RWIProcess.java

@@ -272,7 +272,6 @@ public final class RWIProcess extends Thread
                     || pattern.equals("smb://.*")
                     || pattern.equals("file://.*");
             long remaining;
-            int count = 0;
             pollloop: while ( true ) {
                 remaining = timeout - System.currentTimeMillis();
                 if (remaining <= 0) {
@@ -288,8 +287,6 @@ public final class RWIProcess extends Thread
                     break pollloop;
                 }
                 assert (iEntry.urlhash().length == index.row().primaryKeyLength);
-                //if (iEntry.urlHash().length() != index.row().primaryKeyLength) continue;
-                count++;
 
                 // increase flag counts
                 for ( int j = 0; j < 32; j++ ) {

source/net/yacy/search/query/SnippetProcess.java

@@ -461,7 +461,6 @@ public class SnippetProcess {
             final boolean nav_topics = SnippetProcess.this.query.navigators.equals("all") || SnippetProcess.this.query.navigators.indexOf("topics",0) >= 0;
             try {
                 //System.out.println("DEPLOYED WORKER " + id + " FOR " + this.neededResults + " RESULTS, timeoutd = " + (this.timeout - System.currentTimeMillis()));
-                int loops = 0;
                 while (this.shallrun && System.currentTimeMillis() < this.timeout) {
                     //Log.logInfo("SnippetProcess", "***** timeleft = " + (this.timeout - System.currentTimeMillis()));
                     this.lastLifeSign = System.currentTimeMillis();
@@ -509,7 +508,6 @@ public class SnippetProcess {
                         }
                     }
 
-                    loops++;
                     resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
                     if (resultEntry == null)
                      {

source/net/yacy/search/ranking/BlockRank.java

@@ -56,7 +56,8 @@ import net.yacy.search.index.Segment;
 public class BlockRank {
 
     public static BinSearch[] ybrTables = null; // block-rank tables
-
+    private static File rankingPath;
+    private static int count;
 
     /**
      * collect host index information from other peers. All peers in the seed database are asked
@@ -214,6 +215,7 @@ public class BlockRank {
         byte[] hosth = new byte[6];
         String hosths, hostn;
         HostStat hs;
+        ensureLoaded();
         for (int ybr = 0; ybr < ybrTables.length; ybr++) {
             row: for (int i = 0; i < ybrTables[ybr].size(); i++) {
                 hosth = ybrTables[ybr].get(i, hosth);
@@ -240,11 +242,18 @@ public class BlockRank {
      * @param rankingPath
      * @param count
      */
-    public static void loadBlockRankTable(final File rankingPath, final int count) {
-        if (!rankingPath.exists()) return;
+    public static void loadBlockRankTable(final File rankingPath0, final int count0) {
+    	// lazy initialization to save memory during startup phase
+    	rankingPath = rankingPath0;
+    	count = count0;
+    }
+    
+    public static void ensureLoaded() {
+        if (ybrTables != null) return;
         ybrTables = new BinSearch[count];
         String ybrName;
         File f;
+        Log.logInfo("BlockRank", "loading block rank table from " + rankingPath.toString());
         try {
             for (int i = 0; i < count; i++) {
                 ybrName = "YBR-4-" + Digest.encodeHex(i, 2) + ".idx";
@@ -287,6 +296,7 @@ public class BlockRank {
      * @return
      */
     public static int ranking(final byte[] hash) {
+    	ensureLoaded();
         return ranking(hash, ybrTables);
     }