Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

build.properties

@@ -3,7 +3,7 @@ javacSource=1.6
 javacTarget=1.6
 
 # Release Configuration
-releaseVersion=1.03
+releaseVersion=1.04
 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 sourceReleaseFile=yacy_src_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 releaseFileParentDir=yacy

source/de/anomic/crawler/CrawlStacker.java

@@ -356,7 +356,8 @@ public final class CrawlStacker {
             entry.url().getContentDomain() == ContentDomain.APP  ||
             entry.url().getContentDomain() == ContentDomain.IMAGE  ||
             entry.url().getContentDomain() == ContentDomain.AUDIO  ||
-            entry.url().getContentDomain() == ContentDomain.VIDEO ) {
+            entry.url().getContentDomain() == ContentDomain.VIDEO ||
+            entry.url().getContentDomain() == ContentDomain.CTRL) {
             warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.NOLOAD, entry);
             //if (warning != null) this.log.logWarning("CrawlStacker.stackCrawl of URL " + entry.url().toNormalform(true, false) + " - not pushed: " + warning);
             return null;

source/net/yacy/cora/document/Classification.java

@@ -36,6 +36,7 @@ public class Classification {
     private static final Set<String> audioExtSet = new HashSet<String>();
     private static final Set<String> videoExtSet = new HashSet<String>();
     private static final Set<String> appsExtSet = new HashSet<String>();
+    private static final Set<String> ctrlExtSet = new HashSet<String>();
 
     public enum ContentDomain {
 
@@ -44,7 +45,8 @@ public class Classification {
         IMAGE(1),
         AUDIO(2),
         VIDEO(3),
-        APP(4);
+        APP(4),
+        CTRL(5);
 
         private final int code;
 
@@ -63,6 +65,7 @@ public class Classification {
             else if ("audio".equals(dom)) return AUDIO;
             else if ("video".equals(dom)) return VIDEO;
             else if ("app".equals(dom)) return APP;
+            else if ("ctrl".equals(dom)) return CTRL;
             return TEXT;
         }
 
@@ -74,6 +77,7 @@ public class Classification {
             else if (this == AUDIO) return "audio";
             else if (this == VIDEO) return "video";
             else if (this == APP) return "app";
+            else if (this == CTRL) return "ctrl";
             return "text";
         }
     }
@@ -85,12 +89,14 @@ public class Classification {
         final String audio = "aac,aif,aiff,flac,m4a,m4p,mid,mp2,mp3,oga,ogg,ram,sid,wav,wma";
         final String video = "3g2,3gp,3gp2,3gpp,3gpp2,3ivx,asf,asx,avi,div,divx,dv,dvx,env,f4v,flv,hdmov,m1v,m4v,m-jpeg,moov,mov,movie,mp2v,mp4,mpe,mpeg,mpg,mpg4,mv4,ogm,ogv,qt,rm,rv,vid,swf,webm,wmv";
         final String image = "ai,bmp,cdr,cmx,emf,eps,gif,img,jpeg,jpg,mng,pct,pdd,pdn,pict,png,psb,psd,psp,tif,tiff,wmf";
+        final String ctrl = "sha1,md5,crc32,sfv";
 
         addSet(textExtSet, text); // image formats
         addSet(imageExtSet, image); // image formats
         addSet(audioExtSet, audio); // audio formats
         addSet(videoExtSet, video); // video formats
         addSet(appsExtSet, apps);   // application formats
+        addSet(ctrlExtSet, ctrl);   // control formats
         addSet(mediaExtSet, apps + "," + audio + "," + video + "," + image); // all media formats
     }
 
@@ -129,12 +135,18 @@ public class Classification {
         return appsExtSet.contains(appsExt.trim().toLowerCase());
     }
 
+    public static boolean isControlExtension(final String ctrlExt) {
+        if (ctrlExt == null) return false;
+        return ctrlExtSet.contains(ctrlExt.trim().toLowerCase());
+    }
+
     public static ContentDomain getContentDomain(final String ext) {
         if (isTextExtension(ext)) return ContentDomain.TEXT;
         if (isImageExtension(ext)) return ContentDomain.IMAGE;
         if (isAudioExtension(ext)) return ContentDomain.AUDIO;
         if (isVideoExtension(ext)) return ContentDomain.VIDEO;
         if (isApplicationExtension(ext)) return ContentDomain.APP;
+        if (isControlExtension(ext)) return ContentDomain.CTRL;
         return ContentDomain.ALL;
     }
 

source/net/yacy/kelondro/logging/Log.java

@@ -30,6 +30,7 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.logging.FileHandler;
@@ -411,8 +412,7 @@ public final class Log {
                     exceptionLog.logSevere(msg + "\n" + baos.toString(), e);
                     Log.logException(e);
                     Log.logException(e.getCause());
-                    //System.err.print("Exception in thread \"" + t.getName() + "\" ");
-                    //e.printStackTrace(System.err);
+                    if (e instanceof InvocationTargetException) Log.logException(((InvocationTargetException) e).getTargetException());
                 }
             });
         } finally {

source/net/yacy/kelondro/order/Digest.java

@@ -56,10 +56,11 @@ public class Digest {
 
 	public static BlockingQueue<MessageDigest> digestPool = new LinkedBlockingDeque<MessageDigest>();
 
-    private static final int md5CacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L)));
+    private static final int md5CacheSize = Math.max(1000, Math.min(1000000, (int) (MemoryControl.available() / 50000L)));
     private static ARC<String, byte[]> md5Cache = null;
     static {
         try {
+            Log.logInfo("Digest", "creating hash cache of size " + md5CacheSize);
             md5Cache = new ConcurrentARC<String, byte[]>(md5CacheSize, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
         } catch (final OutOfMemoryError e) {
             md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(2, Runtime.getRuntime().availableProcessors()));

source/net/yacy/search/Switchboard.java

@@ -3425,7 +3425,7 @@ public final class Switchboard extends serverSwitch
                                 if ( (ys != null)
                                     && (!peers.mySeedIsDefined() || !peers.mySeed().hash.equals(ys.hash)) ) {
                                     final long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);
-                                    if ( lastseen < 240 ) {
+                                    if ( lastseen < 60 ) {
                                         if ( peers.peerActions.connectPeer(ys, false) ) {
                                             lc++;
                                         }