diff --git a/build.properties b/build.properties index b3ece00e2..82d145759 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.6 javacTarget=1.6 # Release Configuration -releaseVersion=1.03 +releaseVersion=1.04 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz sourceReleaseFile=yacy_src_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseFileParentDir=yacy diff --git a/source/de/anomic/crawler/CrawlStacker.java b/source/de/anomic/crawler/CrawlStacker.java index 4d1470c96..f21e0b087 100644 --- a/source/de/anomic/crawler/CrawlStacker.java +++ b/source/de/anomic/crawler/CrawlStacker.java @@ -356,7 +356,8 @@ public final class CrawlStacker { entry.url().getContentDomain() == ContentDomain.APP || entry.url().getContentDomain() == ContentDomain.IMAGE || entry.url().getContentDomain() == ContentDomain.AUDIO || - entry.url().getContentDomain() == ContentDomain.VIDEO ) { + entry.url().getContentDomain() == ContentDomain.VIDEO || + entry.url().getContentDomain() == ContentDomain.CTRL) { warning = this.nextQueue.noticeURL.push(NoticedURL.StackType.NOLOAD, entry); //if (warning != null) this.log.logWarning("CrawlStacker.stackCrawl of URL " + entry.url().toNormalform(true, false) + " - not pushed: " + warning); return null; diff --git a/source/net/yacy/cora/document/Classification.java b/source/net/yacy/cora/document/Classification.java index ba44aa79e..6b776e0dd 100644 --- a/source/net/yacy/cora/document/Classification.java +++ b/source/net/yacy/cora/document/Classification.java @@ -36,6 +36,7 @@ public class Classification { private static final Set audioExtSet = new HashSet(); private static final Set videoExtSet = new HashSet(); private static final Set appsExtSet = new HashSet(); + private static final Set ctrlExtSet = new HashSet(); public enum ContentDomain { @@ -44,7 +45,8 @@ public class Classification { IMAGE(1), AUDIO(2), VIDEO(3), - APP(4); + APP(4), + CTRL(5); private final int code; @@ -63,6 +65,7 @@ public class Classification { else if ("audio".equals(dom)) return AUDIO; else if ("video".equals(dom)) return VIDEO; else if ("app".equals(dom)) return APP; + else if ("ctrl".equals(dom)) return CTRL; return TEXT; } @@ -74,6 +77,7 @@ public class Classification { else if (this == AUDIO) return "audio"; else if (this == VIDEO) return "video"; else if (this == APP) return "app"; + else if (this == CTRL) return "ctrl"; return "text"; } } @@ -85,12 +89,14 @@ public class Classification { final String audio = "aac,aif,aiff,flac,m4a,m4p,mid,mp2,mp3,oga,ogg,ram,sid,wav,wma"; final String video = "3g2,3gp,3gp2,3gpp,3gpp2,3ivx,asf,asx,avi,div,divx,dv,dvx,env,f4v,flv,hdmov,m1v,m4v,m-jpeg,moov,mov,movie,mp2v,mp4,mpe,mpeg,mpg,mpg4,mv4,ogm,ogv,qt,rm,rv,vid,swf,webm,wmv"; final String image = "ai,bmp,cdr,cmx,emf,eps,gif,img,jpeg,jpg,mng,pct,pdd,pdn,pict,png,psb,psd,psp,tif,tiff,wmf"; + final String ctrl = "sha1,md5,crc32,sfv"; addSet(textExtSet, text); // image formats addSet(imageExtSet, image); // image formats addSet(audioExtSet, audio); // audio formats addSet(videoExtSet, video); // video formats addSet(appsExtSet, apps); // application formats + addSet(ctrlExtSet, ctrl); // control formats addSet(mediaExtSet, apps + "," + audio + "," + video + "," + image); // all media formats } @@ -129,12 +135,18 @@ public class Classification { return appsExtSet.contains(appsExt.trim().toLowerCase()); } + public static boolean isControlExtension(final String ctrlExt) { + if (ctrlExt == null) return false; + return ctrlExtSet.contains(ctrlExt.trim().toLowerCase()); + } + public static ContentDomain getContentDomain(final String ext) { if (isTextExtension(ext)) return ContentDomain.TEXT; if (isImageExtension(ext)) return ContentDomain.IMAGE; if (isAudioExtension(ext)) return ContentDomain.AUDIO; if (isVideoExtension(ext)) return ContentDomain.VIDEO; if (isApplicationExtension(ext)) return ContentDomain.APP; + if (isControlExtension(ext)) return ContentDomain.CTRL; return ContentDomain.ALL; } diff --git a/source/net/yacy/kelondro/logging/Log.java b/source/net/yacy/kelondro/logging/Log.java index e62dd68e1..e00ee77b5 100644 --- a/source/net/yacy/kelondro/logging/Log.java +++ b/source/net/yacy/kelondro/logging/Log.java @@ -30,6 +30,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.FileHandler; @@ -411,8 +412,7 @@ public final class Log { exceptionLog.logSevere(msg + "\n" + baos.toString(), e); Log.logException(e); Log.logException(e.getCause()); - //System.err.print("Exception in thread \"" + t.getName() + "\" "); - //e.printStackTrace(System.err); + if (e instanceof InvocationTargetException) Log.logException(((InvocationTargetException) e).getTargetException()); } }); } finally { diff --git a/source/net/yacy/kelondro/order/Digest.java b/source/net/yacy/kelondro/order/Digest.java index ebe31d9c4..f8bf4d575 100644 --- a/source/net/yacy/kelondro/order/Digest.java +++ b/source/net/yacy/kelondro/order/Digest.java @@ -56,10 +56,11 @@ public class Digest { public static BlockingQueue digestPool = new LinkedBlockingDeque(); - private static final int md5CacheSize = Math.max(100000, Math.min(1000000, (int) (MemoryControl.available() / 40000L))); + private static final int md5CacheSize = Math.max(1000, Math.min(1000000, (int) (MemoryControl.available() / 50000L))); private static ARC md5Cache = null; static { try { + Log.logInfo("Digest", "creating hash cache of size " + md5CacheSize); md5Cache = new ConcurrentARC(md5CacheSize, Math.max(8, 2 * Runtime.getRuntime().availableProcessors())); } catch (final OutOfMemoryError e) { md5Cache = new ConcurrentARC(1000, Math.max(2, Runtime.getRuntime().availableProcessors())); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 579db6097..f5df9f977 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -3425,7 +3425,7 @@ public final class Switchboard extends serverSwitch if ( (ys != null) && (!peers.mySeedIsDefined() || !peers.mySeed().hash.equals(ys.hash)) ) { final long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60); - if ( lastseen < 240 ) { + if ( lastseen < 60 ) { if ( peers.peerActions.connectPeer(ys, false) ) { lc++; }