diff --git a/build.properties b/build.properties index 1a11630d7..f78ebaf97 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4 # Release Configuration -releaseVersion=0.524 +releaseVersion=0.525 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr} diff --git a/htroot/WebStructurePicture_p.java b/htroot/WebStructurePicture_p.java index 533277726..18195e2e8 100644 --- a/htroot/WebStructurePicture_p.java +++ b/htroot/WebStructurePicture_p.java @@ -81,28 +81,37 @@ public class WebStructurePicture_p { // find domain with most references host = sb.webStructure.hostWithMaxReferences(); } - // find start hash - String hash = null; - try { - hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6); - } catch (MalformedURLException e) {e.printStackTrace();} - assert (sb.webStructure.references(hash) != null); - - // recursively find domains, up to a specific depth - ymageGraph graph = new ymageGraph(); - if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth); - //graph.print(); - - ymageMatrix graphPicture = graph.draw(width, height, 40, 40, 16, 16); - + ymageMatrix graphPicture; + if (host == null) { + // probably no information available + graphPicture = new ymageMatrix(width, height, ymageGraph.color_back); + graphPicture.setMode(ymageMatrix.MODE_SUB); + ymageToolPrint.print(graphPicture, width / 2, height / 2, 0, "NO WEB STRUCTURE DATA AVAILABLE.", 0); + ymageToolPrint.print(graphPicture, width / 2, height / 2 + 16, 0, "START A WEB CRAWL TO OBTAIN STRUCTURE DATA.", 0); + } else { + // find start hash + String hash = null; + try { + hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6); + } catch (MalformedURLException e) {e.printStackTrace();} + assert (sb.webStructure.references(hash) != null); + + // recursively find domains, up to a specific depth + ymageGraph graph = new ymageGraph(); + if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth); + //graph.print(); + + graphPicture = graph.draw(width, height, 40, 40, 16, 16); + } // print headline - graphPicture.setColor(ymageMatrix.SUBTRACTIVE_BLACK); + graphPicture.setColor(ymageGraph.color_text); graphPicture.setMode(ymageMatrix.MODE_SUB); ymageToolPrint.print(graphPicture, 2, 8, 0, "YACY WEB-STRUCTURE ANALYSIS", -1); - ymageToolPrint.print(graphPicture, 2, 16, 0, "LINK ENVIRONMENT OF DOMAIN " + host.toUpperCase(), -1); + if (host != null) ymageToolPrint.print(graphPicture, 2, 16, 0, "LINK ENVIRONMENT OF DOMAIN " + host.toUpperCase(), -1); ymageToolPrint.print(graphPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1); return graphPicture; + } private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, long timeout, double x, double y, int nextlayer, int maxlayer) { diff --git a/htroot/index.html b/htroot/index.html index eae572fad..316e84189 100644 --- a/htroot/index.html +++ b/htroot/index.html @@ -33,7 +33,7 @@ #(searchoptions)# @@ -50,7 +50,6 @@ - @@ -75,11 +74,11 @@ diff --git a/htroot/index.java b/htroot/index.java index 5a54bbe90..ff61f4258 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -66,8 +66,8 @@ public class index { final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0); final int searchoptions = (post == null) ? 0 : post.getInt("searchoptions", 0); final String former = (post == null) ? "" : post.get("former", ""); - final int count = (post == null) ? 10 : post.getInt("count", 10); - final int time = (post == null) ? 10 : post.getInt("time", 6); + final int count = Math.min(100, (post == null) ? 10 : post.getInt("count", 10)); + final int time = Math.min(60, (post == null) ? 4 : post.getInt("time", 4)); final String urlmaskfilter = (post == null) ? ".*" : post.get("urlmaskfilter", ".*"); final String prefermaskfilter = (post == null) ? "" : post.get("prefermaskfilter", ""); final String constraint = (post == null) ? plasmaSearchQuery.catchall_constraint.exportB64() : post.get("constraint", "______"); @@ -127,17 +127,16 @@ public class index { prop.put("searchoptions_count-10", (count == 10) ? 1 : 0); prop.put("searchoptions_count-50", (count == 50) ? 1 : 0); prop.put("searchoptions_count-100", (count == 100) ? 1 : 0); - prop.put("searchoptions_count-1000", (count == 1000) ? 1 : 0); prop.put("searchoptions_resource-global", ((global) ? 1 : 0)); prop.put("searchoptions_resource-global-disabled", (global || clustersearch) ? 0 : 1); prop.put("searchoptions_resource-global-disabled_reason", (indexReceiveGranted) ? 0 : (indexDistributeGranted) ? 1 : 2); prop.put("searchoptions_resource-local", ((global) ? 0 : 1)); prop.put("searchoptions_time-1", (time == 1) ? 1 : 0); - prop.put("searchoptions_time-3", (time == 3) ? 1 : 0); + prop.put("searchoptions_time-2", (time == 2) ? 1 : 0); + prop.put("searchoptions_time-4", (time == 4) ? 1 : 0); prop.put("searchoptions_time-6", (time == 6) ? 1 : 0); + prop.put("searchoptions_time-8", (time == 8) ? 1 : 0); prop.put("searchoptions_time-10", (time == 10) ? 1 : 0); - prop.put("searchoptions_time-30", (time == 30) ? 1 : 0); - prop.put("searchoptions_time-60", (time == 60) ? 1 : 0); prop.put("searchoptions_urlmaskoptions", 0); prop.put("searchoptions_urlmaskoptions_urlmaskfilter", urlmaskfilter); prop.put("searchoptions_prefermaskoptions", 0); diff --git a/source/de/anomic/plasma/plasmaCrawlLoader.java b/source/de/anomic/plasma/plasmaCrawlLoader.java index eac42b16e..33bf5f78a 100644 --- a/source/de/anomic/plasma/plasmaCrawlLoader.java +++ b/source/de/anomic/plasma/plasmaCrawlLoader.java @@ -277,7 +277,9 @@ public final class plasmaCrawlLoader extends Thread { String referer, String initiator, int depth, - plasmaCrawlProfile.entry profile + plasmaCrawlProfile.entry profile, + int timeout, + boolean keepInMemory ) { if (!this.crawlwerPool.isClosed) { @@ -293,8 +295,8 @@ public final class plasmaCrawlLoader extends Thread { profile, // crawling profile crawlingPriority, // crawling priority false, // only download documents whose mimetypes are enabled for the crawler - -1, // use default crawler timeout - false // resource should not be kept in memory + timeout, // -1 = use default crawler timeout + keepInMemory // kept in memory ? ); // adding the message to the queue diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 2728cef68..04a19b9d7 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -191,8 +191,9 @@ public class plasmaCrawlNURL { entry = balancer.pop(minimumDelta, maximumDomAge); if (entry == null) { if (s > balancer.size()) continue; + int aftersize = balancer.size(); balancer.clear(); // the balancer is broken and cannot shrink - throw new IOException("entry is null, balancer cannot shrink; reset of balancer"); + throw new IOException("entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer"); } return entry; } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index ef76d7b55..ab4e222de 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -2639,7 +2639,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } catch (IOException e) { refererURL = null; } - cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile); + cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile, -1, false); log.logInfo(stats + ": enqueued for load " + urlEntry.url() + " [" + urlEntry.urlhash() + "]"); return; } diff --git a/source/de/anomic/ymage/ymageGraph.java b/source/de/anomic/ymage/ymageGraph.java index 35f1062f4..87214bf22 100644 --- a/source/de/anomic/ymage/ymageGraph.java +++ b/source/de/anomic/ymage/ymageGraph.java @@ -124,11 +124,11 @@ public class ymageGraph { } } - private static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE; + public static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE; + public static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK; private static final long color_dot = 0x6633AA; private static final long color_line = 0x333333; private static final long color_lineend = 0x555555; - private static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK; public ymageMatrix draw(int width, int height, int leftborder, int rightborder, int topborder, int bottomborder) { ymageMatrix image = new ymageMatrix(width, height, color_back);