boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- reduced default search time

Browse Source 
- catched case when web structure cannot be painted because of too less data
- better logging when balance fails


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3892 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 506a5a44de
commit 71fd972ac0
8 changed files with 45 additions and 35 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.524
releaseVersion=0.525
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}

41
htroot/WebStructurePicture_p.java
Unescape View File

@ -81,28 +81,37 @@ public class WebStructurePicture_p {
// find domain with most references
host = sb.webStructure.hostWithMaxReferences();
}
// find start hash
String hash = null;
try {
hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6);
} catch (MalformedURLException e) {e.printStackTrace();}
assert (sb.webStructure.references(hash) != null);
// recursively find domains, up to a specific depth
ymageGraph graph = new ymageGraph();
if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth);
//graph.print();
ymageMatrix graphPicture = graph.draw(width, height, 40, 40, 16, 16);
ymageMatrix graphPicture;
if (host == null) {
// probably no information available
graphPicture = new ymageMatrix(width, height, ymageGraph.color_back);
graphPicture.setMode(ymageMatrix.MODE_SUB);
ymageToolPrint.print(graphPicture, width / 2, height / 2, 0, "NO WEB STRUCTURE DATA AVAILABLE.", 0);
ymageToolPrint.print(graphPicture, width / 2, height / 2 + 16, 0, "START A WEB CRAWL TO OBTAIN STRUCTURE DATA.", 0);
} else {
// find start hash
String hash = null;
try {
hash = plasmaURL.urlHash(new URL("http://" + host)).substring(6);
} catch (MalformedURLException e) {e.printStackTrace();}
assert (sb.webStructure.references(hash) != null);
// recursively find domains, up to a specific depth
ymageGraph graph = new ymageGraph();
if (host != null) place(graph, sb.webStructure, hash, host, nodes, timeout, 0.0, 0.0, 0, depth);
//graph.print();
graphPicture = graph.draw(width, height, 40, 40, 16, 16);
}
// print headline
graphPicture.setColor(ymageMatrix.SUBTRACTIVE_BLACK);
graphPicture.setColor(ymageGraph.color_text);
graphPicture.setMode(ymageMatrix.MODE_SUB);
ymageToolPrint.print(graphPicture, 2, 8, 0, "YACY WEB-STRUCTURE ANALYSIS", -1);
ymageToolPrint.print(graphPicture, 2, 16, 0, "LINK ENVIRONMENT OF DOMAIN " + host.toUpperCase(), -1);
if (host != null) ymageToolPrint.print(graphPicture, 2, 16, 0, "LINK ENVIRONMENT OF DOMAIN " + host.toUpperCase(), -1);
ymageToolPrint.print(graphPicture, width - 2, 8, 0, "SNAPSHOT FROM " + new Date().toString().toUpperCase(), 1);
return graphPicture;
}
private static final int place(ymageGraph graph, plasmaWebStructure structure, String centerhash, String centerhost, int maxnodes, long timeout, double x, double y, int nextlayer, int maxlayer) {

9
htroot/index.html
Unescape View File

@ -33,7 +33,7 @@
#(searchoptions)#<!-- default values are hard-coded
<input type="hidden" name="count" value="10" />
<input type="hidden" name="resource" value="global" />
<input type="hidden" name="time" value="6" />
<input type="hidden" name="time" value="4" />
<input type="hidden" name="urlmaskfilter" value=".*" />
<input type="hidden" name="prefermaskfilter" value="" />
<input type="hidden" name="indexof" value="off" />-->
@ -50,7 +50,6 @@
<option #(count-10)#::selected="selected"#(/count-10)#>10</option>
<option #(count-50)#::selected="selected"#(/count-50)#>50</option>
<option #(count-100)#::selected="selected"#(/count-100)#>100</option>
<option #(count-1000)#::selected="selected"#(/count-1000)#>1000</option>
</select>
</td>
</tr>
@ -75,11 +74,11 @@
<td>
<select id="time" name="time">
<option #(time-1)#::selected="selected"#(/time-1)#>1</option>
<option #(time-3)#::selected="selected"#(/time-3)#>3</option>
<option #(time-2)#::selected="selected"#(/time-2)#>2</option>
<option #(time-4)#::selected="selected"#(/time-4)#>4</option>
<option #(time-6)#::selected="selected"#(/time-6)#>6</option>
<option #(time-8)#::selected="selected"#(/time-8)#>8</option>
<option #(time-10)#::selected="selected"#(/time-10)#>10</option>
<option #(time-30)#::selected="selected"#(/time-30)#>30</option>
<option #(time-60)#::selected="selected"#(/time-60)#>60</option>
</select>
</td>
</tr>

11
htroot/index.java
Unescape View File

@ -66,8 +66,8 @@ public class index {
final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
final int searchoptions = (post == null) ? 0 : post.getInt("searchoptions", 0);
final String former = (post == null) ? "" : post.get("former", "");
final int count = (post == null) ? 10 : post.getInt("count", 10);
final int time = (post == null) ? 10 : post.getInt("time", 6);
final int count = Math.min(100, (post == null) ? 10 : post.getInt("count", 10));
final int time = Math.min(60, (post == null) ? 4 : post.getInt("time", 4));
final String urlmaskfilter = (post == null) ? ".*" : post.get("urlmaskfilter", ".*");
final String prefermaskfilter = (post == null) ? "" : post.get("prefermaskfilter", "");
final String constraint = (post == null) ? plasmaSearchQuery.catchall_constraint.exportB64() : post.get("constraint", "______");
@ -127,17 +127,16 @@ public class index {
prop.put("searchoptions_count-10", (count == 10) ? 1 : 0);
prop.put("searchoptions_count-50", (count == 50) ? 1 : 0);
prop.put("searchoptions_count-100", (count == 100) ? 1 : 0);
prop.put("searchoptions_count-1000", (count == 1000) ? 1 : 0);
prop.put("searchoptions_resource-global", ((global) ? 1 : 0));
prop.put("searchoptions_resource-global-disabled", (global || clustersearch) ? 0 : 1);
prop.put("searchoptions_resource-global-disabled_reason", (indexReceiveGranted) ? 0 : (indexDistributeGranted) ? 1 : 2);
prop.put("searchoptions_resource-local", ((global) ? 0 : 1));
prop.put("searchoptions_time-1", (time == 1) ? 1 : 0);
prop.put("searchoptions_time-3", (time == 3) ? 1 : 0);
prop.put("searchoptions_time-2", (time == 2) ? 1 : 0);
prop.put("searchoptions_time-4", (time == 4) ? 1 : 0);
prop.put("searchoptions_time-6", (time == 6) ? 1 : 0);
prop.put("searchoptions_time-8", (time == 8) ? 1 : 0);
prop.put("searchoptions_time-10", (time == 10) ? 1 : 0);
prop.put("searchoptions_time-30", (time == 30) ? 1 : 0);
prop.put("searchoptions_time-60", (time == 60) ? 1 : 0);
prop.put("searchoptions_urlmaskoptions", 0);
prop.put("searchoptions_urlmaskoptions_urlmaskfilter", urlmaskfilter);
prop.put("searchoptions_prefermaskoptions", 0);

8
source/de/anomic/plasma/plasmaCrawlLoader.java
Unescape View File

@ -277,7 +277,9 @@ public final class plasmaCrawlLoader extends Thread {
String referer,
String initiator,
int depth,
plasmaCrawlProfile.entry profile
plasmaCrawlProfile.entry profile,
int timeout,
boolean keepInMemory
) {
if (!this.crawlwerPool.isClosed) {
@ -293,8 +295,8 @@ public final class plasmaCrawlLoader extends Thread {
profile, // crawling profile
crawlingPriority, // crawling priority
false, // only download documents whose mimetypes are enabled for the crawler
-1, // use default crawler timeout
false // resource should not be kept in memory
timeout, // -1 = use default crawler timeout
keepInMemory // kept in memory ?
);
// adding the message to the queue

3
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -191,8 +191,9 @@ public class plasmaCrawlNURL {
entry = balancer.pop(minimumDelta, maximumDomAge);
if (entry == null) {
if (s > balancer.size()) continue;
int aftersize = balancer.size();
balancer.clear(); // the balancer is broken and cannot shrink
throw new IOException("entry is null, balancer cannot shrink; reset of balancer");
throw new IOException("entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer");
}
return entry;
}

2
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -2639,7 +2639,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} catch (IOException e) {
refererURL = null;
}
cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile);
cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile, -1, false);
log.logInfo(stats + ": enqueued for load " + urlEntry.url() + " [" + urlEntry.urlhash() + "]");
return;
}

4
source/de/anomic/ymage/ymageGraph.java
Unescape View File

@ -124,11 +124,11 @@ public class ymageGraph {
}
}
private static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE;
public static final long color_back = ymageMatrix.SUBTRACTIVE_WHITE;
public static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK;
private static final long color_dot = 0x6633AA;
private static final long color_line = 0x333333;
private static final long color_lineend = 0x555555;
private static final long color_text = ymageMatrix.SUBTRACTIVE_BLACK;
public ymageMatrix draw(int width, int height, int leftborder, int rightborder, int topborder, int bottomborder) {
ymageMatrix image = new ymageMatrix(width, height, color_back);

Write Preview
Loading…
Cancel
Save