boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed charset problem in ConfigProfil_p.html (use accept-charset="UTF-8" in forms)

Browse Source 
- fixed wrong XML output if no peers are known in Network.xml
- simplified parsing of table properties in wikiCode and ZTableToken
- reimplemented GC heuristics. They are needed to constantly ensure that an amount of free memory is available which is higher than Java's max. limit for performing a Full GC (please use serverMemory.request(long, boolean) rather than serverMemory.available(long, boolean) to provide data for averaging over the last GCs)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3793 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
karlchenofhell 18 years ago
parent 6c9df13552
commit 30c3d909b1
6 changed files with 175 additions and 148 deletions
Show Stats Download Patch File Download Diff File

2
htroot/ConfigProfile_p.html
Unescape View File

@ -14,7 +14,7 @@
<p>
You do not need to provide any personal data here, but if you want to distribute your contact information, you can do that here.
</p>
<form action="ConfigProfile_p.html" accept-charset="ascii">
<form action="ConfigProfile_p.html" accept-charset="UTF-8">
<fieldset>
<dl>
<dt><label for="name">Name</label></dt>

52
htroot/Network.xml
Unescape View File

@ -1,7 +1,6 @@
<?xml version="1.0"?>
#(table)#
::
<peers>
#(table)#::
#{list}#
<peer>
<hash>#[hash]#</hash>
@ -33,9 +32,7 @@
#(/complete)#
</peer>
#{/list}#
</peers>
::
<peers>
<active>
<count>#[active-count]#</count>
<links>#[active-links]#</links>
@ -57,30 +54,31 @@
<words>#[all-words]#</words>
</all>
<your>
<name>#[my-name]#</name>
<hash>#[my-hash]#</hash>
<type>#(my-info)#virgin::junior::senior::principal#(/my-info)#</type>
<version>#[my-version]#</version>
<utc>#[my-utc]#</utc>
<uptime>#[my-uptime]#</uptime>
<links>#[my-links]#</links>
<words>#[my-words]#</words>
<acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>
<acceptindex>#[my-dhtreceive]#</acceptindex>
<sentwords>#[my-sI]#</sentwords>
<senturls>#[my-sU]#</senturls>
<receivedwords>#[my-rI]#</receivedwords>
<receivedurls>#[my-rU]#</receivedurls>
<ppm>#[my-ppm]#</ppm>
<qph>#[my-qph]#</qph>
<seeds>#[my-seeds]#</seeds>
<connects>#[my-connects]#</connects>
<location>#[my-location]#</location>
<seedurl>#[my-url]#</seedurl>
<name>#[my-name]#</name>
<hash>#[my-hash]#</hash>
<type>#(my-info)#virgin::junior::senior::principal#(/my-info)#</type>
<version>#[my-version]#</version>
<utc>#[my-utc]#</utc>
<uptime>#[my-uptime]#</uptime>
<links>#[my-links]#</links>
<words>#[my-words]#</words>
<acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>
<acceptindex>#[my-dhtreceive]#</acceptindex>
<acceptranking>#[my-rankingreceive]#</acceptranking>
<sentwords>#[my-sI]#</sentwords>
<senturls>#[my-sU]#</senturls>
<receivedwords>#[my-rI]#</receivedwords>
<receivedurls>#[my-rU]#</receivedurls>
<ppm>#[my-ppm]#</ppm>
<qph>#[my-qph]#</qph>
<seeds>#[my-seeds]#</seeds>
<connects>#[my-connects]#</connects>
<location>#[my-location]#</location>
<seedurl>#[my-url]#</seedurl>
</your>
<cluster>
<ppm>#[gppm]#</ppm>
<qph>#[gqph]#</qph>
<ppm>#[gppm]#</ppm>
<qph>#[gqph]#</qph>
</cluster>
<status>#[comment]#</status>
<!--
@ -88,5 +86,5 @@
1 - online mode, but no Internet available
2 - not in online mode
-->
</peers>
#(/table)#
</peers>

34
source/de/anomic/data/wiki/tokens/TableToken.java
Unescape View File

@ -47,6 +47,7 @@
package de.anomic.data.wiki.tokens;
import java.util.Arrays;
import java.util.HashMap;
import java.util.regex.Pattern;
@ -92,10 +93,16 @@ public class TableToken extends AbstractToken {
private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };
private static final HashMap/* <String,String[]> */ ps = new HashMap();
static {
ps.put("frame", new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
ps.put("rules", new String[] { "none", "groups", "rows", "cols", "all" });
ps.put("valign", new String[] { "top", "middle", "bottom", "baseline" });
ps.put("align", new String[] { "left", "right", "center" });
Arrays.sort(tps);
String[] array;
Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
ps.put("frame", array);
Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" });
ps.put("rules", array);
Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" });
ps.put("valign", array);
Arrays.sort(array = new String[] { "left", "right", "center" });
ps.put("align", array);
}
// contributed by [MN]
@ -106,7 +113,7 @@ public class TableToken extends AbstractToken {
* @param properties A string that may contain several table properties and/or junk.
* @return A string that only contains table properties.
*/
private static StringBuffer parseTableProperties(final String properties){
private static StringBuffer parseTableProperties(final String properties) {
String[] values = properties.replaceAll("&quot;", "").split("[= ]"); //splitting the string at = and blanks
StringBuffer sb = new StringBuffer(properties.length());
String key, value;
@ -114,21 +121,20 @@ public class TableToken extends AbstractToken {
int numberofvalues = values.length;
for (int i=0; i<numberofvalues; i++) {
key = values[i].trim();
if (i + 1 < numberofvalues) {
if (key.equals("nowrap")) {
addPair("nowrap", "nowrap", sb);
} else if (i + 1 < numberofvalues) {
value = values[++i].trim();
if (
(key.equals("summary")) ||
(key.equals("bgcolor") && value.matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")) ||
((key.equals("width") || key.equals("height")) && value.matches("\\d+%{0,1}")) ||
((posVals = (String[])ps.get(key)) != null && isInArray(posVals, value)) ||
(isInArray(tps, key) && value.matches("\\d+"))
((posVals = (String[])ps.get(key)) != null && Arrays.binarySearch(posVals, value) >= 0) ||
(Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+"))
) {
addPair(key, value, sb);
continue;
}
}
if (key.equals("nowrap"))
addPair("nowrap", "nowrap", sb);
}
return sb;
}
@ -136,12 +142,6 @@ public class TableToken extends AbstractToken {
private static StringBuffer addPair(String key, String value, StringBuffer sb) {
return sb.append(" ").append(key).append("=\"").append(value).append("\"");
}
private static boolean isInArray(Object[] array, Object find) {
for (int i=array.length-1; i>-1; i--)
if (array[i].equals(find)) return true;
return false;
}
public Pattern[] getRegex() { return pattern; }
public String[] getBlockElementNames() { return blockElementNames; }

149
source/de/anomic/data/wikiCode.java
Unescape View File

@ -49,6 +49,8 @@ package de.anomic.data;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import de.anomic.data.wiki.abstractWikiParser;
import de.anomic.data.wiki.wikiParser;
@ -60,6 +62,23 @@ import de.anomic.yacy.yacyCore;
* parts of YaCy that use this class, like the blog or the profile.
*/
public class wikiCode extends abstractWikiParser implements wikiParser {
/* Table properties */
private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };
private static final HashMap/* <String,String[]> */ ps = new HashMap();
static {
Arrays.sort(tps);
String[] array;
Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
ps.put("frame", array);
Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" });
ps.put("rules", array);
Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" });
ps.put("valign", array);
Arrays.sort(array = new String[] { "left", "right", "center" });
ps.put("align", array);
}
private String numListLevel="";
private String ListLevel="";
private String defListLevel="";
@ -119,7 +138,7 @@ public class wikiCode extends abstractWikiParser implements wikiParser {
newrowstart=true;
line="<table";
if (result.trim().length()>lenTableStart) {
line+=parseTableProperties(result.substring(lenTableStart).trim());
line+=parseTableProperties(result.substring(lenTableStart).trim()).toString();
}
line+=">";
result=line;
@ -157,7 +176,7 @@ public class wikiCode extends abstractWikiParser implements wikiParser {
propEnd=lenCellDivider;
}
else {
line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim());
line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim()).toString();
}
// quick&dirty fix for http://www.yacy-forum.de/viewtopic.php?t=2825 [MN]
if(propEnd > cellEnd){
@ -178,112 +197,44 @@ public class wikiCode extends abstractWikiParser implements wikiParser {
}
return result;
}
//contributed by [MN]
// contributed by [MN], changes by [FB]
/** This method takes possible table properties and tests if they are valid.
* Valid in this case means if they are a property for the table, tr or td
* tag as stated in the HTML Pocket Reference by Jennifer Niederst (1st edition)
* The method is important to avoid XSS attacks on the wiki via table properties.
* @param str A string that may contain several table properties and/or junk.
* @param properties A string that may contain several table properties and/or junk.
* @return A string that only contains table properties.
*/
private String parseTableProperties(String str){
str = str.replaceAll("&quot;", ""); //killing all quotationmarks
String[] values = str.split("[= ]"); //splitting the string at = and blanks
str=""; //recycling... ;-)
int numberofvalues = values.length;
for(int i=0;i<numberofvalues;i++){
if((values[i].equals("rowspan")) ||
(values[i].equals("colspan")) ||
(values[i].equals("vspace")) ||
(values[i].equals("hspace")) ||
(values[i].equals("cellspacing")) ||
(values[i].equals("cellpadding")) ||
(values[i].equals("border"))){
if(i+1<numberofvalues){
if(values[i+1].matches("\\d+")){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if((values[i].equals("width"))||(values[i].equals("height"))){
if(i+1<numberofvalues){
if(values[i+1].matches("\\d+%{0,1}")){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("align")){
if(i+1<numberofvalues) {
if((values[i+1].equals("left")) ||
(values[i+1].equals("right")) ||
(values[i+1].equals("center"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("valign")){
if(i+1<numberofvalues) {
if((values[i+1].equals("top")) ||
(values[i+1].equals("middle")) ||
(values[i+1].equals("bottom")) ||
(values[i+1].equals("baseline"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
private static StringBuffer parseTableProperties(final String properties) {
final String[] values = properties.replaceAll("&quot;", "").split("[= ]"); //splitting the string at = and blanks
final StringBuffer sb = new StringBuffer(properties.length());
String key, value;
String[] posVals;
final int numberofvalues = values.length;
for (int i=0; i<numberofvalues; i++) {
key = values[i].trim();
if (key.equals("nowrap")) {
addPair("nowrap", "nowrap", sb);
} else if (i + 1 < numberofvalues) {
value = values[++i].trim();
if (
(key.equals("summary")) ||
(key.equals("bgcolor") && value.matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")) ||
((key.equals("width") || key.equals("height")) && value.matches("\\d+%{0,1}")) ||
((posVals = (String[])ps.get(key)) != null && Arrays.binarySearch(posVals, value) >= 0) ||
(Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+"))
) {
addPair(key, value, sb);
}
}
else if(values[i].equals("bgcolor")){
if(i+1<numberofvalues){
if(values[i+1].matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("rules")){
if(i+1<numberofvalues) {
if((values[i+1].equals("none")) ||
(values[i+1].equals("groups")) ||
(values[i+1].equals("rows")) ||
(values[i+1].equals("cols")) ||
(values[i+1].equals("all"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("frame")){
if(i+1<numberofvalues) {
if((values[i+1].equals("void")) ||
(values[i+1].equals("above")) ||
(values[i+1].equals("below")) ||
(values[i+1].equals("hsides")) ||
(values[i+1].equals("lhs")) ||
(values[i+1].equals("rhs")) ||
(values[i+1].equals("vsides")) ||
(values[i+1].equals("box")) ||
(values[i+1].equals("border"))) {
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
}
else if(values[i].equals("summary")){
if(i+1<numberofvalues){
str = str + " "+values[i]+"=\""+values[i+1]+"\"";
i++;
}
}
else if(values[i].equals("nowrap")){
str = str + " nowrap";
}
}
return str;
} //end contrib [MN]
return sb;
}
private static StringBuffer addPair(String key, String value, StringBuffer sb) {
return sb.append(" ").append(key).append("=\"").append(value).append("\"");
}
/** This method processes ordered lists.
*/

7
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -250,7 +250,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
logSystem("thread '" + this.getName() + "' deployed, " + ((this.busyPause < 0) ? "starting job." : "starting loop."));
}
long timestamp;
long memstamp0, memstamp1, memnow;
long memstamp0, memstamp1;
boolean isBusy;
//Runtime rt = Runtime.getRuntime();
@ -273,7 +273,8 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
timestamp = System.currentTimeMillis();
ratz(this.idlePause);
idletime += System.currentTimeMillis() - timestamp;
} else if ((memnow = serverMemory.available()) > memprereq) try {
//} else if ((memnow = serverMemory.available()) > memprereq) try {
} else if (serverMemory.request(memprereq, false)) try {
// do job
timestamp = System.currentTimeMillis();
memstamp0 = serverMemory.used();
@ -307,7 +308,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
busyCycles++;
} else {
log.logWarning("Thread '" + this.getName() + "' runs short memory cycle. Free mem: " +
(memnow / 1024) + " KB, needed: " + (memprereq / 1024) + " KB");
(serverMemory.available() / 1024) + " KB, needed: " + (memprereq / 1024) + " KB");
// omit job, not enough memory
// process scheduled pause
timestamp = System.currentTimeMillis();

79
source/de/anomic/server/serverMemory.java
Unescape View File

@ -28,17 +28,60 @@ package de.anomic.server;
import java.text.DecimalFormat;
import de.anomic.server.logging.serverLog;
public class serverMemory {
public static boolean vm14 = System.getProperty("java.vm.version").startsWith("1.4");
public static final long max = (vm14) ? computedMaxMemory() : Runtime.getRuntime().maxMemory() ; // patch for maxMemory bug in Java 1.4.2
private static final Runtime runtime = Runtime.getRuntime();
private static final long[] gcs = new long[5];
private static int gcs_pos = 0;
/** @return the amount of freed bytes by a forced GC this method performes */
private static long runGC() {
long memnow = available();
System.gc();
if (++gcs_pos >= gcs.length) gcs_pos = 0;
return (gcs[gcs_pos] = available() - memnow);
}
/**
* This method calculates the average amount of bytes freed by the last GCs forced by this class
* @return the average amount of freed bytes of the last forced GCs or <code>0</code> if no
* GC has been run yet
*/
public static long getAverageGCFree() {
long x = 0;
int y = 0;
for (int i=0; i<gcs.length; i++)
if (gcs[i] != 0) {
x += gcs[i];
y++;
}
return (y == 0) ? 0 : x / y;
}
public static long free() {
// memory that is free without increasing of total memory taken from os
return runtime.freeMemory();
}
/**
* Tries to free a specified amount of bytes. If the currently available memory is enough, the
* method returns <code>true</code> without performing additional steps. Otherwise - if
* <code>gciffail</code> is set - a Full GC is run, if <code>gciffail</code> is set to
* <code>false</code> and not enough memory is available, this method returns <code>false</code>.
*
* @see serverMemory#request(long, boolean) for another implementation
* @param memory amount of bytes to be assured to be free
* @param gciffail if not enough memory is available, this parameter specifies whether to perform
* a Full GC to free enough RAM
* @return whether enough RAM is available
* @deprecated use {@link serverMemory#request(long, boolean)} instead to enable the collection of
* heuristics about explicit usage of Full GCs.
*/
public static boolean available(long memory, boolean gciffail) {
if (available() >= memory) return true;
if (!gciffail) return false;
@ -51,6 +94,40 @@ public class serverMemory {
return max - runtime.totalMemory() + runtime.freeMemory();
}
/**
* <p>Tries to free a specified amount of bytes.</p>
* <p>
* If the currently available memory is enough, the method returns <code>true</code> without
* performing additional steps. If not, the behaviour depends on the parameter <code>force</code>.
* If <code>false</code>, a Full GC is only performed if former GCs indicated that a GC should
* provide enough free memory. If former GCs didn't but <code>force</code> is set to <code>true</code>
* a Full GC is performed nevertheless.
* </p>
* <p>
* Setting the <code>force</code> parameter to false doesn't necessarily mean, that no GC may be
* performed by this method, if the currently available memory doesn't suffice!
* </p>
* <p><em>Be careful with this method as GCs should always be the last measure to take</em></p>
*
* @param size the requested amount of free memory in bytes
* @param specifies whether a GC should be run even in case former GCs didn't provide enough memory
* @return whether enough memory could be freed (or is free) or not
*/
public static boolean request(long size, boolean force) {
long avail = available();
if (avail >= size) return true;
long avg;
if (force || ((avg = getAverageGCFree()) == 0 || avg + avail >= size)) {
long freed = runGC();
avail = available();
serverLog.logInfo("MEMORY", "performed explicit GC, freed " + (freed / 1024) + " KB (requested/available: " + size + " / " + avail + " KB)");
return avail >= size;
} else {
serverLog.logInfo("MEMORY", "couldn't free enough memory (requested/available " + size + "/" + avail + " KB)");
return false;
}
}
public static long used() {
// memory that is currently bound in objects
return runtime.totalMemory() - runtime.freeMemory();

Write Preview
Loading…
Cancel
Save