From 30c3d909b12d17610b87171f23bbcf6bd41c1ee4 Mon Sep 17 00:00:00 2001 From: karlchenofhell Date: Tue, 5 Jun 2007 11:37:19 +0000 Subject: [PATCH] - fixed charset problem in ConfigProfil_p.html (use accept-charset="UTF-8" in forms) - fixed wrong XML output if no peers are known in Network.xml - simplified parsing of table properties in wikiCode and ZTableToken - reimplemented GC heuristics. They are needed to constantly ensure that an amount of free memory is available which is higher than Java's max. limit for performing a Full GC (please use serverMemory.request(long, boolean) rather than serverMemory.available(long, boolean) to provide data for averaging over the last GCs) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3793 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ConfigProfile_p.html | 2 +- htroot/Network.xml | 52 +++--- .../anomic/data/wiki/tokens/TableToken.java | 34 ++-- source/de/anomic/data/wikiCode.java | 149 ++++++------------ .../anomic/server/serverAbstractThread.java | 7 +- source/de/anomic/server/serverMemory.java | 79 +++++++++- 6 files changed, 175 insertions(+), 148 deletions(-) diff --git a/htroot/ConfigProfile_p.html b/htroot/ConfigProfile_p.html index 40cb39380..bba9ecaff 100644 --- a/htroot/ConfigProfile_p.html +++ b/htroot/ConfigProfile_p.html @@ -14,7 +14,7 @@

You do not need to provide any personal data here, but if you want to distribute your contact information, you can do that here.

-
+
diff --git a/htroot/Network.xml b/htroot/Network.xml index eb244c310..c5c961665 100644 --- a/htroot/Network.xml +++ b/htroot/Network.xml @@ -1,7 +1,6 @@ -#(table)# -:: +#(table)#:: #{list}# #[hash]# @@ -33,9 +32,7 @@ #(/complete)# #{/list}# - :: - #[active-count]# #[active-links]# @@ -57,30 +54,31 @@ #[all-words]# - #[my-name]# - #[my-hash]# - #(my-info)#virgin::junior::senior::principal#(/my-info)# - #[my-version]# - #[my-utc]# - #[my-uptime]# - #[my-links]# - #[my-words]# - #[my-acceptcrawl]# - #[my-dhtreceive]# - #[my-sI]# - #[my-sU]# - #[my-rI]# - #[my-rU]# - #[my-ppm]# - #[my-qph]# - #[my-seeds]# - #[my-connects]# - #[my-location]# - #[my-url]# + #[my-name]# + #[my-hash]# + #(my-info)#virgin::junior::senior::principal#(/my-info)# + #[my-version]# + #[my-utc]# + #[my-uptime]# + #[my-links]# + #[my-words]# + #[my-acceptcrawl]# + #[my-dhtreceive]# + #[my-rankingreceive]# + #[my-sI]# + #[my-sU]# + #[my-rI]# + #[my-rU]# + #[my-ppm]# + #[my-qph]# + #[my-seeds]# + #[my-connects]# + #[my-location]# + #[my-url]# - #[gppm]# - #[gqph]# + #[gppm]# + #[gqph]# #[comment]# - #(/table)# + diff --git a/source/de/anomic/data/wiki/tokens/TableToken.java b/source/de/anomic/data/wiki/tokens/TableToken.java index a9aee689d..2fd8e13b1 100644 --- a/source/de/anomic/data/wiki/tokens/TableToken.java +++ b/source/de/anomic/data/wiki/tokens/TableToken.java @@ -47,6 +47,7 @@ package de.anomic.data.wiki.tokens; +import java.util.Arrays; import java.util.HashMap; import java.util.regex.Pattern; @@ -92,10 +93,16 @@ public class TableToken extends AbstractToken { private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" }; private static final HashMap/* */ ps = new HashMap(); static { - ps.put("frame", new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" }); - ps.put("rules", new String[] { "none", "groups", "rows", "cols", "all" }); - ps.put("valign", new String[] { "top", "middle", "bottom", "baseline" }); - ps.put("align", new String[] { "left", "right", "center" }); + Arrays.sort(tps); + String[] array; + Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" }); + ps.put("frame", array); + Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" }); + ps.put("rules", array); + Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" }); + ps.put("valign", array); + Arrays.sort(array = new String[] { "left", "right", "center" }); + ps.put("align", array); } // contributed by [MN] @@ -106,7 +113,7 @@ public class TableToken extends AbstractToken { * @param properties A string that may contain several table properties and/or junk. * @return A string that only contains table properties. */ - private static StringBuffer parseTableProperties(final String properties){ + private static StringBuffer parseTableProperties(final String properties) { String[] values = properties.replaceAll(""", "").split("[= ]"); //splitting the string at = and blanks StringBuffer sb = new StringBuffer(properties.length()); String key, value; @@ -114,21 +121,20 @@ public class TableToken extends AbstractToken { int numberofvalues = values.length; for (int i=0; i= 0) || + (Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+")) ) { addPair(key, value, sb); - continue; } } - if (key.equals("nowrap")) - addPair("nowrap", "nowrap", sb); } return sb; } @@ -136,12 +142,6 @@ public class TableToken extends AbstractToken { private static StringBuffer addPair(String key, String value, StringBuffer sb) { return sb.append(" ").append(key).append("=\"").append(value).append("\""); } - - private static boolean isInArray(Object[] array, Object find) { - for (int i=array.length-1; i>-1; i--) - if (array[i].equals(find)) return true; - return false; - } public Pattern[] getRegex() { return pattern; } public String[] getBlockElementNames() { return blockElementNames; } diff --git a/source/de/anomic/data/wikiCode.java b/source/de/anomic/data/wikiCode.java index 878d54cc2..6031ff075 100644 --- a/source/de/anomic/data/wikiCode.java +++ b/source/de/anomic/data/wikiCode.java @@ -49,6 +49,8 @@ package de.anomic.data; import java.io.BufferedReader; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import de.anomic.data.wiki.abstractWikiParser; import de.anomic.data.wiki.wikiParser; @@ -60,6 +62,23 @@ import de.anomic.yacy.yacyCore; * parts of YaCy that use this class, like the blog or the profile. */ public class wikiCode extends abstractWikiParser implements wikiParser { + + /* Table properties */ + private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" }; + private static final HashMap/* */ ps = new HashMap(); + static { + Arrays.sort(tps); + String[] array; + Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" }); + ps.put("frame", array); + Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" }); + ps.put("rules", array); + Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" }); + ps.put("valign", array); + Arrays.sort(array = new String[] { "left", "right", "center" }); + ps.put("align", array); + } + private String numListLevel=""; private String ListLevel=""; private String defListLevel=""; @@ -119,7 +138,7 @@ public class wikiCode extends abstractWikiParser implements wikiParser { newrowstart=true; line="lenTableStart) { - line+=parseTableProperties(result.substring(lenTableStart).trim()); + line+=parseTableProperties(result.substring(lenTableStart).trim()).toString(); } line+=">"; result=line; @@ -157,7 +176,7 @@ public class wikiCode extends abstractWikiParser implements wikiParser { propEnd=lenCellDivider; } else { - line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim()); + line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim()).toString(); } // quick&dirty fix for http://www.yacy-forum.de/viewtopic.php?t=2825 [MN] if(propEnd > cellEnd){ @@ -178,112 +197,44 @@ public class wikiCode extends abstractWikiParser implements wikiParser { } return result; } - - //contributed by [MN] + + // contributed by [MN], changes by [FB] /** This method takes possible table properties and tests if they are valid. * Valid in this case means if they are a property for the table, tr or td * tag as stated in the HTML Pocket Reference by Jennifer Niederst (1st edition) * The method is important to avoid XSS attacks on the wiki via table properties. - * @param str A string that may contain several table properties and/or junk. + * @param properties A string that may contain several table properties and/or junk. * @return A string that only contains table properties. */ - private String parseTableProperties(String str){ - str = str.replaceAll(""", ""); //killing all quotationmarks - String[] values = str.split("[= ]"); //splitting the string at = and blanks - str=""; //recycling... ;-) - int numberofvalues = values.length; - for(int i=0;i= 0) || + (Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+")) + ) { + addPair(key, value, sb); } } - else if(values[i].equals("bgcolor")){ - if(i+1 memprereq) try { + //} else if ((memnow = serverMemory.available()) > memprereq) try { + } else if (serverMemory.request(memprereq, false)) try { // do job timestamp = System.currentTimeMillis(); memstamp0 = serverMemory.used(); @@ -307,7 +308,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea busyCycles++; } else { log.logWarning("Thread '" + this.getName() + "' runs short memory cycle. Free mem: " + - (memnow / 1024) + " KB, needed: " + (memprereq / 1024) + " KB"); + (serverMemory.available() / 1024) + " KB, needed: " + (memprereq / 1024) + " KB"); // omit job, not enough memory // process scheduled pause timestamp = System.currentTimeMillis(); diff --git a/source/de/anomic/server/serverMemory.java b/source/de/anomic/server/serverMemory.java index d5e711895..dd8f7c350 100644 --- a/source/de/anomic/server/serverMemory.java +++ b/source/de/anomic/server/serverMemory.java @@ -28,17 +28,60 @@ package de.anomic.server; import java.text.DecimalFormat; +import de.anomic.server.logging.serverLog; + public class serverMemory { public static boolean vm14 = System.getProperty("java.vm.version").startsWith("1.4"); public static final long max = (vm14) ? computedMaxMemory() : Runtime.getRuntime().maxMemory() ; // patch for maxMemory bug in Java 1.4.2 private static final Runtime runtime = Runtime.getRuntime(); + private static final long[] gcs = new long[5]; + private static int gcs_pos = 0; + + /** @return the amount of freed bytes by a forced GC this method performes */ + private static long runGC() { + long memnow = available(); + System.gc(); + if (++gcs_pos >= gcs.length) gcs_pos = 0; + return (gcs[gcs_pos] = available() - memnow); + } + + /** + * This method calculates the average amount of bytes freed by the last GCs forced by this class + * @return the average amount of freed bytes of the last forced GCs or 0 if no + * GC has been run yet + */ + public static long getAverageGCFree() { + long x = 0; + int y = 0; + for (int i=0; itrue without performing additional steps. Otherwise - if + * gciffail is set - a Full GC is run, if gciffail is set to + * false and not enough memory is available, this method returns false. + * + * @see serverMemory#request(long, boolean) for another implementation + * @param memory amount of bytes to be assured to be free + * @param gciffail if not enough memory is available, this parameter specifies whether to perform + * a Full GC to free enough RAM + * @return whether enough RAM is available + * @deprecated use {@link serverMemory#request(long, boolean)} instead to enable the collection of + * heuristics about explicit usage of Full GCs. + */ public static boolean available(long memory, boolean gciffail) { if (available() >= memory) return true; if (!gciffail) return false; @@ -51,6 +94,40 @@ public class serverMemory { return max - runtime.totalMemory() + runtime.freeMemory(); } + /** + *

Tries to free a specified amount of bytes.

+ *

+ * If the currently available memory is enough, the method returns true without + * performing additional steps. If not, the behaviour depends on the parameter force. + * If false, a Full GC is only performed if former GCs indicated that a GC should + * provide enough free memory. If former GCs didn't but force is set to true + * a Full GC is performed nevertheless. + *

+ *

+ * Setting the force parameter to false doesn't necessarily mean, that no GC may be + * performed by this method, if the currently available memory doesn't suffice! + *

+ *

Be careful with this method as GCs should always be the last measure to take

+ * + * @param size the requested amount of free memory in bytes + * @param specifies whether a GC should be run even in case former GCs didn't provide enough memory + * @return whether enough memory could be freed (or is free) or not + */ + public static boolean request(long size, boolean force) { + long avail = available(); + if (avail >= size) return true; + long avg; + if (force || ((avg = getAverageGCFree()) == 0 || avg + avail >= size)) { + long freed = runGC(); + avail = available(); + serverLog.logInfo("MEMORY", "performed explicit GC, freed " + (freed / 1024) + " KB (requested/available: " + size + " / " + avail + " KB)"); + return avail >= size; + } else { + serverLog.logInfo("MEMORY", "couldn't free enough memory (requested/available " + size + "/" + avail + " KB)"); + return false; + } + } + public static long used() { // memory that is currently bound in objects return runtime.totalMemory() - runtime.freeMemory();