diff --git a/htroot/IndexCreateParserErrors_p.java b/htroot/IndexCreateParserErrors_p.java index 6f1146f08..f583803f1 100644 --- a/htroot/IndexCreateParserErrors_p.java +++ b/htroot/IndexCreateParserErrors_p.java @@ -23,6 +23,8 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import java.util.ArrayList; + import net.yacy.kelondro.data.meta.DigestURI; import de.anomic.crawler.ZURL; import de.anomic.http.server.RequestHeader; @@ -71,22 +73,25 @@ public class IndexCreateParserErrors_p { String initiatorHash, executorHash; yacySeed initiatorSeed, executorSeed; int j=0; - for (ZURL.Entry entry: sb.crawlQueues.errorURL) { - if (entry == null) continue; - url = entry.url(); - if (url == null) continue; - - initiatorHash = entry.initiator(); - executorHash = entry.executor(); - initiatorSeed = sb.peers.getConnected(initiatorHash); - executorSeed = sb.peers.getConnected(executorHash); - prop.putHTML("rejected_list_"+j+"_initiator", ((initiatorSeed == null) ? "proxy" : initiatorSeed.getName())); - prop.putHTML("rejected_list_"+j+"_executor", ((executorSeed == null) ? "proxy" : executorSeed.getName())); - prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false, true)); - prop.putHTML("rejected_list_"+j+"_failreason", entry.anycause()); - prop.put("rejected_list_"+j+"_dark", dark ? "1" : "0"); - dark = !dark; - j++; + ArrayList l = sb.crawlQueues.errorURL.list(100); + ZURL.Entry entry; + for (int i = l.size() - 1; i >= 0; i--) { + entry = l.get(i); + if (entry == null) continue; + url = entry.url(); + if (url == null) continue; + + initiatorHash = entry.initiator(); + executorHash = entry.executor(); + initiatorSeed = sb.peers.getConnected(initiatorHash); + executorSeed = sb.peers.getConnected(executorHash); + prop.putHTML("rejected_list_"+j+"_initiator", ((initiatorSeed == null) ? "proxy" : initiatorSeed.getName())); + prop.putHTML("rejected_list_"+j+"_executor", ((executorSeed == null) ? "proxy" : executorSeed.getName())); + prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false, true)); + prop.putHTML("rejected_list_"+j+"_failreason", entry.anycause()); + prop.put("rejected_list_"+j+"_dark", dark ? "1" : "0"); + dark = !dark; + j++; } prop.put("rejected_list", j); } diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 5f772b69f..308047d65 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -92,7 +92,7 @@ public final class transferRWI { // response values int pause = 0; String result = "ok"; - final StringBuilder unknownURLs = new StringBuilder(); + final StringBuilder unknownURLs = new StringBuilder(6000); if ((youare == null) || (!youare.equals(sb.peers.mySeed().hash))) { sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.peers.mySeed().hash); diff --git a/source/de/anomic/crawler/RobotsEntry.java b/source/de/anomic/crawler/RobotsEntry.java index 25d42fbbc..c30bcd431 100644 --- a/source/de/anomic/crawler/RobotsEntry.java +++ b/source/de/anomic/crawler/RobotsEntry.java @@ -109,7 +109,7 @@ public class RobotsEntry { if ((allowPathList != null)&&(allowPathList.size()>0)) { this.allowPathList.addAll(allowPathList); - final StringBuilder pathListStr = new StringBuilder(); + final StringBuilder pathListStr = new StringBuilder(allowPathList.size() * 30); for (int i=0; i0)) { this.denyPathList.addAll(disallowPathList); - final StringBuilder pathListStr = new StringBuilder(); + final StringBuilder pathListStr = new StringBuilder(disallowPathList.size() * 30); for (int i=0; i { ); // the class object - private final ObjectIndex urlIndex; - private final ConcurrentLinkedQueue stack; + protected final ObjectIndex urlIndex; + protected final ConcurrentLinkedQueue stack; public ZURL( final File cachePath, @@ -133,6 +134,15 @@ public class ZURL implements Iterable { return new EntryIterator(); } + public ArrayList list(int max) { + ArrayList l = new ArrayList(); + for (ZURL.Entry entry: this) { + l.add(entry); + if (max-- <= 0) l.remove(0); + } + return l; + } + private class EntryIterator implements Iterator { private Iterator hi; public EntryIterator() { @@ -186,7 +196,7 @@ public class ZURL implements Iterable { private final String anycause; // string describing reason for load fail private boolean stored; - private Entry( + protected Entry( final Request bentry, final String executor, final Date workdate, @@ -203,7 +213,7 @@ public class ZURL implements Iterable { stored = false; } - private Entry(final Row.Entry entry) throws IOException { + protected Entry(final Row.Entry entry) throws IOException { assert (entry != null); this.executor = entry.getColString(1, "UTF-8"); this.workdate = new Date(entry.getColLong(2)); diff --git a/source/de/anomic/crawler/retrieval/HTTPLoader.java b/source/de/anomic/crawler/retrieval/HTTPLoader.java index 0a1102fc9..d42ccf55c 100644 --- a/source/de/anomic/crawler/retrieval/HTTPLoader.java +++ b/source/de/anomic/crawler/retrieval/HTTPLoader.java @@ -144,7 +144,7 @@ public final class HTTPLoader { String supportError = TextParser.supports(request.url(), res.getResponseHeader().mime()); if (supportError != null) { sb.crawlQueues.errorURL.push(request, sb.peers.mySeed().hash, new Date(), 1, supportError); - throw new IOException("REJECTED WRONG MIME TYPE: " + supportError); + throw new IOException("REJECTED WRONG MIME TYPE, mime = " + res.getResponseHeader().mime() + ": " + supportError); } } diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 6cc8d6972..cec0b2450 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -595,10 +595,7 @@ public class bookmarksDB { } public Iterator getTagIterator(final boolean priv, final int c) { - Comparator comp; - if (c == SORT_SIZE) comp = new tagSizeComparator(); - else comp = new tagComparator(); - final TreeSet set=new TreeSet(comp); + final TreeSet set=new TreeSet((c == SORT_SIZE) ? tagSizeComparator : tagComparator); final Iterator it = tagIterator(true); Tag tag; while(it.hasNext()){ @@ -614,10 +611,7 @@ public class bookmarksDB { if (max==SHOW_ALL) return getTagIterator(priv, comp); final Iterator it = getTagIterator(priv, SORT_SIZE); - Comparator c; - if (comp == SORT_SIZE) c = new tagSizeComparator(); - else c = new tagComparator(); - final TreeSet set=new TreeSet(c); + final TreeSet set=new TreeSet((comp == SORT_SIZE) ? tagSizeComparator : tagComparator); int count = 0; while (it.hasNext() && count<=max) { set.add(it.next()); @@ -631,10 +625,7 @@ public class bookmarksDB { } */ public Iterator getTagIterator(final String tagName, final boolean priv, final int comp){ - Comparator c; - if (comp == SORT_SIZE) c = new tagSizeComparator(); - else c = new tagComparator(); - final TreeSet set=new TreeSet(c); + final TreeSet set=new TreeSet((comp == SORT_SIZE) ? tagSizeComparator : tagComparator); Iterator it=null; final Iterator bit=getBookmarksIterator(tagName, priv); Bookmark bm; @@ -657,11 +648,8 @@ public class bookmarksDB { public Iterator getTagIterator(final String tagName, final boolean priv, final int comp, final int max){ if (max==SHOW_ALL) return getTagIterator(priv, comp); - final Iterator it = getTagIterator(tagName, priv, SORT_SIZE); - Comparator c; - if (comp == SORT_SIZE) c = new tagSizeComparator(); - else c = new tagComparator(); - final TreeSet set=new TreeSet(c); + final Iterator it = getTagIterator(tagName, priv, SORT_SIZE); + final TreeSet set=new TreeSet((comp == SORT_SIZE) ? tagSizeComparator : tagComparator); int count = 0; while (it.hasNext() && count<=max) { set.add(it.next()); @@ -1410,10 +1398,13 @@ public class bookmarksDB { } } + public static final TagComparator tagComparator = new TagComparator(); + public static final TagSizeComparator tagSizeComparator = new TagSizeComparator(); + /** * Comparator to sort objects of type Tag according to their names */ - public static class tagComparator implements Comparator, Serializable { + public static class TagComparator implements Comparator, Serializable { /** * generated serial @@ -1426,7 +1417,7 @@ public class bookmarksDB { } - public static class tagSizeComparator implements Comparator, Serializable { + public static class TagSizeComparator implements Comparator, Serializable { /** * generated serial diff --git a/source/de/anomic/data/wiki/tokens/LinkToken.java b/source/de/anomic/data/wiki/tokens/LinkToken.java index 2c65cab5a..01369f3ec 100644 --- a/source/de/anomic/data/wiki/tokens/LinkToken.java +++ b/source/de/anomic/data/wiki/tokens/LinkToken.java @@ -87,7 +87,7 @@ public class LinkToken extends AbstractToken { } protected void parse() throws wikiParserException { - final StringBuilder stringBuilder = new StringBuilder(); + final StringBuilder stringBuilder = new StringBuilder(6000); if (this.patternNr < 0 || this.patternNr >= patterns.length) { throw new wikiParserException("patternNr was not set correctly: " + this.patternNr); @@ -179,7 +179,7 @@ public class LinkToken extends AbstractToken { @Override public String toString() { - final StringBuilder stringBuilder = new StringBuilder(); + final StringBuilder stringBuilder = new StringBuilder(300); stringBuilder.append(""); diff --git a/source/de/anomic/http/server/HTTPDProxyHandler.java b/source/de/anomic/http/server/HTTPDProxyHandler.java index 2811ceea5..09b81b0f3 100644 --- a/source/de/anomic/http/server/HTTPDProxyHandler.java +++ b/source/de/anomic/http/server/HTTPDProxyHandler.java @@ -1194,7 +1194,7 @@ public final class HTTPDProxyHandler { if (myAddress != null) { // getting header set by other proxies in the chain - final StringBuilder viaValue = new StringBuilder(); + final StringBuilder viaValue = new StringBuilder(80); if (header.containsKey(HeaderFramework.VIA)) viaValue.append(header.get(HeaderFramework.VIA)); if (viaValue.length() > 0) viaValue.append(", "); diff --git a/source/de/anomic/http/server/HeaderFramework.java b/source/de/anomic/http/server/HeaderFramework.java index 89795cf2d..477f5ff51 100644 --- a/source/de/anomic/http/server/HeaderFramework.java +++ b/source/de/anomic/http/server/HeaderFramework.java @@ -406,7 +406,7 @@ public class HeaderFramework extends TreeMap implements Map SUPPORTED_MIME_TYPES = new HashSet(); public static final Set SUPPORTED_EXTENSIONS = new HashSet(); static { - SUPPORTED_EXTENSIONS.add("xml"); SUPPORTED_EXTENSIONS.add("rss"); - SUPPORTED_EXTENSIONS.add("rdf"); + SUPPORTED_EXTENSIONS.add("xml"); + SUPPORTED_MIME_TYPES.add("XML"); SUPPORTED_MIME_TYPES.add("text/rss"); - SUPPORTED_MIME_TYPES.add("application/rdf+xml"); SUPPORTED_MIME_TYPES.add("application/rss+xml"); SUPPORTED_MIME_TYPES.add("application/atom+xml"); }