diff --git a/source/de/anomic/http/client/Client.java b/source/de/anomic/http/client/Client.java index 6ba72f6cb..f245a4882 100644 --- a/source/de/anomic/http/client/Client.java +++ b/source/de/anomic/http/client/Client.java @@ -208,8 +208,10 @@ public class Client { * @see de.anomic.http.HttpClient#setProxy(de.anomic.http.httpRemoteProxyConfig) */ public void setProxy(final RemoteProxyConfig proxyConfig) { - this.useGlobalProxyConfig = false; - this.proxyConfig = proxyConfig; + if (proxyConfig != null) { + this.useGlobalProxyConfig = false; + this.proxyConfig = proxyConfig; + } } /* @@ -566,16 +568,16 @@ public class Client { */ private RemoteProxyConfig getProxyConfig(final String hostname) { final RemoteProxyConfig hostProxyConfig; - if (!useGlobalProxyConfig) { + if (useGlobalProxyConfig) { + // default settings + hostProxyConfig = RemoteProxyConfig.getProxyConfigForHost(hostname); + } else { // client specific - if(proxyConfig == null) { + if (proxyConfig == null) { hostProxyConfig = null; } else { hostProxyConfig = proxyConfig.useForHost(hostname) ? proxyConfig : null; } - } else { - // default settings - hostProxyConfig = RemoteProxyConfig.getProxyConfigForHost(hostname); } return hostProxyConfig; } diff --git a/source/de/anomic/http/client/RemoteProxyConfig.java b/source/de/anomic/http/client/RemoteProxyConfig.java index 7a8a6ec29..888f4f72b 100644 --- a/source/de/anomic/http/client/RemoteProxyConfig.java +++ b/source/de/anomic/http/client/RemoteProxyConfig.java @@ -63,8 +63,8 @@ public final class RemoteProxyConfig { /** * @param remoteProxyConfig the remoteProxyConfig to set */ - public static synchronized void setRemoteProxyConfig(final RemoteProxyConfig remoteProxyConfig) { - RemoteProxyConfig.remoteProxyConfig = remoteProxyConfig; + public static synchronized void setRemoteProxyConfig(final RemoteProxyConfig newConfig) { + remoteProxyConfig = newConfig; } /** diff --git a/source/de/anomic/kelondro/order/Base64Order.java b/source/de/anomic/kelondro/order/Base64Order.java index f1fc66b6a..78f5cb7bf 100644 --- a/source/de/anomic/kelondro/order/Base64Order.java +++ b/source/de/anomic/kelondro/order/Base64Order.java @@ -403,17 +403,50 @@ public class Base64Order extends AbstractOrder implements ByteOrder, Com } public final int compare(final byte[] a, final byte[] b) { + try { return (asc) ? ((zero == null) ? compares(a, b) : compare0(a, 0, a.length, b, 0, b.length)) : ((zero == null) ? compares(b, a) : compare0(b, 0, b.length, a, 0, a.length)); + } catch (Exception e) { + // if a or b is not well-formed, an ArrayIndexOutOfBoundsException may occur + // in that case we don't want that the exception makes databse functions + // unusable and effective creates a showstopper. In such cases we apply + // a different order on the objects and treat not well-formed objects + // as bigger as all others. If both object are not well-formed, they are + // compared with the natural order. + boolean wfa = wellformed(a); + boolean wfb = wellformed(b); + if (wfa && wfb) { + // uh strange. throw the exception + if (e instanceof ArrayIndexOutOfBoundsException) throw (ArrayIndexOutOfBoundsException) e; + throw new RuntimeException(e.getMessage()); + } + if (wfa) return (asc) ? -1 : 1; + if (wfb) return (asc) ? 1 : -1; + return ((asc) ? 1 : -1) * NaturalOrder.naturalOrder.compare(a, b); + } } public final int compare(final byte[] a, final int aoffset, final int alength, final byte[] b, final int boffset, final int blength) { - return (asc) ? - compare0(a, aoffset, alength, b, boffset, blength) - : - compare0(b, boffset, blength, a, aoffset, alength); + try { + return (asc) ? + compare0(a, aoffset, alength, b, boffset, blength) + : + compare0(b, boffset, blength, a, aoffset, alength); + } catch (Exception e) { + // same handling as in simple compare method above + boolean wfa = wellformed(a, aoffset, alength); + boolean wfb = wellformed(b, boffset, blength); + if (wfa && wfb) { + // uh strange. throw the exception + if (e instanceof ArrayIndexOutOfBoundsException) throw (ArrayIndexOutOfBoundsException) e; + throw new RuntimeException(e.getMessage()); + } + if (wfa) return (asc) ? -1 : 1; + if (wfb) return (asc) ? 1 : -1; + return ((asc) ? 1 : -1) * NaturalOrder.naturalOrder.compare(a, aoffset, alength, b, boffset, blength); + } } private final int compare0(final byte[] a, final int aoffset, final int alength, final byte[] b, final int boffset, final int blength) { diff --git a/source/de/anomic/kelondro/order/ByteOrder.java b/source/de/anomic/kelondro/order/ByteOrder.java index 7b5767a10..6bbe46a0d 100644 --- a/source/de/anomic/kelondro/order/ByteOrder.java +++ b/source/de/anomic/kelondro/order/ByteOrder.java @@ -27,11 +27,11 @@ package de.anomic.kelondro.order; -import java.util.Comparator; - public interface ByteOrder extends Order { + public boolean wellformed(byte[] a); + public boolean wellformed(byte[] a, int start, int len); public int compare(byte[] a, int astart, int alen, byte[] b, int bstart, int blen); @@ -41,21 +41,5 @@ public interface ByteOrder extends Order { public boolean equal(final byte[] a, int astart, final byte[] b, int bstart, int length); public long cardinal(final byte[] a, int off, int len); - - public final static class StringOrder implements Comparator { - public ByteOrder baseOrder; - public StringOrder(final ByteOrder base) { - this.baseOrder = base; - } - - public StringOrder(final Order base) { - this.baseOrder = (ByteOrder) base; - } - - public final int compare(final String s1, final String s2) { - return baseOrder.compare(s1.getBytes(), s2.getBytes()); - } - - } } diff --git a/source/de/anomic/kelondro/order/NaturalOrder.java b/source/de/anomic/kelondro/order/NaturalOrder.java index c15c2b51d..8d16cd757 100644 --- a/source/de/anomic/kelondro/order/NaturalOrder.java +++ b/source/de/anomic/kelondro/order/NaturalOrder.java @@ -32,7 +32,7 @@ import java.util.Iterator; public final class NaturalOrder extends AbstractOrder implements ByteOrder, Comparator, Cloneable { public static final ByteOrder naturalOrder = new NaturalOrder(true); - public static final Comparator naturalComparator = new ByteOrder.StringOrder(naturalOrder); + public static final Comparator naturalComparator = new StringOrder(naturalOrder); public NaturalOrder(final boolean ascending) { this.asc = ascending; this.zero = null; diff --git a/source/de/anomic/kelondro/order/StringOrder.java b/source/de/anomic/kelondro/order/StringOrder.java new file mode 100644 index 000000000..76abaf968 --- /dev/null +++ b/source/de/anomic/kelondro/order/StringOrder.java @@ -0,0 +1,46 @@ +// StringOrder.java +// ----------------------------- +// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 10.01.2008 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro.order; + +import java.util.Comparator; + +public class StringOrder implements Comparator { + + public ByteOrder baseOrder; + public StringOrder(final ByteOrder base) { + this.baseOrder = base; + } + + public StringOrder(final Order base) { + this.baseOrder = (ByteOrder) base; + } + + public final int compare(final String s1, final String s2) { + return baseOrder.compare(s1.getBytes(), s2.getBytes()); + } +} diff --git a/source/de/anomic/kelondro/table/Table.java b/source/de/anomic/kelondro/table/Table.java index 057d68496..50ec67cb9 100644 --- a/source/de/anomic/kelondro/table/Table.java +++ b/source/de/anomic/kelondro/table/Table.java @@ -1,4 +1,4 @@ -// kelondroEcoIndex.java +// Table.java // (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // first published 14.01.2008 on http://yacy.net // @@ -133,6 +133,7 @@ public class Table implements ObjectIndex { Log.logSevere("TABLE", tablefile + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB"); } index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, 100000); + HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, 10); Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED")); // read all elements from the file into the copy table @@ -146,7 +147,11 @@ public class Table implements ObjectIndex { // write the key into the index table assert key != null; if (key == null) {i++; continue;} - index.putUnique(key, i++); + if (rowdef.objectOrder.wellformed(key)) { + index.putUnique(key, i++); + } else { + errors.putUnique(key, i++); + } } } else { byte[] record; @@ -159,13 +164,16 @@ public class Table implements ObjectIndex { System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength); // write the key into the index table - index.putUnique(key, i++); - - // write the tail into the table - table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); - if (abandonTable()) { - table = null; - break; + if (rowdef.objectOrder.wellformed(key)) { + index.putUnique(key, i++); + // write the tail into the table + table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); + if (abandonTable()) { + table = null; + break; + } + } else { + errors.putUnique(key, i++); } } } @@ -173,6 +181,18 @@ public class Table implements ObjectIndex { // open the file this.file = new BufferedEcoFS(new EcoFS(tablefile, rowdef.objectsize), this.buffersize); + // clean up the file by cleaning badly formed entries + int errorc = errors.size(); + int errorcc = 0; + int idx; + for (Entry entry: errors) { + key = entry.getPrimaryKeyBytes(); + idx = (int) entry.getColLong(1); + Log.logWarning("Table", "removing not well-formed entry " + idx + " with key: " + NaturalOrder.arrayList(key, 0, key.length) + ", " + errorcc++ + "/" + errorc); + removeInFile(idx); + } + assert index.size() == this.file.size() : "index.size() = " + index.size() + ", this.file.size() = " + this.file.size(); + // remove doubles if (!freshFile) { final ArrayList doubles = index.removeDoubles(); @@ -444,6 +464,11 @@ public class Table implements ObjectIndex { return replace(row); } + /** + * remove one entry from the file + * @param i an index position within the file (not a byte position) + * @throws IOException + */ private void removeInFile(final int i) throws IOException { assert i >= 0;