boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- patch to omit IndexOutOfBoundsException when a b64-encoded key appears not to be well-formed. In that case the key is still accepted but rated higher than other regular keys to create a virtual ordering between well-formed and ill-formed keys

Browse Source 
- check routine at the beginning of the import of table keys that check that all imported keys are well-formed. All records that have a ill-formed key are deleted. This is a hack and is not tested since I don't have bad data here to test with. If the effect is seen in the wild, please report in the forum.


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6245 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 65b1d51e70
commit fbfdaf063d
7 changed files with 131 additions and 41 deletions
Show Stats Download Patch File Download Diff File

16
source/de/anomic/http/client/Client.java
Unescape View File

@ -208,8 +208,10 @@ public class Client {
* @see de.anomic.http.HttpClient#setProxy(de.anomic.http.httpRemoteProxyConfig)
*/
public void setProxy(final RemoteProxyConfig proxyConfig) {
this.useGlobalProxyConfig = false;
this.proxyConfig = proxyConfig;
if (proxyConfig != null) {
this.useGlobalProxyConfig = false;
this.proxyConfig = proxyConfig;
}
}
/*
@ -566,16 +568,16 @@ public class Client {
*/
private RemoteProxyConfig getProxyConfig(final String hostname) {
final RemoteProxyConfig hostProxyConfig;
if (!useGlobalProxyConfig) {
if (useGlobalProxyConfig) {
// default settings
hostProxyConfig = RemoteProxyConfig.getProxyConfigForHost(hostname);
} else {
// client specific
if(proxyConfig == null) {
if (proxyConfig == null) {
hostProxyConfig = null;
} else {
hostProxyConfig = proxyConfig.useForHost(hostname) ? proxyConfig : null;
}
} else {
// default settings
hostProxyConfig = RemoteProxyConfig.getProxyConfigForHost(hostname);
}
return hostProxyConfig;
}

4
source/de/anomic/http/client/RemoteProxyConfig.java
Unescape View File

@ -63,8 +63,8 @@ public final class RemoteProxyConfig {
/**
* @param remoteProxyConfig the remoteProxyConfig to set
*/
public static synchronized void setRemoteProxyConfig(final RemoteProxyConfig remoteProxyConfig) {
RemoteProxyConfig.remoteProxyConfig = remoteProxyConfig;
public static synchronized void setRemoteProxyConfig(final RemoteProxyConfig newConfig) {
remoteProxyConfig = newConfig;
}
/**

41
source/de/anomic/kelondro/order/Base64Order.java
Unescape View File

@ -403,17 +403,50 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
}
public final int compare(final byte[] a, final byte[] b) {
try {
return (asc) ?
((zero == null) ? compares(a, b) : compare0(a, 0, a.length, b, 0, b.length))
:
((zero == null) ? compares(b, a) : compare0(b, 0, b.length, a, 0, a.length));
} catch (Exception e) {
// if a or b is not well-formed, an ArrayIndexOutOfBoundsException may occur
// in that case we don't want that the exception makes databse functions
// unusable and effective creates a showstopper. In such cases we apply
// a different order on the objects and treat not well-formed objects
// as bigger as all others. If both object are not well-formed, they are
// compared with the natural order.
boolean wfa = wellformed(a);
boolean wfb = wellformed(b);
if (wfa && wfb) {
// uh strange. throw the exception
if (e instanceof ArrayIndexOutOfBoundsException) throw (ArrayIndexOutOfBoundsException) e;
throw new RuntimeException(e.getMessage());
}
if (wfa) return (asc) ? -1 : 1;
if (wfb) return (asc) ? 1 : -1;
return ((asc) ? 1 : -1) * NaturalOrder.naturalOrder.compare(a, b);
}
}
public final int compare(final byte[] a, final int aoffset, final int alength, final byte[] b, final int boffset, final int blength) {
return (asc) ?
compare0(a, aoffset, alength, b, boffset, blength)
:
compare0(b, boffset, blength, a, aoffset, alength);
try {
return (asc) ?
compare0(a, aoffset, alength, b, boffset, blength)
:
compare0(b, boffset, blength, a, aoffset, alength);
} catch (Exception e) {
// same handling as in simple compare method above
boolean wfa = wellformed(a, aoffset, alength);
boolean wfb = wellformed(b, boffset, blength);
if (wfa && wfb) {
// uh strange. throw the exception
if (e instanceof ArrayIndexOutOfBoundsException) throw (ArrayIndexOutOfBoundsException) e;
throw new RuntimeException(e.getMessage());
}
if (wfa) return (asc) ? -1 : 1;
if (wfb) return (asc) ? 1 : -1;
return ((asc) ? 1 : -1) * NaturalOrder.naturalOrder.compare(a, aoffset, alength, b, boffset, blength);
}
}
private final int compare0(final byte[] a, final int aoffset, final int alength, final byte[] b, final int boffset, final int blength) {

20
source/de/anomic/kelondro/order/ByteOrder.java
Unescape View File

@ -27,11 +27,11 @@
package de.anomic.kelondro.order;
import java.util.Comparator;
public interface ByteOrder extends Order<byte[]> {
public boolean wellformed(byte[] a);
public boolean wellformed(byte[] a, int start, int len);
public int compare(byte[] a, int astart, int alen, byte[] b, int bstart, int blen);
@ -41,21 +41,5 @@ public interface ByteOrder extends Order<byte[]> {
public boolean equal(final byte[] a, int astart, final byte[] b, int bstart, int length);
public long cardinal(final byte[] a, int off, int len);
public final static class StringOrder implements Comparator<String> {
public ByteOrder baseOrder;
public StringOrder(final ByteOrder base) {
this.baseOrder = base;
}
public StringOrder(final Order<byte[]> base) {
this.baseOrder = (ByteOrder) base;
}
public final int compare(final String s1, final String s2) {
return baseOrder.compare(s1.getBytes(), s2.getBytes());
}
}
}

2
source/de/anomic/kelondro/order/NaturalOrder.java
Unescape View File

@ -32,7 +32,7 @@ import java.util.Iterator;
public final class NaturalOrder extends AbstractOrder<byte[]> implements ByteOrder, Comparator<byte[]>, Cloneable {
public static final ByteOrder naturalOrder = new NaturalOrder(true);
public static final Comparator<String> naturalComparator = new ByteOrder.StringOrder(naturalOrder);
public static final Comparator<String> naturalComparator = new StringOrder(naturalOrder);
public NaturalOrder(final boolean ascending) {
this.asc = ascending;
this.zero = null;

46
source/de/anomic/kelondro/order/StringOrder.java
Unescape View File

@ -0,0 +1,46 @@
// StringOrder.java
// -----------------------------
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 10.01.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro.order;
import java.util.Comparator;
public class StringOrder implements Comparator<String> {
public ByteOrder baseOrder;
public StringOrder(final ByteOrder base) {
this.baseOrder = base;
}
public StringOrder(final Order<byte[]> base) {
this.baseOrder = (ByteOrder) base;
}
public final int compare(final String s1, final String s2) {
return baseOrder.compare(s1.getBytes(), s2.getBytes());
}
}

43
source/de/anomic/kelondro/table/Table.java
Unescape View File

@ -1,4 +1,4 @@
// kelondroEcoIndex.java
// Table.java
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 14.01.2008 on http://yacy.net
//
@ -133,6 +133,7 @@ public class Table implements ObjectIndex {
Log.logSevere("TABLE", tablefile + ": RAM after releasing the table: " + (MemoryControl.available() / 1024 / 1024) + "MB");
}
index = new HandleMap(rowdef.primaryKeyLength, rowdef.objectOrder, 4, records, 100000);
HandleMap errors = new HandleMap(rowdef.primaryKeyLength, NaturalOrder.naturalOrder, 4, records, 10);
Log.logInfo("TABLE", tablefile + ": TABLE " + tablefile.toString() + " has table copy " + ((table == null) ? "DISABLED" : "ENABLED"));
// read all elements from the file into the copy table
@ -146,7 +147,11 @@ public class Table implements ObjectIndex {
// write the key into the index table
assert key != null;
if (key == null) {i++; continue;}
index.putUnique(key, i++);
if (rowdef.objectOrder.wellformed(key)) {
index.putUnique(key, i++);
} else {
errors.putUnique(key, i++);
}
}
} else {
byte[] record;
@ -159,13 +164,16 @@ public class Table implements ObjectIndex {
System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
// write the key into the index table
index.putUnique(key, i++);
// write the tail into the table
table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true));
if (abandonTable()) {
table = null;
break;
if (rowdef.objectOrder.wellformed(key)) {
index.putUnique(key, i++);
// write the tail into the table
table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true));
if (abandonTable()) {
table = null;
break;
}
} else {
errors.putUnique(key, i++);
}
}
}
@ -173,6 +181,18 @@ public class Table implements ObjectIndex {
// open the file
this.file = new BufferedEcoFS(new EcoFS(tablefile, rowdef.objectsize), this.buffersize);
// clean up the file by cleaning badly formed entries
int errorc = errors.size();
int errorcc = 0;
int idx;
for (Entry entry: errors) {
key = entry.getPrimaryKeyBytes();
idx = (int) entry.getColLong(1);
Log.logWarning("Table", "removing not well-formed entry " + idx + " with key: " + NaturalOrder.arrayList(key, 0, key.length) + ", " + errorcc++ + "/" + errorc);
removeInFile(idx);
}
assert index.size() == this.file.size() : "index.size() = " + index.size() + ", this.file.size() = " + this.file.size();
// remove doubles
if (!freshFile) {
final ArrayList<Long[]> doubles = index.removeDoubles();
@ -444,6 +464,11 @@ public class Table implements ObjectIndex {
return replace(row);
}
/**
* remove one entry from the file
* @param i an index position within the file (not a byte position)
* @throws IOException
*/
private void removeInFile(final int i) throws IOException {
assert i >= 0;

Write Preview
Loading…
Cancel
Save