From b03caaa57a94d8a2778d865bb6b0fa564f2d2779 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 15 Jun 2010 19:44:05 +0000 Subject: [PATCH] better handling of OOM situations git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6918 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/Table_API_p.java | 3 + htroot/Tables_p.java | 7 +- source/de/anomic/crawler/CrawlProfile.java | 3 + source/de/anomic/crawler/RobotsTxt.java | 16 ++++- source/de/anomic/data/BookmarkDate.java | 5 +- source/de/anomic/data/WorkTables.java | 3 + source/de/anomic/data/blogBoard.java | 4 ++ source/de/anomic/data/blogBoardComments.java | 5 ++ source/de/anomic/data/bookmarksDB.java | 5 ++ source/de/anomic/data/messageBoard.java | 5 ++ source/de/anomic/data/userDB.java | 4 ++ source/de/anomic/data/wiki/wikiBoard.java | 5 ++ source/de/anomic/http/client/Cache.java | 7 ++ source/de/anomic/yacy/yacyNewsDB.java | 4 +- source/de/anomic/yacy/yacySeedDB.java | 3 + source/net/yacy/kelondro/blob/ArrayStack.java | 57 ++++------------ .../net/yacy/kelondro/blob/BEncodedHeap.java | 12 ++-- source/net/yacy/kelondro/blob/BLOB.java | 7 +- source/net/yacy/kelondro/blob/Compressor.java | 19 +++--- source/net/yacy/kelondro/blob/Heap.java | 55 ++++++++-------- .../net/yacy/kelondro/blob/HeapModifier.java | 3 +- source/net/yacy/kelondro/blob/HeapReader.java | 8 +-- .../net/yacy/kelondro/blob/MapDataMining.java | 29 +++++++-- source/net/yacy/kelondro/blob/MapHeap.java | 7 +- source/net/yacy/kelondro/blob/Stack.java | 15 ++++- source/net/yacy/kelondro/blob/Stacks.java | 6 +- source/net/yacy/kelondro/blob/Tables.java | 65 +++++++------------ source/net/yacy/kelondro/rwi/IndexCell.java | 16 ++++- .../kelondro/rwi/ReferenceContainerArray.java | 2 +- .../yacy/kelondro/util/LookAheadIterator.java | 6 +- 30 files changed, 228 insertions(+), 158 deletions(-) diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index 797cd6fb3..367278bec 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.TreeSet; import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import de.anomic.data.WorkTables; @@ -79,6 +80,8 @@ public class Table_API_p { } } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java index 2b6e76047..f0b93739d 100644 --- a/htroot/Tables_p.java +++ b/htroot/Tables_p.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.regex.Pattern; import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import de.anomic.http.server.RequestHeader; @@ -146,6 +147,8 @@ public class Tables_p { } } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } else if (post.containsKey("addrow")) try { // get a new key @@ -153,6 +156,8 @@ public class Tables_p { setEdit(sb, prop, table, pk, columns); } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } else { prop.put("showtable", 1); prop.put("showtable_table", table); @@ -209,7 +214,7 @@ public class Tables_p { return prop; } - private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List columns) throws IOException { + private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List columns) throws IOException, RowSpaceExceededException { prop.put("showedit", 1); prop.put("showedit_table", table); prop.put("showedit_pk", pk); diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index 3fdf01585..2ffe20319 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -208,6 +208,9 @@ public class CrawlProfile { } catch (final IOException e) { Log.logException(e); return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; } if (m == null) return null; return new entry(m); diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index cefac4f5a..939407ff3 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.kelondro.blob.BEncodedHeap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.DateFormatter; @@ -82,7 +83,13 @@ public class RobotsTxt { // this method will always return a non-null value String urlHostPort = getHostPort(theURL); RobotsEntry robotsTxt4Host = null; - Map record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort)); + Map record; + try { + record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort)); + } catch (RowSpaceExceededException e) { + Log.logException(e); + record = null; + } if (record != null) robotsTxt4Host = new RobotsEntry(urlHostPort, record); if (fetchOnlineIfNotAvailableOrNotFresh && ( @@ -105,7 +112,12 @@ public class RobotsTxt { // check the robots table again for all threads that come here because they waited for another one // to complete a download - record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort)); + try { + record = this.robotsTable.get(this.robotsTable.encodedKey(urlHostPort)); + } catch (RowSpaceExceededException e) { + Log.logException(e); + record = null; + } if (record != null) robotsTxt4Host = new RobotsEntry(urlHostPort, record); if (robotsTxt4Host != null && robotsTxt4Host.getLoadedDate() != null && diff --git a/source/de/anomic/data/BookmarkDate.java b/source/de/anomic/data/BookmarkDate.java index 786b04ced..c13940209 100644 --- a/source/de/anomic/data/BookmarkDate.java +++ b/source/de/anomic/data/BookmarkDate.java @@ -35,6 +35,7 @@ import java.util.Map; import de.anomic.data.bookmarksDB.Bookmark; import net.yacy.kelondro.blob.MapHeap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.NaturalOrder; @@ -57,8 +58,10 @@ public class BookmarkDate { map = datesTable.get(date.getBytes()); } catch (final IOException e) { map = null; + } catch (RowSpaceExceededException e) { + map = null; } - if (map==null) return new Entry(date); + if (map == null) return new Entry(date); return new Entry(date, map); } diff --git a/source/de/anomic/data/WorkTables.java b/source/de/anomic/data/WorkTables.java index 4b5d149a7..6ae2d78d3 100644 --- a/source/de/anomic/data/WorkTables.java +++ b/source/de/anomic/data/WorkTables.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.Date; import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.DateFormatter; import de.anomic.server.serverObjects; @@ -66,6 +67,8 @@ public class WorkTables extends Tables { ); } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } Log.logInfo("APICALL", apiurl); } diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index a23bf7a05..e476584c4 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -142,6 +142,10 @@ public class blogBoard { try { record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)).getBytes()); } catch (final IOException e) { + Log.logException(e); + record = null; + } catch (RowSpaceExceededException e) { + Log.logException(e); record = null; } return (record == null) ? diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java index 51d34ef5a..37515f4d2 100644 --- a/source/de/anomic/data/blogBoardComments.java +++ b/source/de/anomic/data/blogBoardComments.java @@ -43,6 +43,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import net.yacy.kelondro.blob.MapHeap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; @@ -130,6 +131,10 @@ public class blogBoardComments { try { record = base.get(copyOfKey.getBytes()); } catch (final IOException e) { + Log.logException(e); + record = null; + } catch (RowSpaceExceededException e) { + Log.logException(e); record = null; } return (record == null) ? diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 76985c00d..54668dd00 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -46,6 +46,7 @@ import java.util.regex.Pattern; import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.util.DateFormatter; @@ -332,6 +333,10 @@ public class bookmarksDB { final Map map = bookmarks.get(urlHash.getBytes()); return (map == null) ? null : new Bookmark(map); } catch (final IOException e) { + Log.logException(e); + return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 5b1115b03..45f53d872 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.TimeZone; import net.yacy.kelondro.blob.MapHeap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; @@ -195,6 +196,10 @@ public class messageBoard { try { record = database.get(key.getBytes()); } catch (final IOException e) { + Log.logException(e); + return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); return null; } return new entry(key, record); diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 0998b2512..f285b98cf 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -97,6 +97,10 @@ public final class userDB { try { record = userTable.get(userName.getBytes()); } catch (final IOException e) { + Log.logException(e); + return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); return null; } if (record == null) return null; diff --git a/source/de/anomic/data/wiki/wikiBoard.java b/source/de/anomic/data/wiki/wikiBoard.java index 2e5eab96b..e4ac69e2c 100644 --- a/source/de/anomic/data/wiki/wikiBoard.java +++ b/source/de/anomic/data/wiki/wikiBoard.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.TimeZone; import net.yacy.kelondro.blob.MapHeap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; @@ -287,6 +288,10 @@ public class wikiBoard { if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes()); return new entry(key, record); } catch (final IOException e) { + Log.logException(e); + return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index 5e57b928e..d5160d1f5 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -45,6 +45,7 @@ import net.yacy.kelondro.blob.Compressor; import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; @@ -175,6 +176,9 @@ public final class Cache { } catch (final IOException e) { Log.logException(e); return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; } if (hdb == null) return null; @@ -203,6 +207,9 @@ public final class Cache { } catch (IOException e) { Log.logException(e); return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; } } diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index e08764a4b..78e88edd7 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -260,7 +260,7 @@ public class yacyNewsDB { private final Map attributes; // elements of the news for a special category - public Record(final String newsString) { + protected Record(final String newsString) { this.attributes = MapTools.string2map(newsString, ","); if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : ""; @@ -272,7 +272,7 @@ public class yacyNewsDB { removeStandards(); } - public Record(final yacySeed mySeed, final String category, final Map attributes) { + protected Record(final yacySeed mySeed, final String category, final Map attributes) { if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")"); if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); this.attributes = attributes; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index ec5823fc6..dcb4158c2 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -554,6 +554,9 @@ public final class yacySeedDB implements AlternativeDomainNames { } catch (final IOException e) { Log.logException(e); return null; + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; } return new yacySeed(hash, entry); } diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 4f9ad54ca..a4d8d01db 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -60,6 +60,7 @@ import net.yacy.kelondro.rwi.ReferenceFactory; import net.yacy.kelondro.rwi.ReferenceIterator; import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.LookAheadIterator; import net.yacy.kelondro.util.NamePrefixThreadFactory; @@ -574,7 +575,7 @@ public class ArrayStack implements BLOB { * @return * @throws IOException */ - public synchronized byte[] get(byte[] key) throws IOException { + public synchronized byte[] get(byte[] key) throws IOException, RowSpaceExceededException { //blobItem bi = keeperOf(key); //return (bi == null) ? null : bi.blob.get(key); @@ -595,65 +596,36 @@ public class ArrayStack implements BLOB { * @throws IOException */ public Iterable getAll(byte[] key) throws IOException { - /* - byte[] b; - ArrayList l = new ArrayList(blobs.size()); - for (blobItem bi: blobs) { - b = bi.blob.get(key); - if (b != null) l.add(b); - } - return l; - */ return new BlobValues(key); } - public class BlobValues implements Iterator, Iterable { + public class BlobValues extends LookAheadIterator { private final Iterator bii; private final byte[] key; - private byte[] next; public BlobValues(byte[] key) { this.bii = blobs.iterator(); this.key = key; - this.next = null; next0(); } - private void next0() { + protected byte[] next0() { while (this.bii.hasNext()) { BLOB b = this.bii.next().blob; if (b == null) continue; try { - this.next = b.get(key); - if (this.next != null) return; + byte[] n = b.get(key); + if (n != null) return n; } catch (IOException e) { Log.logSevere("ArrayStack", "", e); - this.next = null; - return; + return null; + } catch (RowSpaceExceededException e) { + continue; } } - this.next = null; - } - - public Iterator iterator() { - return this; - } - - public boolean hasNext() { - return this.next != null; - } - - public byte[] next() { - byte[] n = this.next; - next0(); - return n; - } - - public void remove() { - throw new UnsupportedOperationException("no remove in BlobValues"); - } - + return null; + } } /** @@ -692,7 +664,7 @@ public class ArrayStack implements BLOB { * @throws IOException * @throws RowSpaceExceededException */ - public synchronized void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException { + public synchronized void put(byte[] key, byte[] b) throws IOException { blobItem bi = (blobs.isEmpty()) ? null : blobs.get(blobs.size() - 1); if (bi == null) System.out.println("bi == null"); @@ -714,8 +686,9 @@ public class ArrayStack implements BLOB { * replace a BLOB entry with another which must be smaller or same size * @param key the primary key * @throws IOException + * @throws RowSpaceExceededException */ - public synchronized int replace(byte[] key, Rewriter rewriter) throws IOException { + public synchronized int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException { int d = 0; for (blobItem bi: blobs) { d += bi.blob.replace(key, rewriter); @@ -1013,8 +986,6 @@ public class ArrayStack implements BLOB { heap.close(true); } catch (final IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); } } diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java index 7161f983a..d4ce2548f 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeap.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java @@ -118,7 +118,7 @@ public class BEncodedHeap implements Iterable get(byte[] pk) throws IOException { + public Map get(byte[] pk) throws IOException, RowSpaceExceededException { byte[] b = this.table.get(pk); if (b == null) return null; return b2m(b); } - public byte[] getProp(byte[] pk, String key) throws IOException { + public byte[] getProp(byte[] pk, String key) throws IOException, RowSpaceExceededException { byte[] b = this.table.get(pk); if (b == null) return null; Map map = b2m(b); diff --git a/source/net/yacy/kelondro/blob/BLOB.java b/source/net/yacy/kelondro/blob/BLOB.java index a7f064ba8..de280da79 100644 --- a/source/net/yacy/kelondro/blob/BLOB.java +++ b/source/net/yacy/kelondro/blob/BLOB.java @@ -103,7 +103,7 @@ public interface BLOB { * @return * @throws IOException */ - public byte[] get(byte[] key) throws IOException; + public byte[] get(byte[] key) throws IOException, RowSpaceExceededException; /** * retrieve the size of the database @@ -128,7 +128,7 @@ public interface BLOB { * @throws IOException * @throws RowSpaceExceededException */ - public void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException; + public void put(byte[] key, byte[] b) throws IOException; /** * replace an existing entry in the BLOB with a new entry @@ -142,8 +142,9 @@ public interface BLOB { * @param b * @return the number of bytes that the rewriter reduced the BLOB * @throws IOException + * @throws RowSpaceExceededException */ - public int replace(byte[] key, Rewriter rewriter) throws IOException; + public int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException; /** * remove a BLOB diff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.java index c147c3f21..9fbf874f9 100644 --- a/source/net/yacy/kelondro/blob/Compressor.java +++ b/source/net/yacy/kelondro/blob/Compressor.java @@ -103,9 +103,6 @@ public class Compressor implements BLOB { while ((entry = writeQueue.take()) != poisonWorkerEntry) { try { Compressor.this.backend.put(entry.getKey().getBytes(), compress(entry.getValue())); - } catch (RowSpaceExceededException e) { - Log.logException(e); - buffer.put(entry.getKey(), entry.getValue()); } catch (IOException e) { Log.logException(e); buffer.put(entry.getKey(), entry.getValue()); @@ -223,7 +220,7 @@ public class Compressor implements BLOB { } } - public synchronized byte[] get(byte[] key) throws IOException { + public synchronized byte[] get(byte[] key) throws IOException, RowSpaceExceededException { // depending on the source of the result, we additionally do entry compression // because if a document was read once, we think that it will not be retrieved another time again soon byte[] b = buffer.remove(new String(key)); @@ -266,10 +263,14 @@ public class Compressor implements BLOB { public synchronized long length(byte[] key) throws IOException { byte[] b = buffer.get(new String(key)); if (b != null) return b.length; - b = this.backend.get(key); - if (b == null) return 0; - b = decompress(b); - return (b == null) ? 0 : b.length; + try { + b = this.backend.get(key); + if (b == null) return 0; + b = decompress(b); + return (b == null) ? 0 : b.length; + } catch (RowSpaceExceededException e) { + throw new IOException(e.getMessage()); + } } private int removeFromQueues(byte[] key) { @@ -350,7 +351,7 @@ public class Compressor implements BLOB { } } - public int replace(byte[] key, Rewriter rewriter) throws IOException { + public int replace(byte[] key, Rewriter rewriter) throws IOException, RowSpaceExceededException { byte[] b = get(key); if (b == null) return 0; byte[] c = rewriter.rewrite(b); diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 06b407bd0..483217253 100755 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -137,15 +137,19 @@ public final class Heap extends HeapModifier implements BLOB { * @throws IOException * @throws RowSpaceExceededException */ - private void add(byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException { + private void add(byte[] key, final byte[] blob) throws IOException { assert blob.length > 0; if ((blob == null) || (blob.length == 0)) return; final int pos = (int) file.length(); - index.put(key, pos); - file.seek(pos); - file.writeInt(this.keylength + blob.length); - file.write(key); - file.write(blob, 0, blob.length); + try { + index.put(key, pos); + file.seek(pos); + file.writeInt(this.keylength + blob.length); + file.write(key); + file.write(blob, 0, blob.length); + } catch (RowSpaceExceededException e) { + throw new IOException(e.getMessage()); // should never occur; + } } /** @@ -154,7 +158,7 @@ public final class Heap extends HeapModifier implements BLOB { * @throws IOException * @throws RowSpaceExceededException */ - private void flushBuffer() throws IOException, RowSpaceExceededException { + private void flushBuffer() throws IOException { // check size of buffer Iterator> i = this.buffer.entrySet().iterator(); int l = 0; @@ -181,11 +185,17 @@ public final class Heap extends HeapModifier implements BLOB { posBuffer = 0; byte[] ba = new byte[l + (4 + this.keylength) * this.buffer.size()]; byte[] b; - while (i.hasNext()) { + TreeMap nextBuffer = new TreeMap(ordering); + flush: while (i.hasNext()) { entry = i.next(); key = normalizeKey(entry.getKey()); blob = entry.getValue(); - index.put(key, posFile); + try { + index.put(key, posFile); + } catch (RowSpaceExceededException e) { + nextBuffer.put(entry.getKey(), blob); + continue flush; + } b = AbstractWriter.int2array(this.keylength + blob.length); assert b.length == 4; assert posBuffer + 4 < ba.length : "posBuffer = " + posBuffer + ", ba.length = " + ba.length; @@ -202,7 +212,7 @@ public final class Heap extends HeapModifier implements BLOB { assert ba.length == posBuffer; // must fit exactly this.file.seek(pos); this.file.write(ba); - this.buffer.clear(); + this.buffer = nextBuffer; this.buffersize = 0; } @@ -213,7 +223,7 @@ public final class Heap extends HeapModifier implements BLOB { * @throws IOException */ @Override - public byte[] get(byte[] key) throws IOException { + public byte[] get(byte[] key) throws IOException, RowSpaceExceededException { key = normalizeKey(key); synchronized (this) { @@ -265,8 +275,6 @@ public final class Heap extends HeapModifier implements BLOB { flushBuffer(); } catch (IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); } } this.buffer = null; @@ -294,9 +302,10 @@ public final class Heap extends HeapModifier implements BLOB { * @param b * @throws IOException * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ @Override - public void put(byte[] key, final byte[] b) throws IOException, RowSpaceExceededException { + public void put(byte[] key, final byte[] b) throws IOException { key = normalizeKey(key); // we do not write records of length 0 into the BLOB @@ -308,7 +317,9 @@ public final class Heap extends HeapModifier implements BLOB { this.remove(key); // then look if we can use a free entry - if (putToGap(key, b)) return; + try { + if (putToGap(key, b)) return; + } catch (RowSpaceExceededException e) {} // too less space can be ignored, we have a second try // if there is not enough space in the buffer, flush all if (this.buffersize + b.length > buffermax) { @@ -446,11 +457,7 @@ public final class Heap extends HeapModifier implements BLOB { */ @Override public synchronized CloneableIterator keys(final boolean up, final boolean rotating) throws IOException { - try { - this.flushBuffer(); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } + this.flushBuffer(); return super.keys(up, rotating); } @@ -463,11 +470,7 @@ public final class Heap extends HeapModifier implements BLOB { */ @Override public synchronized CloneableIterator keys(final boolean up, final byte[] firstKey) throws IOException { - try { - this.flushBuffer(); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } + this.flushBuffer(); return super.keys(up, firstKey); } @@ -500,8 +503,6 @@ public final class Heap extends HeapModifier implements BLOB { heap.close(true); } catch (final IOException e) { Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); } } diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java index 3b0f864c5..b1cdad2ef 100644 --- a/source/net/yacy/kelondro/blob/HeapModifier.java +++ b/source/net/yacy/kelondro/blob/HeapModifier.java @@ -28,7 +28,6 @@ import java.io.File; import java.io.IOException; import java.util.SortedMap; -import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.io.CachedFileWriter; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; @@ -235,7 +234,7 @@ public class HeapModifier extends HeapReader implements BLOB { } } - public void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException { + public void put(byte[] key, byte[] b) throws IOException { throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier"); } diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 79ca2b58b..6747241c2 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -344,7 +344,7 @@ public class HeapReader { * @return the entry which key is the smallest in the heap * @throws IOException */ - protected byte[] first() throws IOException { + protected byte[] first() throws IOException, RowSpaceExceededException { synchronized (this.index) { byte[] key = index.smallestKey(); if (key == null) return null; @@ -372,7 +372,7 @@ public class HeapReader { * @return the entry which key is the smallest in the heap * @throws IOException */ - protected byte[] last() throws IOException { + protected byte[] last() throws IOException, RowSpaceExceededException { synchronized (this.index) { byte[] key = index.largestKey(); if (key == null) return null; @@ -386,7 +386,7 @@ public class HeapReader { * @return * @throws IOException */ - public byte[] get(byte[] key) throws IOException { + public byte[] get(byte[] key) throws IOException, RowSpaceExceededException { key = normalizeKey(key); synchronized (this.index) { @@ -398,7 +398,7 @@ public class HeapReader { file.seek(pos); final int len = file.readInt() - index.row().primaryKeyLength; if (MemoryControl.available() < len * 2 + keepFreeMem) { - if (!MemoryControl.request(len * 2 + keepFreeMem, true)) return null; // not enough memory available for this blob + if (!MemoryControl.request(len * 2 + keepFreeMem, true)) throw new RowSpaceExceededException(len * 2 + keepFreeMem, "HeapReader.get()"); // not enough memory available for this blob } // read the key diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 034e33b5c..adc0cdd6f 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.util.ScoreCluster; @@ -107,7 +108,12 @@ public class MapDataMining extends MapHeap { Map map; while (it.hasNext()) { mapnameb = it.next(); - map = super.get(mapnameb); + try { + map = super.get(mapnameb); + } catch (RowSpaceExceededException e) { + Log.logWarning("MapDataMining", e.getMessage()); + break; + } if (map == null) break; if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) { @@ -250,14 +256,20 @@ public class MapDataMining extends MapHeap { // update elementCount if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) { - final Map map = super.get(key); - if (map != null) { + Map map; + try { + map = super.get(key); + if (map != null) { - // update accumulators (subtract) - if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false); + // update accumulators (subtract) + if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false); - // remove from sortCluster - if (sortfields != null) deleteSortCluster(new String(key)); + // remove from sortCluster + if (sortfields != null) deleteSortCluster(new String(key)); + } + } catch (RowSpaceExceededException e) { + map = null; + Log.logException(e); } } super.remove(key); @@ -383,6 +395,9 @@ public class MapDataMining extends MapHeap { map = get(nextKey); } catch (final IOException e) { break; + } catch (RowSpaceExceededException e) { + Log.logException(e); + continue; } assert map != null; if (map == null) continue; // circumvention of a modified exception diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 8dd665308..5fb4b4ac0 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -189,7 +189,7 @@ public class MapHeap { * @return * @throws IOException */ - public Map get(final byte[] key) throws IOException { + public Map get(final byte[] key) throws IOException, RowSpaceExceededException { if (key == null) return null; return get(key, true); } @@ -210,7 +210,7 @@ public class MapHeap { return key; } - protected Map get(byte[] key, final boolean storeCache) throws IOException { + protected Map get(byte[] key, final boolean storeCache) throws IOException, RowSpaceExceededException { // load map from cache assert key != null; if (cache == null) return null; // case may appear during shutdown @@ -350,6 +350,9 @@ public class MapHeap { } catch (final IOException e) { finish = true; return null; + } catch (final RowSpaceExceededException e) { + finish = true; + return null; } } diff --git a/source/net/yacy/kelondro/blob/Stack.java b/source/net/yacy/kelondro/blob/Stack.java index 59abf47e0..d8373eed6 100644 --- a/source/net/yacy/kelondro/blob/Stack.java +++ b/source/net/yacy/kelondro/blob/Stack.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Iterator; import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.NaturalOrder; @@ -116,8 +117,9 @@ public class Stack { * @return the object that belongs to the handle * or null if no such element exists * @throws IOException + * @throws RowSpaceExceededException */ - public synchronized byte[] get(final long handle) throws IOException { + public synchronized byte[] get(final long handle) throws IOException, RowSpaceExceededException { byte[] k = NaturalOrder.encodeLong(handle, 8); byte[] b = this.stack.get(k); if (b == null) return null; @@ -129,8 +131,9 @@ public class Stack { * @param handle * @return the removed element * @throws IOException + * @throws RowSpaceExceededException */ - public synchronized byte[] remove(final long handle) throws IOException { + public synchronized byte[] remove(final long handle) throws IOException, RowSpaceExceededException { byte[] k = NaturalOrder.encodeLong(handle, 8); byte[] b = this.stack.get(k); if (b == null) return null; @@ -181,7 +184,13 @@ public class Stack { private Entry po(final byte[] k, final boolean remove) throws IOException { if (k == null) return null; assert k.length == 8; - byte[] b = this.stack.get(k); + byte[] b; + try { + b = this.stack.get(k); + } catch (RowSpaceExceededException e) { + Log.logException(e); + b = null; + } assert b != null; if (b == null) return null; if (remove) this.stack.remove(k); diff --git a/source/net/yacy/kelondro/blob/Stacks.java b/source/net/yacy/kelondro/blob/Stacks.java index 7d5b7f6a2..1a0d4f409 100644 --- a/source/net/yacy/kelondro/blob/Stacks.java +++ b/source/net/yacy/kelondro/blob/Stacks.java @@ -178,8 +178,9 @@ public class Stacks { * @return the object that belongs to the handle * or null if no such element exists * @throws IOException + * @throws RowSpaceExceededException */ - public byte[] get(final String stack, final long handle) throws IOException { + public byte[] get(final String stack, final long handle) throws IOException, RowSpaceExceededException { Stack s = getStack(stack); if (s == null) return null; return s.get(handle); @@ -191,8 +192,9 @@ public class Stacks { * @param handle * @return the removed element * @throws IOException + * @throws RowSpaceExceededException */ - public byte[] remove(final String stack, final long handle) throws IOException { + public byte[] remove(final String stack, final long handle) throws IOException, RowSpaceExceededException { Stack s = getStack(stack); if (s == null) return null; return s.remove(handle); diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index 46bc59ce9..4dc6bdd35 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -130,7 +130,7 @@ public class Tables { return heap.size(); } - private byte[] ukey(String tablename) throws IOException { + private byte[] ukey(String tablename) throws IOException, RowSpaceExceededException { Row row = select(system_table_pkcounter, tablename.getBytes()); if (row == null) { // table counter entry in pkcounter table does not exist: make a new table entry @@ -166,15 +166,16 @@ public class Tables { * @param map * @throws RowSpaceExceededException * @throws IOException + * @throws RowSpaceExceededException */ - public byte[] insert(final String tablename, Map map) throws IOException { + public byte[] insert(final String tablename, Map map) throws IOException, RowSpaceExceededException { byte[] uk = ukey(tablename); insert(tablename, uk, map); insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk); return uk; } - public byte[] insert(final String tablename, String key, byte[] value) throws IOException { + public byte[] insert(final String tablename, String key, byte[] value) throws IOException, RowSpaceExceededException { byte[] uk = ukey(tablename); insert(tablename, uk, key, value); insert(system_table_pkcounter, tablename.getBytes(), system_table_pkcounter_counterName, uk); @@ -184,7 +185,7 @@ public class Tables { public byte[] insert(final String tablename, String key0, byte[] value0, String key1, byte[] value1 - ) throws IOException { + ) throws IOException, RowSpaceExceededException { byte[] uk = ukey(tablename); insert(tablename, uk, key0, value0, @@ -198,7 +199,7 @@ public class Tables { String key0, byte[] value0, String key1, byte[] value1, String key2, byte[] value2 - ) throws IOException { + ) throws IOException, RowSpaceExceededException { byte[] uk = ukey(tablename); insert(tablename, uk, key0, value0, @@ -214,7 +215,7 @@ public class Tables { String key1, byte[] value1, String key2, byte[] value2, String key3, byte[] value3 - ) throws IOException { + ) throws IOException, RowSpaceExceededException { byte[] uk = ukey(tablename); insert(tablename, uk, key0, value0, @@ -230,11 +231,7 @@ public class Tables { String key, byte[] value ) throws IOException { BEncodedHeap heap = getHeap(table); - try { - heap.put(pk, key, value); - } catch (RowSpaceExceededException e) { - throw new IOException(e.getMessage()); - } + heap.put(pk, key, value); } public void insert(final String table, byte[] pk, @@ -242,14 +239,10 @@ public class Tables { String key1, byte[] value1 ) throws IOException { BEncodedHeap heap = getHeap(table); - try { - heap.put(pk, - key0, value0, - key1, value1 - ); - } catch (RowSpaceExceededException e) { - throw new IOException(e.getMessage()); - } + heap.put(pk, + key0, value0, + key1, value1 + ); } public void insert(final String table, byte[] pk, @@ -258,15 +251,11 @@ public class Tables { String key2, byte[] value2 ) throws IOException { BEncodedHeap heap = getHeap(table); - try { - heap.put(pk, - key0, value0, - key1, value1, - key2, value2 - ); - } catch (RowSpaceExceededException e) { - throw new IOException(e.getMessage()); - } + heap.put(pk, + key0, value0, + key1, value1, + key2, value2 + ); } public void insert(final String table, byte[] pk, @@ -276,16 +265,12 @@ public class Tables { String key3, byte[] value3 ) throws IOException { BEncodedHeap heap = getHeap(table); - try { - heap.put(pk, - key0, value0, - key1, value1, - key2, value2, - key3, value3 - ); - } catch (RowSpaceExceededException e) { - throw new IOException(e.getMessage()); - } + heap.put(pk, + key0, value0, + key1, value1, + key2, value2, + key3, value3 + ); } public void insert(final String table, byte[] pk, Map map) throws IOException { @@ -306,11 +291,11 @@ public class Tables { } } - public byte[] createRow(String table) throws IOException { + public byte[] createRow(String table) throws IOException, RowSpaceExceededException { return this.insert(table, new ConcurrentHashMap()); } - public Row select(final String table, byte[] pk) throws IOException { + public Row select(final String table, byte[] pk) throws IOException, RowSpaceExceededException { BEncodedHeap heap = getHeap(table); if (heap.has(pk)) return new Row(pk, heap.get(pk)); return null; diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index b09e62dd8..eef2d2f16 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -248,13 +248,25 @@ public final class IndexCell extends AbstractBu */ public int remove(byte[] termHash, HandleSet urlHashes) throws IOException { int removed = this.ram.remove(termHash, urlHashes); - int reduced = this.array.replace(termHash, new RemoveRewriter(urlHashes)); + int reduced; + try { + reduced = this.array.replace(termHash, new RemoveRewriter(urlHashes)); + } catch (RowSpaceExceededException e) { + reduced = 0; + Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment"); + } return removed + (reduced / this.array.rowdef().objectsize); } public boolean remove(byte[] termHash, byte[] urlHashBytes) throws IOException { boolean removed = this.ram.remove(termHash, urlHashBytes); - int reduced = this.array.replace(termHash, new RemoveRewriter(urlHashBytes)); + int reduced; + try { + reduced = this.array.replace(termHash, new RemoveRewriter(urlHashBytes)); + } catch (RowSpaceExceededException e) { + reduced = 0; + Log.logWarning("IndexCell", "not possible to remove urlHashes from a RWI because of too low memory. Remove was not applied. Please increase RAM assignment"); + } return removed || (reduced > 0); } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index a45a03f18..bcb67ab52 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -247,7 +247,7 @@ public final class ReferenceContainerArray { array.remove(termHash); } - public int replace(final byte[] termHash, ContainerRewriter rewriter) throws IOException { + public int replace(final byte[] termHash, ContainerRewriter rewriter) throws IOException, RowSpaceExceededException { return array.replace(termHash, new BLOBRewriter(termHash, rewriter)); } diff --git a/source/net/yacy/kelondro/util/LookAheadIterator.java b/source/net/yacy/kelondro/util/LookAheadIterator.java index 16b5f7cac..1f5d660a6 100644 --- a/source/net/yacy/kelondro/util/LookAheadIterator.java +++ b/source/net/yacy/kelondro/util/LookAheadIterator.java @@ -35,13 +35,17 @@ import java.util.Iterator; * latest return value of next0() * To use this class just implement the next0() method */ -public abstract class LookAheadIterator implements Iterator { +public abstract class LookAheadIterator implements Iterator, Iterable { private boolean fresh = true; private A next = null; public LookAheadIterator() { } + + public Iterator iterator() { + return this; + } /** * the internal next-method