diff --git a/htroot/ConfigAccounts_p.java b/htroot/ConfigAccounts_p.java index 6882ee043..bde2fc19d 100644 --- a/htroot/ConfigAccounts_p.java +++ b/htroot/ConfigAccounts_p.java @@ -29,10 +29,10 @@ //javac -classpath .:../Classes Message.java //if the shell's current path is HTROOT -import java.io.IOException; import java.util.HashMap; import java.util.Iterator; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; @@ -209,7 +209,8 @@ public class ConfigAccounts_p { entry.setProperty(userDB.Entry.TIME_USED, timeUsed); for(i=0;i icc = new ReferenceContainerCache(Segment.wordReferenceFactory, index.rowdef, Segment.wordOrder); - icc.add(index); + try { + icc.add(index); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } // transport to other peer final String gzipBody = sb.getConfig("indexControl.gzipBody","false"); diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 51ff28dbb..f7f9442ce 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -242,7 +242,7 @@ public class IndexControlURLs_p { // parse format int format = 0; final String fname = post.get("format", "url-text"); - final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, othervise only the domain + final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain if (fname.endsWith("text")) format = 0; if (fname.endsWith("html")) format = 1; if (fname.endsWith("rss")) format = 2; diff --git a/htroot/News.java b/htroot/News.java index be6045f39..99c169e15 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -24,7 +24,6 @@ // javac -classpath .:../classes Network.java // if the shell's current path is HTROOT -import java.io.IOException; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -65,7 +64,7 @@ public class News { id = check.substring(4); try { sb.peers.newsPool.moveOff(tableID, id); - } catch (final IOException ee) {Log.logException(ee);} + } catch (final Exception ee) {Log.logException(ee);} } } } @@ -81,7 +80,7 @@ public class News { } else { sb.peers.newsPool.moveOffAll(tableID); } - } catch (final IOException e) { + } catch (final Exception e) { Log.logException(e); } } diff --git a/htroot/User.java b/htroot/User.java index 1b51cfa3a..b6c3c18c9 100644 --- a/htroot/User.java +++ b/htroot/User.java @@ -27,8 +27,7 @@ //javac -classpath .:../Classes Message.java //if the shell's current path is HTROOT -import java.io.IOException; - +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; @@ -131,7 +130,9 @@ public class User{ try { entry.setProperty(userDB.Entry.MD5ENCODED_USERPWD_STRING, Digest.encodeMD5Hex(entry.getUserName()+":"+post.get("newpass", ""))); prop.put("status_password", "0"); //changes - } catch (final IOException e) {} + } catch (final Exception e) { + Log.logException(e); + } }else{ prop.put("status_password", "3"); //empty } diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index c23477a64..ba58cbfec 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -41,6 +41,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Properties; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import de.anomic.http.server.RequestHeader; @@ -100,7 +101,9 @@ public class ViewProfile { try { final yacyNewsRecord record = sb.peers.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash); if (record != null) sb.peers.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id()); - } catch (final IOException e) {} + } catch (final Exception e) { + Log.logException(e); + } // try to get the profile from remote peer if (sb.clusterhashes != null) seed.setAlternativeAddress(sb.clusterhashes.get(seed.hash.getBytes())); diff --git a/htroot/api/timeline.java b/htroot/api/timeline.java index 0bf23e5fc..cd83c520a 100644 --- a/htroot/api/timeline.java +++ b/htroot/api/timeline.java @@ -30,6 +30,8 @@ import java.util.TreeSet; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.TermSearch; import net.yacy.kelondro.util.DateFormatter; @@ -89,12 +91,12 @@ public final class timeline { //yacyCore.log.logInfo("INIT TIMELINE SEARCH: " + plasmaSearchQuery.anonymizedQueryHashes(query[0]) + " - " + count + " links"); // get the index container with the result vector - final TermSearch search = segment.termIndex().query( - q, - Word.words2hashes(query[1]), - null, - Segment.wordReferenceFactory, - maxdist); + TermSearch search = null; + try { + search = segment.termIndex().query(q, Word.words2hashes(query[1]), null, Segment.wordReferenceFactory, maxdist); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } ReferenceContainer index = search.joined(); Iterator i = index.entries(); diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index fd2bacca8..4e5ffe52e 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -27,7 +27,6 @@ // javac -classpath .:../classes transferRWI.java -import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; @@ -183,7 +182,7 @@ public final class transferRWI { // learn entry try { sb.indexSegments.termIndex(Segments.Process.DHTIN).add(wordHash.getBytes(), iEntry); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } serverCore.checkInterruption(); diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 277ed5073..cf409f78e 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.table.Table; @@ -77,7 +78,15 @@ public class Balancer { if (!(cachePath.exists())) cachePath.mkdir(); // make the path cacheStacksPath.mkdirs(); File f = new File(cacheStacksPath, stackname + indexSuffix); - urlFileIndex = new Table(f, Request.rowdef, EcoFSBufferSize, 0, useTailCache, exceed134217727); + try { + urlFileIndex = new Table(f, Request.rowdef, EcoFSBufferSize, 0, useTailCache, exceed134217727); + } catch (RowSpaceExceededException e) { + try { + urlFileIndex = new Table(f, Request.rowdef, 0, 0, false, exceed134217727); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + } + } lastDomainStackFill = 0; Log.logInfo("Balancer", "opened balancer file with " + urlFileIndex.size() + " entries from " + f.toString()); } @@ -229,7 +238,7 @@ public class Balancer { return false; } - public void push(final Request entry) throws IOException { + public void push(final Request entry) throws IOException, RowSpaceExceededException { assert entry != null; String hash = entry.url().hash(); synchronized (this) { @@ -303,6 +312,7 @@ public class Balancer { * @param profile * @return a url in a CrawlEntry object * @throws IOException + * @throws RowSpaceExceededException */ public Request pop(final boolean delay, final CrawlProfile profile) throws IOException { // returns a crawl entry from the stack and ensures minimum delta times @@ -389,9 +399,13 @@ public class Balancer { //System.out.println("*** delayed +=" + nexthash); this.delayed.put(new Long(System.currentTimeMillis() + sleeptime + 1), nexthash); } - this.urlFileIndex.put(rowEntry); - this.domainStacks.remove(nexthash.substring(6)); - failhash = nexthash; + try { + this.urlFileIndex.put(rowEntry); + this.domainStacks.remove(nexthash.substring(6)); + failhash = nexthash; + } catch (RowSpaceExceededException e) { + Log.logException(e); + } continue; } break; diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index a5e6f4a06..8adb3814a 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -35,6 +35,7 @@ import net.yacy.kelondro.blob.Heap; import net.yacy.kelondro.blob.MapView; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.CloneableIterator; @@ -155,19 +156,11 @@ public class CrawlProfile { final entry ne = new entry(mem); try { profileTable.put(ne.handle(), ne.map()); - } catch (final kelondroException e) { + } catch (final Exception e) { clear(); try { profileTable.put(ne.handle(), ne.map()); - } catch (final IOException ee) { - Log.logException(e); - System.exit(0); - } - } catch (final IOException e) { - clear(); - try { - profileTable.put(ne.handle(), ne.map()); - } catch (final IOException ee) { + } catch (final Exception ee) { Log.logException(e); System.exit(0); } @@ -200,19 +193,11 @@ public class CrawlProfile { cacheStrategy); try { profileTable.put(ne.handle(), ne.map()); - } catch (final kelondroException e) { - clear(); - try { - profileTable.put(ne.handle(), ne.map()); - } catch (final IOException ee) { - Log.logException(e); - System.exit(0); - } - } catch (final IOException e) { + } catch (final Exception e) { clear(); try { profileTable.put(ne.handle(), ne.map()); - } catch (final IOException ee) { + } catch (final Exception ee) { Log.logException(e); System.exit(0); } @@ -241,7 +226,7 @@ public class CrawlProfile { return new entry(m); } - public void changeEntry(final entry e, final String propName, final String newValue) throws IOException { + public void changeEntry(final entry e, final String propName, final String newValue) throws IOException, RowSpaceExceededException { e.mem.put(propName, newValue); profileTable.put(e.handle(), e.mem); } @@ -269,7 +254,7 @@ public class CrawlProfile { public final static int CACHE_STRATEGY_NOCACHE = 0; // never use the cache, all content from fresh internet source public final static int CACHE_STRATEGY_IFFRESH = 1; // use the cache if the cache exists and is fresh using the proxy-fresh rules - public final static int CACHE_STRATEGY_IFEXIST = 2; // use the cache if the cache exist. Do no check freshness. Othervise use online source. + public final static int CACHE_STRATEGY_IFEXIST = 2; // use the cache if the cache exist. Do no check freshness. Otherwise use online source. public final static int CACHE_STRATEGY_CACHEONLY = 3; // never go online, use all content from cache. If no cache exist, treat content as unavailable public static class entry { diff --git a/source/de/anomic/crawler/NoticedURL.java b/source/de/anomic/crawler/NoticedURL.java index 5f077c94e..bf76603b3 100755 --- a/source/de/anomic/crawler/NoticedURL.java +++ b/source/de/anomic/crawler/NoticedURL.java @@ -161,7 +161,9 @@ public class NoticedURL { break; default: break; } - } catch (final IOException er) {} + } catch (final Exception er) { + Log.logException(er); + } } public Request get(final String urlhash) { diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index c6c3d3037..ad555f178 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -256,7 +256,8 @@ public class RobotsTxt { try { this.robotsTable.put(entry.hostName, entry.mem); return entry.hostName; - } catch (final IOException e) { + } catch (final Exception e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 288cad8d6..94b51767f 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -38,6 +38,7 @@ import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.table.SplitTable; @@ -62,7 +63,7 @@ public class ZURL implements Iterable { ); // the class object - protected final ObjectIndex urlIndex; + protected ObjectIndex urlIndex; protected final ConcurrentLinkedQueue stack; public ZURL( @@ -79,7 +80,15 @@ public class ZURL implements Iterable { if (f.isDirectory()) SplitTable.delete(cachePath, tablename); else FileUtils.deletedelete(f); } } - this.urlIndex = new Table(f, rowdef, EcoFSBufferSize, 0, useTailCache, exceed134217727); + try { + this.urlIndex = new Table(f, rowdef, EcoFSBufferSize, 0, useTailCache, exceed134217727); + } catch (RowSpaceExceededException e) { + try { + this.urlIndex = new Table(f, rowdef, 0, 0, false, exceed134217727); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + } + } //urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, 0, true); this.stack = new ConcurrentLinkedQueue(); } @@ -236,8 +245,8 @@ public class ZURL implements Iterable { //System.out.println("*** DEBUG ZURL " + urlIndex.filename() + " store " + newrow.getColString(0, "UTF-8")); if (urlIndex != null) urlIndex.put(newrow); this.stored = true; - } catch (final IOException e) { - System.out.println("INTERNAL ERROR AT plasmaEURL:url2hash:" + e.toString()); + } catch (final Exception e) { + Log.logException(e); } } diff --git a/source/de/anomic/data/URLAnalysis.java b/source/de/anomic/data/URLAnalysis.java index af0717070..74dea3ce2 100644 --- a/source/de/anomic/data/URLAnalysis.java +++ b/source/de/anomic/data/URLAnalysis.java @@ -55,6 +55,7 @@ import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.index.HandleMap; import net.yacy.kelondro.index.HandleSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.rwi.ReferenceContainerArray; @@ -406,12 +407,12 @@ public class URLAnalysis { System.out.println("INDEX REFERENCE COLLECTION starting dump of statistics"); idx.dump(new File(statisticPath)); System.out.println("INDEX REFERENCE COLLECTION finished dump, wrote " + idx.size() + " entries to " + statisticPath); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } } - public static int diffurlcol(String metadataPath, String statisticFile, String diffFile) throws IOException { + public static int diffurlcol(String metadataPath, String statisticFile, String diffFile) throws IOException, RowSpaceExceededException { System.out.println("INDEX DIFF URL-COL startup"); HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0); MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false); @@ -438,7 +439,7 @@ public class URLAnalysis { return c; } - public static void export(String metadataPath, int format, String export, String diffFile) throws IOException { + public static void export(String metadataPath, int format, String export, String diffFile) throws IOException, RowSpaceExceededException { // format: 0=text, 1=html, 2=rss/xml System.out.println("URL EXPORT startup"); MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false); @@ -453,7 +454,7 @@ public class URLAnalysis { System.out.println("URL EXPORT finished export, wrote " + ((hs == null) ? mr.size() : hs.size()) + " entries"); } - public static void delete(String metadataPath, String diffFile) throws IOException { + public static void delete(String metadataPath, String diffFile) throws IOException, RowSpaceExceededException { System.out.println("URL DELETE startup"); MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false); int mrSize = mr.size(); @@ -488,7 +489,7 @@ public class URLAnalysis { // java -Xmx1000m -cp classes de.anomic.data.URLAnalysis -diffurlcol DATA/INDEX/freeworld/TEXT/METADATA used.dump diffurlcol.dump try { diffurlcol(args[1], args[2], args[3]); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } } else if (args[0].equals("-export") && args.length >= 4) { @@ -499,7 +500,7 @@ public class URLAnalysis { int format = (args[2].equals("xml")) ? 2 : (args[2].equals("html")) ? 1 : 0; try { export(args[1], format, args[3], (args.length >= 5) ? args[4] : null); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } } else if (args[0].equals("-delete") && args.length >= 3) { @@ -509,7 +510,7 @@ public class URLAnalysis { // instead of 'xml' (which is in fact a rss), the format can also be 'text' and 'html' try { delete(args[1], args[2]); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } } else { diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 94f62830f..e669ddf60 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -133,7 +133,8 @@ public class blogBoard { try { database.put(page.key, page.record); return page.key; - } catch (final IOException e) { + } catch (final Exception e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java index 2ebc03b4d..779dd1f37 100644 --- a/source/de/anomic/data/blogBoardComments.java +++ b/source/de/anomic/data/blogBoardComments.java @@ -111,7 +111,8 @@ public class blogBoardComments { try { database.put(page.key, page.record); return page.key; - } catch (final IOException e) { + } catch (final Exception e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index cec0b2450..309c5ae55 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -400,8 +400,7 @@ public class bookmarksDB { public void saveBookmark(final Bookmark bookmark){ try { bookmarksTable.put(bookmark.getUrlHash(), bookmark.entry); - } catch (final IOException e) { - // TODO Auto-generated catch block + } catch (final Exception e) { Log.logException(e); } } @@ -539,13 +538,19 @@ public class bookmarksDB { */ public void storeTag(final Tag tag){ if (tag == null) return; - try { - if(tag.size() >0){ + if (tag.size() >0) { + try { bookmarksDB.this.tagsTable.put(tag.getTagHash(), tag.getMap()); - }else{ + } catch (Exception e) { + Log.logException(e); + } + } else { + try { bookmarksDB.this.tagsTable.remove(tag.getTagHash()); + } catch (IOException e) { + Log.logException(e); } - } catch (final IOException e) {} + } } /** * save a Tag in tagCache; see also flushTagCache(), addTag(), loadTag() @@ -1040,13 +1045,19 @@ public class bookmarksDB { this.mem.put(URL_HASHES, listManager.collection2string(list)); } public void setDatesTable(){ - try { - if(this.size() >0){ + if (this.size() >0) { + try { bookmarksDB.this.datesTable.put(getDateString(), mem); - }else{ + } catch (Exception e) { + Log.logException(e); + } + } else { + try { bookmarksDB.this.datesTable.remove(getDateString()); + } catch (IOException e) { + Log.logException(e); } - } catch (final IOException e) {} + } } public String getDateString(){ return date; diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 84008770b..a6d7c5a7b 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -33,6 +33,7 @@ import java.util.TimeZone; import net.yacy.kelondro.blob.Heap; import net.yacy.kelondro.blob.MapView; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; @@ -193,7 +194,8 @@ public class messageBoard { try { database.put(message.key, message.record); return message.key; - } catch (final IOException e) { + } catch (final Exception e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 22a8c18b1..f6426beb8 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -37,6 +37,7 @@ import java.util.Random; import net.yacy.kelondro.blob.Heap; import net.yacy.kelondro.blob.MapView; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.CloneableIterator; @@ -113,7 +114,8 @@ public final class userDB { try { userTable.put(entry.userName,entry.mem); return entry.userName; - } catch (final IOException e) { + } catch (final Exception e) { + Log.logException(e); return null; } } @@ -203,16 +205,19 @@ public final class userDB { } return null; } - public Entry passwordAuth(final String user, final String password){ + + public Entry passwordAuth(final String user, final String password) { final Entry entry=this.getEntry(user); - if( entry != null && entry.getMD5EncodedUserPwd().equals(Digest.encodeMD5Hex(user+":"+password)) ){ - if(entry.isLoggedOut()){ - try{ - entry.setProperty(Entry.LOGGED_OUT, "false"); - }catch(final IOException e){} - return null; + if (entry != null && entry.getMD5EncodedUserPwd().equals(Digest.encodeMD5Hex(user+":"+password))) { + if (entry.isLoggedOut()){ + try { + entry.setProperty(Entry.LOGGED_OUT, "false"); + } catch (final Exception e) { + Log.logException(e); } - return entry; + return null; + } + return entry; } return null; } @@ -225,13 +230,15 @@ public final class userDB { this.ipUsers.put(ip, entry.getUserName()); //XXX: This is insecure. TODO: use cookieauth return entry; } - public Entry md5Auth(final String user, final String md5){ + public Entry md5Auth(final String user, final String md5) { final Entry entry=this.getEntry(user); - if( entry != null && entry.getMD5EncodedUserPwd().equals(md5)){ - if(entry.isLoggedOut()){ - try{ + if (entry != null && entry.getMD5EncodedUserPwd().equals(md5)) { + if (entry.isLoggedOut()){ + try { entry.setProperty(Entry.LOGGED_OUT, "false"); - }catch(final IOException e){} + } catch (final Exception e) { + Log.logException(e); + } return null; } return entry; @@ -369,7 +376,9 @@ public final class userDB { } try { this.setProperty(TIME_USED,"0"); - } catch (final IOException e) {} + } catch (final Exception e) { + Log.logException(e); + } return 0; } @@ -383,7 +392,9 @@ public final class userDB { } try { this.setProperty(TIME_LIMIT,"0"); - } catch (final IOException e) {} + } catch (final Exception e) { + Log.logException(e); + } return 0; } @@ -393,7 +404,7 @@ public final class userDB { } try { this.setProperty(TRAFFIC_SIZE,"0"); - } catch (final IOException e) { + } catch (final Exception e) { Log.logException(e); } return 0; @@ -410,7 +421,7 @@ public final class userDB { final long newTrafficSize = currentTrafficSize + responseSize; try { this.setProperty(TRAFFIC_SIZE,Long.toString(newTrafficSize)); - } catch (final IOException e) { + } catch (final Exception e) { Log.logException(e); } return newTrafficSize; @@ -483,7 +494,7 @@ public final class userDB { return this.mem; } - public void setProperty(final String propName, final String newValue) throws IOException { + public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException { this.mem.put(propName, newValue); userDB.this.userTable.put(getUserName(), this.mem); } @@ -552,13 +563,15 @@ public final class userDB { cookieUsers.remove(logintoken); } } - public void logout(final String ip){ - try{ - setProperty(LOGGED_OUT, "true"); - if(ipUsers.containsKey(ip)){ - ipUsers.remove(ip); - } - }catch(final IOException e){} + public void logout(final String ip) { + try { + setProperty(LOGGED_OUT, "true"); + if (ipUsers.containsKey(ip)){ + ipUsers.remove(ip); + } + } catch (final Exception e) { + Log.logException(e); + } } public void logout(){ logout("xxxxxx"); diff --git a/source/de/anomic/data/wiki/wikiBoard.java b/source/de/anomic/data/wiki/wikiBoard.java index 2c17f3ca0..3efb2aa7f 100644 --- a/source/de/anomic/data/wiki/wikiBoard.java +++ b/source/de/anomic/data/wiki/wikiBoard.java @@ -36,6 +36,7 @@ import java.util.TimeZone; import net.yacy.kelondro.blob.Heap; import net.yacy.kelondro.blob.MapView; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; @@ -268,7 +269,8 @@ public class wikiBoard { // write the new page datbase.put(page.key, page.record); return page.key; - } catch (final IOException e) { + } catch (final Exception e) { + Log.logException(e); return null; } } diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index e013ad4c2..9803138c9 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -121,7 +121,7 @@ public final class Cache { hm.put("@@URL", url.toNormalform(true, false)); try { responseHeaderDB.put(url.hash(), hm); - } catch (IOException e) { + } catch (Exception e) { throw new IOException("Cache.store: cannot write to headerDB: " + e.getMessage()); } diff --git a/source/de/anomic/search/MetadataRepository.java b/source/de/anomic/search/MetadataRepository.java index 890dc098c..fb1b153d6 100644 --- a/source/de/anomic/search/MetadataRepository.java +++ b/source/de/anomic/search/MetadataRepository.java @@ -46,6 +46,7 @@ import net.yacy.kelondro.index.Cache; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.table.SplitTable; @@ -76,8 +77,18 @@ public final class MetadataRepository implements Iterable { final String tablename, final boolean useTailCache, final boolean exceed134217727) { - this.location = path; - this.urlIndexFile = new Cache(new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727), 20000000, 20000000); + this.location = path; + ObjectIndex backupIndex = null; + try { + backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727); + } catch (RowSpaceExceededException e) { + try { + backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, false, exceed134217727); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + } + } + this.urlIndexFile = new Cache(backupIndex, 20000000, 20000000); this.exportthread = null; // will have a export thread assigned if exporter is running this.statsDump = null; } @@ -144,7 +155,11 @@ public final class MetadataRepository implements Iterable { return; // this did not need to be stored, but is updated } - urlIndexFile.put(entry.toRowEntry()); + try { + urlIndexFile.put(entry.toRowEntry()); + } catch (RowSpaceExceededException e) { + throw new IOException("RowSpaceExceededException in " + this.urlIndexFile.filename() + ": " + e.getMessage()); + } statsDump = null; } diff --git a/source/de/anomic/search/Segment.java b/source/de/anomic/search/Segment.java index 1f67b5f13..c233a378b 100644 --- a/source/de/anomic/search/Segment.java +++ b/source/de/anomic/search/Segment.java @@ -223,7 +223,7 @@ public class Segment { ientry.setWord(wprop); try { this.termIndex.add(Word.word2hash(word), ientry); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } wordCount++; diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 213d2f3c0..45f579d25 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -691,7 +691,7 @@ public final class Switchboard extends serverSwitch { if (k.startsWith("network.unit.update.location")) d.add(k); if (k.startsWith("network.unit.bootstrap")) d.add(k); } - for (String s:d) this.removeConfig(s); // must be removed afterwards othervise a ki.remove() would not remove the property on file + for (String s:d) this.removeConfig(s); // must be removed afterwards otherwise a ki.remove() would not remove the property on file // include additional network definition properties into our settings // note that these properties cannot be set in the application because they are @@ -1142,7 +1142,7 @@ public final class Switchboard extends serverSwitch { /** * pass a response to the indexer * @param response - * @return null if successful, an error message othervise + * @return null if successful, an error message otherwise */ public String toIndexer(final Response response) { assert response != null; @@ -1380,7 +1380,9 @@ public final class Switchboard extends serverSwitch { crawler.profilesActiveCrawls.changeEntry(selentry, CrawlProfile.entry.RECRAWL_IF_OLDER, Long.toString(crawler.profilesActiveCrawls.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE))); } - } catch (final IOException e) {}; + } catch (final Exception e) { + Log.logException(e); + } // close unused connections Client.cleanup(); @@ -1432,7 +1434,9 @@ public final class Switchboard extends serverSwitch { try { if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + this.peers.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack"); if (this.peers.newsPool.automaticProcess(peers) > 0) hasDoneSomething = true; - } catch (final IOException e) {} + } catch (final Exception e) { + Log.logException(e); + } if (getConfigBool("cleanup.deletionProcessedNews", true)) { this.peers.newsPool.clear(yacyNewsPool.PROCESSED_DB); } @@ -1959,19 +1963,14 @@ public final class Switchboard extends serverSwitch { } log.logInfo("dhtTransferJob: selected " + new String(startHash) + " as start hash"); log.logInfo("dhtTransferJob: selected " + new String(limitHash) + " as limit hash"); - try { - boolean enqueued = this.dhtDispatcher.selectContainersEnqueueToCloud( - startHash, - limitHash, - dhtMaxContainerCount, - dhtMaxReferenceCount, - 5000); - hasDoneSomething = hasDoneSomething | enqueued; - log.logInfo("dhtTransferJob: result from enqueueing: " + ((enqueued) ? "true" : "false")); - } catch (IOException e) { - log.logSevere("dhtTransferJob: interrupted with exception: " + e.getMessage(), e); - return false; - } + boolean enqueued = this.dhtDispatcher.selectContainersEnqueueToCloud( + startHash, + limitHash, + dhtMaxContainerCount, + dhtMaxReferenceCount, + 5000); + hasDoneSomething = hasDoneSomething | enqueued; + log.logInfo("dhtTransferJob: result from enqueueing: " + ((enqueued) ? "true" : "false")); } if (this.dhtDispatcher.transmissionSize() >= 10) { log.logInfo("dhtTransferJob: no dequeueing from cloud to transmission: too many concurrent sessions: " + this.dhtDispatcher.transmissionSize()); diff --git a/source/de/anomic/search/blockrank/CRProcess.java b/source/de/anomic/search/blockrank/CRProcess.java index 44d5e4117..0b4930b09 100644 --- a/source/de/anomic/search/blockrank/CRProcess.java +++ b/source/de/anomic/search/blockrank/CRProcess.java @@ -34,6 +34,7 @@ import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Bitfield; @@ -145,7 +146,7 @@ public class CRProcess { return true; } - public static boolean accumulate_upd(final File f, final ObjectIndex acc) throws IOException { + public static boolean accumulate_upd(final File f, final ObjectIndex acc) throws IOException, RowSpaceExceededException { // open file AttrSeq source_cr = null; try { @@ -231,7 +232,7 @@ public class CRProcess { final File bkp_dir, final File to_file, int max_files, - final boolean newdb) throws IOException { + final boolean newdb) throws IOException, RowSpaceExceededException { if (!(from_dir.isDirectory())) { System.out.println("source path " + from_dir + " is not a directory."); return; @@ -390,7 +391,7 @@ public class CRProcess { return count; } - public static int genrcix(final File cr_path_in, final File rci_path_out) throws IOException { + public static int genrcix(final File cr_path_in, final File rci_path_out) throws IOException, RowSpaceExceededException { //kelondroFlexTable acc = new kelondroFlexTable(cr_path_in, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true); final IndexCell seq = new IndexCell( cr_path_in, "index", Segment.wordReferenceFactory, Base64Order.enhancedCoder, CRG_colrow, 10000, 1000000000L, 20, null, 1000000); @@ -530,7 +531,7 @@ public class CRProcess { } } */ - } catch (final IOException e) { + } catch (final Exception e) { Log.logException(e); } } diff --git a/source/de/anomic/yacy/dht/Dispatcher.java b/source/de/anomic/yacy/dht/Dispatcher.java index c8add1a26..2493aac5c 100755 --- a/source/de/anomic/yacy/dht/Dispatcher.java +++ b/source/de/anomic/yacy/dht/Dispatcher.java @@ -34,6 +34,7 @@ import java.util.Map; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReferenceRow; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.rwi.ReferenceContainer; @@ -225,9 +226,10 @@ public class Dispatcher { * @param containers * @param scheme * @return + * @throws RowSpaceExceededException */ @SuppressWarnings("unchecked") - private ArrayList>[] splitContainers(ArrayList> containers) { + private ArrayList>[] splitContainers(ArrayList> containers) throws RowSpaceExceededException { // init the result vector int partitionCount = this.seeds.scheme.verticalPartitions(); @@ -302,7 +304,12 @@ public class Dispatcher { // fill the entry with the containers for (ReferenceContainer c: containers[vertical]) { - entry.add(c); + try { + entry.add(c); + } catch (RowSpaceExceededException e) { + Log.logException(e); + break; + } } // put the entry into the cloud @@ -315,31 +322,43 @@ public class Dispatcher { final byte[] limitHash, final int maxContainerCount, final int maxReferenceCount, - final int maxtime) throws IOException { + final int maxtime) { if (this.transmissionCloud == null) return false; - ArrayList> selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime); - this.log.logInfo("selectContainersToCache: selectedContainerCache was filled with " + selectedContainerCache.size() + " entries"); + ArrayList> selectedContainerCache; + try { + selectedContainerCache = selectContainers(hash, limitHash, maxContainerCount, maxReferenceCount, maxtime); + } catch (IOException e) { + this.log.logSevere("selectContainersEnqueueToCloud: selectedContainer failed", e); + return false; + } + this.log.logInfo("selectContainersEnqueueToCloud: selectedContainerCache was filled with " + selectedContainerCache.size() + " entries"); if (selectedContainerCache == null || selectedContainerCache.isEmpty()) { - this.log.logInfo("splitContainersFromCache: selectedContainerCache is empty, cannot do anything here."); + this.log.logInfo("selectContainersEnqueueToCloud: selectedContainerCache is empty, cannot do anything here."); return false; } - ArrayList>[] splittedContainerCache = splitContainers(selectedContainerCache); + ArrayList>[] splittedContainerCache; + try { + splittedContainerCache = splitContainers(selectedContainerCache); + } catch (RowSpaceExceededException e) { + this.log.logSevere("selectContainersEnqueueToCloud: splitContainers failed because of too low RAM", e); + return false; + } selectedContainerCache = null; if (splittedContainerCache == null) { - this.log.logInfo("enqueueContainersFromCache: splittedContainerCache is empty, cannot do anything here."); + this.log.logInfo("selectContainersEnqueueToCloud: splittedContainerCache is empty, cannot do anything here."); return false; } this.log.logInfo("splitContainersFromCache: splittedContainerCache filled with " + splittedContainerCache.length + " partitions, deleting selectedContainerCache"); if (splittedContainerCache.length != this.seeds.scheme.verticalPartitions()) { - this.log.logWarning("enqueueContainersFromCache: splittedContainerCache has wrong length."); + this.log.logWarning("selectContainersEnqueueToCloud: splittedContainerCache has wrong length."); return false; } enqueueContainersToCloud(splittedContainerCache); splittedContainerCache = null; - this.log.logInfo("enqueueContainersFromCache: splittedContainerCache enqueued to cloud array which has now " + this.transmissionCloud.size() + " entries."); + this.log.logInfo("selectContainersEnqueueToCloud: splittedContainerCache enqueued to cloud array which has now " + this.transmissionCloud.size() + " entries."); return true; } @@ -401,8 +420,13 @@ public class Dispatcher { // removes all entries from the dispatcher and puts them back to a RAMRI if (indexingTransmissionProcessor != null) this.indexingTransmissionProcessor.announceShutdown(); if (this.transmissionCloud != null) { - for (Map.Entry e : this.transmissionCloud.entrySet()) { - for (ReferenceContainer i : e.getValue()) try {this.segment.termIndex().add(i);} catch (IOException e1) {} + outerLoop: for (Map.Entry e : this.transmissionCloud.entrySet()) { + for (ReferenceContainer i : e.getValue()) try { + this.segment.termIndex().add(i); + } catch (Exception e1) { + Log.logException(e1); + break outerLoop; + } } this.transmissionCloud.clear(); } diff --git a/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java b/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java index 55adf8ba1..9552801ef 100755 --- a/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java +++ b/source/de/anomic/yacy/dht/FlatWordPartitionScheme.java @@ -29,6 +29,8 @@ import java.util.Random; import java.util.TreeMap; import net.yacy.kelondro.index.HandleMap; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.util.MemoryControl; import de.anomic.yacy.yacySeed; @@ -111,7 +113,12 @@ public class FlatWordPartitionScheme implements PartitionScheme { long a = MemoryControl.available(); HandleMap idx = new HandleMap(12, Base64Order.enhancedCoder, 4, 0, 150000); for (int i = 0; i < count; i++) { - idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count))); + try { + idx.inc(FlatWordPartitionScheme.positionToHash(r.nextInt(count))); + } catch (RowSpaceExceededException e) { + Log.logException(e); + break; + } } long timek = ((long) count) * 1000L / (System.currentTimeMillis() - start); System.out.println("Result HandleMap: " + timek + " inc per second"); diff --git a/source/de/anomic/yacy/dht/Transmission.java b/source/de/anomic/yacy/dht/Transmission.java index 20a099969..13af4fd5b 100644 --- a/source/de/anomic/yacy/dht/Transmission.java +++ b/source/de/anomic/yacy/dht/Transmission.java @@ -24,7 +24,6 @@ package de.anomic.yacy.dht; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -33,6 +32,7 @@ import java.util.Iterator; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainerCache; @@ -115,8 +115,9 @@ public class Transmission { * add a container to the Entry cache. * all entries in the container are checked and only such are stored which have a reference entry * @param container + * @throws RowSpaceExceededException */ - public void add(ReferenceContainer container) { + public void add(ReferenceContainer container) throws RowSpaceExceededException { // iterate through the entries in the container and check if the reference is in the repository Iterator i = container.entries(); ArrayList notFound = new ArrayList(); @@ -247,7 +248,13 @@ public class Transmission { } public void restore() { - for (ReferenceContainer ic : this) try { segment.termIndex().add(ic); } catch (IOException e) {} + for (ReferenceContainer ic : this) try { + segment.termIndex().add(ic); + } catch (Exception e) { + Log.logException(e); + } } + } + } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index a5b57d760..299a96ee8 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -60,6 +60,7 @@ import net.yacy.document.parser.xml.RSSReader; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Bitfield; @@ -594,7 +595,12 @@ public final class yacyClient { // add the url entry to the word indexes for (int m = 0; m < words; m++) { - container[m].add(entry); + try { + container[m].add(entry); + } catch (RowSpaceExceededException e) { + Log.logException(e); + break; + } } // store url hash for statistics @@ -647,7 +653,7 @@ public final class yacyClient { // insert the containers to the index for (int m = 0; m < words; m++) try { indexSegment.termIndex().add(container[m]); - } catch (IOException e) { + } catch (Exception e) { Log.logException(e); } diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index c90239353..aa007f444 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -38,7 +38,6 @@ package de.anomic.yacy; -import java.io.IOException; import java.net.MalformedURLException; import java.util.Collections; import java.util.Date; @@ -353,7 +352,7 @@ public class yacyCore { } else { sb.peers.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString())); } - } catch (final IOException e) { + } catch (final Exception e) { log.logSevere("publishMySeed: problem with news encoding", e); } sb.peers.mySeed().setUnusedFlags(); diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 5e2776333..1aa026a51 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -51,6 +51,8 @@ import java.util.Iterator; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.table.Table; import net.yacy.kelondro.util.DateFormatter; @@ -69,14 +71,29 @@ public class yacyNewsDB { final boolean useTailCache, final boolean exceed134217727) { this.path = path; - this.news = new Table(path, yacyNewsRecord.rowdef, 10, 0, useTailCache, exceed134217727); - //this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef)); + try { + this.news = new Table(path, yacyNewsRecord.rowdef, 10, 0, useTailCache, exceed134217727); + } catch (RowSpaceExceededException e) { + try { + this.news = new Table(path, yacyNewsRecord.rowdef, 0, 0, false, exceed134217727); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + } + } } private void resetDB() { try {close();} catch (final Exception e) {} if (path.exists()) FileUtils.deletedelete(path); - this.news = new Table(path, yacyNewsRecord.rowdef, 10, 0, false, false); + try { + this.news = new Table(path, yacyNewsRecord.rowdef, 10, 0, false, false); + } catch (RowSpaceExceededException e) { + try { + this.news = new Table(path, yacyNewsRecord.rowdef, 0, 0, false, false); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + } + } } public void close() { @@ -96,10 +113,10 @@ public class yacyNewsDB { news.remove(id.getBytes()); } - public synchronized yacyNewsRecord put(final yacyNewsRecord record) throws IOException { + public synchronized yacyNewsRecord put(final yacyNewsRecord record) throws IOException, RowSpaceExceededException { try { return b2r(news.replace(r2b(record))); - } catch (final kelondroException e) { + } catch (final Exception e) { resetDB(); return b2r(news.replace(r2b(record))); } diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 9e4e1b762..665149019 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -51,6 +51,8 @@ import java.util.Iterator; import java.util.Map; import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; import net.yacy.repository.Blacklist; import de.anomic.search.Switchboard; @@ -303,10 +305,12 @@ public class yacyNewsPool { incomingNews.push(record); // we want to see our own news.. outgoingNews.push(record); // .. and put it on the publishing list } - } catch (final IOException e) {} + } catch (final Exception e) { + Log.logException(e); + } } - public yacyNewsRecord myPublication() throws IOException { + public yacyNewsRecord myPublication() throws IOException, RowSpaceExceededException { // generate a record for next peer-ping if (outgoingNews.isEmpty()) return null; final yacyNewsRecord record = outgoingNews.topInc(); @@ -317,7 +321,7 @@ public class yacyNewsPool { return record; } - public void enqueueIncomingNews(final yacyNewsRecord record) throws IOException { + public void enqueueIncomingNews(final yacyNewsRecord record) throws IOException, RowSpaceExceededException { // called if a news is attached to a seed // check consistency @@ -352,7 +356,7 @@ public class yacyNewsPool { return switchQueue(dbKey).size(); } - public int automaticProcess(final yacySeedDB seedDB) throws IOException, InterruptedException { + public int automaticProcess(final yacySeedDB seedDB) throws IOException, InterruptedException, RowSpaceExceededException { // processes news in the incoming-db // returns number of processes yacyNewsRecord record; @@ -468,7 +472,7 @@ public class yacyNewsPool { } } - public void moveOff(final int dbKey, final String id) throws IOException { + public void moveOff(final int dbKey, final String id) throws IOException, RowSpaceExceededException { // this is called if a queue element shall be moved to another queue or off the queue // it depends on the dbKey how the record is handled switch (dbKey) { @@ -479,7 +483,7 @@ public class yacyNewsPool { } } - private boolean moveOff(final yacyNewsQueue fromqueue, final yacyNewsQueue toqueue, final String id) throws IOException { + private boolean moveOff(final yacyNewsQueue fromqueue, final yacyNewsQueue toqueue, final String id) throws IOException, RowSpaceExceededException { // called if a published news shall be removed final yacyNewsRecord record = fromqueue.remove(id); if (record == null) { @@ -493,7 +497,7 @@ public class yacyNewsPool { return true; } - public void moveOffAll(final int dbKey) throws IOException { + public void moveOffAll(final int dbKey) throws IOException, RowSpaceExceededException { // this is called if a queue element shall be moved to another queue or off the queue // it depends on the dbKey how the record is handled switch (dbKey) { @@ -504,7 +508,7 @@ public class yacyNewsPool { } } - private int moveOffAll(final yacyNewsQueue fromqueue, final yacyNewsQueue toqueue) throws IOException { + private int moveOffAll(final yacyNewsQueue fromqueue, final yacyNewsQueue toqueue) throws IOException, RowSpaceExceededException { // move off all news from a specific queue to another queue final Iterator i = fromqueue.records(true); yacyNewsRecord record; diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 810aa718b..79acc6514 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -52,6 +52,7 @@ import java.util.Iterator; import net.yacy.kelondro.index.Column; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.table.RecordStack; import net.yacy.kelondro.util.DateFormatter; @@ -104,7 +105,7 @@ public class yacyNewsQueue { return queueStack.isEmpty(); } - public synchronized void push(final yacyNewsRecord entry) throws IOException { + public synchronized void push(final yacyNewsRecord entry) throws IOException, RowSpaceExceededException { queueStack.push(r2b(entry, true)); } @@ -118,7 +119,7 @@ public class yacyNewsQueue { return b2r(queueStack.top()); } - public synchronized yacyNewsRecord topInc() throws IOException { + public synchronized yacyNewsRecord topInc() throws IOException, RowSpaceExceededException { if (queueStack.isEmpty()) return null; final yacyNewsRecord entry = pop(); if (entry != null) { @@ -158,7 +159,7 @@ public class yacyNewsQueue { return newsDB.get(id); } - private Row.Entry r2b(final yacyNewsRecord r, final boolean updateDB) throws IOException { + private Row.Entry r2b(final yacyNewsRecord r, final boolean updateDB) throws IOException, RowSpaceExceededException { if (r == null) return null; if (updateDB) { newsDB.put(r); diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 1a995f05e..018359b1d 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -24,7 +24,6 @@ package de.anomic.yacy; -import java.io.IOException; import java.util.HashMap; import net.yacy.document.content.RSSMessage; @@ -239,7 +238,7 @@ public class yacyPeerActions { } try { synchronized (this.newsPool) {this.newsPool.enqueueIncomingNews(record);} - } catch (final IOException e) { + } catch (final Exception e) { Log.logSevere("YACY", "processPeerArrival", e); } } diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 1db61ee9f..db3e2bf0a 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -396,13 +396,7 @@ public final class yacySeedDB implements AlternativeDomainNames { } seedPassiveDB.remove(seed.hash); seedPotentialDB.remove(seed.hash); - } catch (final IOException e) { - yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - resetActiveTable(); - } catch (final kelondroException e) { - yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - resetActiveTable(); - } catch (final IllegalArgumentException e) { + } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); resetActiveTable(); } @@ -421,13 +415,7 @@ public final class yacySeedDB implements AlternativeDomainNames { synchronized (seedPropMap) { seedPassiveDB.put(seed.hash, seedPropMap); } - } catch (final IOException e) { - yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - resetPassiveTable(); - } catch (final kelondroException e) { - yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - resetPassiveTable(); - } catch (final IllegalArgumentException e) { + } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); resetPassiveTable(); } @@ -446,13 +434,7 @@ public final class yacySeedDB implements AlternativeDomainNames { synchronized (seedPropMap) { seedPotentialDB.put(seed.hash, seedPropMap); } - } catch (final IOException e) { - yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - resetPotentialTable(); - } catch (final kelondroException e) { - yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); - resetPotentialTable(); - } catch (final IllegalArgumentException e) { + } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); resetPotentialTable(); } @@ -536,13 +518,13 @@ public final class yacySeedDB implements AlternativeDomainNames { } yacySeed s = get(hash, seedActiveDB); - if (s != null) try { seedActiveDB.put(hash, seed.getMap()); return;} catch (final IOException e) {} + if (s != null) try { seedActiveDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} s = get(hash, seedPassiveDB); - if (s != null) try { seedPassiveDB.put(hash, seed.getMap()); return;} catch (final IOException e) {} + if (s != null) try { seedPassiveDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} s = get(hash, seedPotentialDB); - if (s != null) try { seedPotentialDB.put(hash, seed.getMap()); return;} catch (final IOException e) {} + if (s != null) try { seedPotentialDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} } public yacySeed lookupByName(String peerName) { diff --git a/source/net/yacy/dbtest.java b/source/net/yacy/dbtest.java index 342710ea9..6c0778399 100644 --- a/source/net/yacy/dbtest.java +++ b/source/net/yacy/dbtest.java @@ -17,6 +17,7 @@ import net.yacy.kelondro.index.ObjectArrayCache; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.CloneableIterator; @@ -132,6 +133,10 @@ public class dbtest { System.err.println(e); Log.logException(e); System.exit(0); + } catch (RowSpaceExceededException e) { + System.err.println(e); + Log.logException(e); + System.exit(0); } } } diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 18b69d99b..3563acb57 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -48,6 +48,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; @@ -679,8 +680,9 @@ public class ArrayStack implements BLOB { * @param key the primary key * @param b * @throws IOException + * @throws RowSpaceExceededException */ - public synchronized void put(byte[] key, byte[] b) throws IOException { + public synchronized void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException { blobItem bi = (blobs.isEmpty()) ? null : blobs.get(blobs.size() - 1); if (bi == null) System.out.println("bi == null"); @@ -729,23 +731,43 @@ public class ArrayStack implements BLOB { blobs = null; } - public File mergeMount(File f1, File f2, ReferenceFactory factory, Row payloadrow, File newFile, int writeBuffer) throws IOException { + public File mergeMount(File f1, File f2, + ReferenceFactory factory, + Row payloadrow, File newFile, int writeBuffer) { Log.logInfo("BLOBArray", "merging " + f1.getName() + " with " + f2.getName()); File resultFile = mergeWorker(factory, this.keylength, this.ordering, f1, f2, payloadrow, newFile, writeBuffer); if (resultFile == null) { Log.logWarning("BLOBArray", "merge of files " + f1 + ", " + f2 + " returned null. newFile = " + newFile); return null; } - mountBLOB(resultFile, false); + try { + mountBLOB(resultFile, false); + } catch (IOException e) { + Log.logWarning("BLOBArray", "merge of files " + f1 + ", " + f2 + " successfull, but read failed. resultFile = " + resultFile); + return null; + } Log.logInfo("BLOBArray", "merged " + f1.getName() + " with " + f2.getName() + " into " + resultFile); return resultFile; } - private static File mergeWorker(ReferenceFactory factory, int keylength, ByteOrder order, File f1, File f2, Row payloadrow, File newFile, int writeBuffer) throws IOException { + private static File mergeWorker( + ReferenceFactory factory, + int keylength, ByteOrder order, File f1, File f2, Row payloadrow, File newFile, int writeBuffer) { // iterate both files and write a new one - CloneableIterator> i1 = new ReferenceIterator(f1, factory, payloadrow); - CloneableIterator> i2 = new ReferenceIterator(f2, factory, payloadrow); + CloneableIterator> i1 = null, i2 = null; + try { + i1 = new ReferenceIterator(f1, factory, payloadrow); + } catch (IOException e) { + Log.logSevere("ArrayStack", "cannot merge because input files cannot be read, f1 = " + f1.toString() + ": " + e.getMessage(), e); + return null; + } + try { + i2 = new ReferenceIterator(f2, factory, payloadrow); + } catch (IOException e) { + Log.logSevere("ArrayStack", "cannot merge because input files cannot be read, f2 = " + f2.toString() + ": " + e.getMessage(), e); + return null; + } if (!i1.hasNext()) { if (i2.hasNext()) { FileUtils.deletedelete(f1); @@ -763,23 +785,31 @@ public class ArrayStack implements BLOB { assert i1.hasNext(); assert i2.hasNext(); File tmpFile = new File(newFile.getParentFile(), newFile.getName() + ".prt"); - HeapWriter writer = new HeapWriter(tmpFile, newFile, keylength, order, writeBuffer); - merge(i1, i2, order, writer); try { + HeapWriter writer = new HeapWriter(tmpFile, newFile, keylength, order, writeBuffer); + merge(i1, i2, order, writer); writer.close(true); - // we don't need the old files any more - FileUtils.deletedelete(f1); - FileUtils.deletedelete(f2); - return newFile; } catch (IOException e) { - Log.logSevere("ArrayStack", "cannot close writing: " + e.getMessage(), e); + Log.logSevere("ArrayStack", "cannot writing or close writing merge, newFile = " + newFile.toString() + ", tmpFile = " + tmpFile.toString() + ": " + e.getMessage(), e); + FileUtils.deletedelete(tmpFile); + FileUtils.deletedelete(newFile); + return null; + } catch (RowSpaceExceededException e) { + Log.logSevere("ArrayStack", "cannot merge because of memory failure: " + e.getMessage(), e); FileUtils.deletedelete(tmpFile); FileUtils.deletedelete(newFile); return null; } + // we don't need the old files any more + FileUtils.deletedelete(f1); + FileUtils.deletedelete(f2); + return newFile; } - private static void merge(CloneableIterator> i1, CloneableIterator> i2, ByteOrder ordering, HeapWriter writer) throws IOException { + private static void merge( + CloneableIterator> i1, + CloneableIterator> i2, + ByteOrder ordering, HeapWriter writer) throws IOException, RowSpaceExceededException { assert i1.hasNext(); assert i2.hasNext(); ReferenceContainer c1, c2, c1o, c2o; @@ -871,6 +901,8 @@ public class ArrayStack implements BLOB { heap.close(true); } catch (final IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } diff --git a/source/net/yacy/kelondro/blob/BLOB.java b/source/net/yacy/kelondro/blob/BLOB.java index 5b37f0c1c..6b7b3e6b1 100644 --- a/source/net/yacy/kelondro/blob/BLOB.java +++ b/source/net/yacy/kelondro/blob/BLOB.java @@ -28,6 +28,7 @@ package net.yacy.kelondro.blob; import java.io.IOException; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; @@ -125,8 +126,9 @@ public interface BLOB { * @param key the primary key * @param b * @throws IOException + * @throws RowSpaceExceededException */ - public void put(byte[] key, byte[] b) throws IOException; + public void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException; /** * replace an existing entry in the BLOB with a new entry diff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.java index 33e823eda..0ba6a8d9b 100644 --- a/source/net/yacy/kelondro/blob/Compressor.java +++ b/source/net/yacy/kelondro/blob/Compressor.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; @@ -169,9 +170,14 @@ public class Compressor implements BLOB { if (b != null) { // compress the entry now and put it to the backend byte[] bb = compress(b); - this.backend.put(key, bb); - this.bufferlength = this.bufferlength - b.length; - return b; + try { + this.backend.put(key, bb); + this.bufferlength = this.bufferlength - b.length; + return b; + } catch (RowSpaceExceededException e) { + buffer.put(new String(key), b); + return b; + } } // return from the backend @@ -222,7 +228,12 @@ public class Compressor implements BLOB { if (this.bufferlength + b.length * 2 > this.maxbufferlength) { // in case that we compress, just compress as much as is necessary to get enough room while (this.bufferlength + b.length * 2 > this.maxbufferlength && !this.buffer.isEmpty()) { - flushOne(); + try { + flushOne(); + } catch (RowSpaceExceededException e) { + Log.logException(e); + break; + } } // in case that this was not enough, just flush all if (this.bufferlength + b.length * 2 > this.maxbufferlength) flushAll(); @@ -261,21 +272,31 @@ public class Compressor implements BLOB { return this.backend.keys(up, firstKey); } - private boolean flushOne() throws IOException { + private boolean flushOne() throws IOException, RowSpaceExceededException { if (this.buffer.isEmpty()) return false; // depending on process case, write it to the file or compress it to the other queue Map.Entry entry = this.buffer.entrySet().iterator().next(); this.buffer.remove(entry.getKey()); byte[] b = entry.getValue(); this.bufferlength -= b.length; - b = compress(b); - this.backend.put(entry.getKey().getBytes(), b); + byte[] bb = compress(b); + try { + this.backend.put(entry.getKey().getBytes(), bb); + } catch (RowSpaceExceededException e) { + this.buffer.put(entry.getKey(), b); + throw e; + } return true; } private void flushAll() throws IOException { while (!this.buffer.isEmpty()) { - if (!flushOne()) break; + try { + if (!flushOne()) break; + } catch (RowSpaceExceededException e) { + Log.logException(e); + break; + } } assert this.bufferlength == 0; } diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 712c9735e..ac323900c 100755 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.io.AbstractWriter; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; @@ -130,26 +131,28 @@ public final class Heap extends HeapModifier implements BLOB { * @param key * @param blob * @throws IOException + * @throws RowSpaceExceededException */ - private void add(final byte[] key, final byte[] blob) throws IOException { + private void add(final byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException { assert blob.length > 0; assert key.length == this.keylength; assert this.keylength == key.length : this.keylength + "!=" + key.length; if ((blob == null) || (blob.length == 0)) return; final int pos = (int) file.length(); + index.put(key, pos); file.seek(pos); file.writeInt(key.length + blob.length); file.write(key); file.write(blob, 0, blob.length); - index.put(key, pos); } /** * flush the buffer completely * this is like adding all elements of the buffer, but it needs only one IO access * @throws IOException + * @throws RowSpaceExceededException */ - private void flushBuffer() throws IOException { + private void flushBuffer() throws IOException, RowSpaceExceededException { // check size of buffer Iterator> i = this.buffer.entrySet().iterator(); int l = 0; @@ -254,6 +257,8 @@ public final class Heap extends HeapModifier implements BLOB { flushBuffer(); } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } this.buffer = null; @@ -274,8 +279,9 @@ public final class Heap extends HeapModifier implements BLOB { * @param key the primary key * @param b * @throws IOException + * @throws RowSpaceExceededException */ - public synchronized void put(final byte[] key, final byte[] b) throws IOException { + public synchronized void put(final byte[] key, final byte[] b) throws IOException, RowSpaceExceededException { assert this.keylength == key.length : this.keylength + "!=" + key.length; // we do not write records of length 0 into the BLOB @@ -307,7 +313,7 @@ public final class Heap extends HeapModifier implements BLOB { this.buffersize += b.length; } - private boolean putToGap(final byte[] key, final byte[] b) throws IOException { + private boolean putToGap(final byte[] key, final byte[] b) throws IOException, RowSpaceExceededException { assert this.keylength == key.length : this.keylength + "!=" + key.length; // we do not write records of length 0 into the BLOB @@ -327,15 +333,17 @@ public final class Heap extends HeapModifier implements BLOB { if (entry.getValue().intValue() == reclen) { // we found an entry that has exactly the size that we need! // we use that entry and stop looking for a larger entry + + // add the entry to the index + this.index.put(key, entry.getKey()); + + // write to file file.seek(entry.getKey().longValue()); final int reclenf = file.readInt(); assert reclenf == reclen; file.write(key); file.write(b); - - // add the entry to the index - this.index.put(key, entry.getKey()); - + // remove the entry from the free list i.remove(); @@ -413,7 +421,11 @@ public final class Heap extends HeapModifier implements BLOB { * @throws IOException */ public synchronized CloneableIterator keys(final boolean up, final boolean rotating) throws IOException { - this.flushBuffer(); + try { + this.flushBuffer(); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } return super.keys(up, rotating); } @@ -425,7 +437,11 @@ public final class Heap extends HeapModifier implements BLOB { * @throws IOException */ public synchronized CloneableIterator keys(final boolean up, final byte[] firstKey) throws IOException { - this.flushBuffer(); + try { + this.flushBuffer(); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } return super.keys(up, firstKey); } @@ -457,6 +473,8 @@ public final class Heap extends HeapModifier implements BLOB { heap.close(true); } catch (final IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } @@ -481,6 +499,8 @@ public final class Heap extends HeapModifier implements BLOB { heap.close(); } catch (final IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java index 03c0e2344..6d2d658ad 100644 --- a/source/net/yacy/kelondro/blob/HeapModifier.java +++ b/source/net/yacy/kelondro/blob/HeapModifier.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; import java.util.SortedMap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.io.CachedFileWriter; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; @@ -225,7 +226,7 @@ public class HeapModifier extends HeapReader implements BLOB { } } - public void put(byte[] key, byte[] b) throws IOException { + public void put(byte[] key, byte[] b) throws IOException, RowSpaceExceededException { throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier"); } diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 7f9d53cb8..998498488 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -35,6 +35,7 @@ import java.util.Map.Entry; import java.util.concurrent.ExecutionException; import net.yacy.kelondro.index.HandleMap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.io.CachedFileWriter; import net.yacy.kelondro.io.Writer; import net.yacy.kelondro.logging.Log; @@ -134,6 +135,9 @@ public class HeapReader { } catch (IOException e) { Log.logException(e); return false; + } catch (RowSpaceExceededException e) { + Log.logException(e); + return false; } // check saturation diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java index bd2e72185..e17b74934 100644 --- a/source/net/yacy/kelondro/blob/HeapWriter.java +++ b/source/net/yacy/kelondro/blob/HeapWriter.java @@ -31,6 +31,7 @@ import java.io.FileOutputStream; import java.io.IOException; import net.yacy.kelondro.index.HandleMap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.Digest; @@ -89,19 +90,21 @@ public final class HeapWriter { * @param key * @param blob * @throws IOException + * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ - public synchronized void add(final byte[] key, final byte[] blob) throws IOException { + public synchronized void add(final byte[] key, final byte[] blob) throws IOException, RowSpaceExceededException { //System.out.println("HeapWriter.add: " + new String(key)); assert blob.length > 0; assert key.length == this.keylength; assert index.row().primaryKeyLength == key.length : index.row().primaryKeyLength + "!=" + key.length; assert index.get(key) < 0 : "index.get(key) = " + index.get(key) + ", index.size() = " + index.size() + ", file.length() = " + this.heapFileTMP.length() + ", key = " + new String(key); // must not occur before if ((blob == null) || (blob.length == 0)) return; + index.putUnique(key, seek); int chunkl = key.length + blob.length; os.writeInt(chunkl); os.write(key); os.write(blob); - index.putUnique(key, seek); //assert (this.doublecheck.add(new String(key))) : "doublecheck failed for " + new String(key); this.seek += chunkl + 4; } diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 5648562b1..64b0ed0f7 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.util.ScoreCluster; @@ -169,11 +170,13 @@ public class MapDataMining extends MapView { } } - public synchronized void put(final String key, final Map newMap) throws IOException { + public synchronized void put(final String key, final Map newMap) throws IOException, RowSpaceExceededException { assert (key != null); assert (key.length() > 0); assert (newMap != null); - + + super.put(key, newMap); + // update elementCount if ((longaccfields != null) || (doubleaccfields != null)) { final Map oldMap = super.get(key, false); @@ -183,8 +186,6 @@ public class MapDataMining extends MapView { } } - super.put(key, newMap); - // update sortCluster if (sortClusterMap != null) updateSortCluster(key, newMap); diff --git a/source/net/yacy/kelondro/blob/MapView.java b/source/net/yacy/kelondro/blob/MapView.java index 14347760a..d37db3f37 100644 --- a/source/net/yacy/kelondro/blob/MapView.java +++ b/source/net/yacy/kelondro/blob/MapView.java @@ -38,6 +38,7 @@ import java.util.Map; import net.yacy.kelondro.index.ARC; import net.yacy.kelondro.index.ConcurrentARC; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; @@ -136,8 +137,9 @@ public class MapView { * @param key the primary key * @param newMap * @throws IOException + * @throws RowSpaceExceededException */ - public void put(String key, final Map newMap) throws IOException { + public void put(String key, final Map newMap) throws IOException, RowSpaceExceededException { assert key != null; assert key.length() > 0; assert newMap != null; @@ -373,6 +375,8 @@ public class MapView { map.close(); } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } diff --git a/source/net/yacy/kelondro/blob/Stack.java b/source/net/yacy/kelondro/blob/Stack.java index 0db39f9c5..542a08dcf 100644 --- a/source/net/yacy/kelondro/blob/Stack.java +++ b/source/net/yacy/kelondro/blob/Stack.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; import java.util.Iterator; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.order.NaturalOrder; @@ -88,8 +89,9 @@ public class Stack { * @param b the new stack element * @return the handle used to store the new element * @throws IOException + * @throws RowSpaceExceededException */ - public synchronized long push(byte[] b) throws IOException { + public synchronized long push(byte[] b) throws IOException, RowSpaceExceededException { long handle = nextHandle(); this.stack.put(NaturalOrder.encodeLong(handle, 8), b); return handle; @@ -102,8 +104,9 @@ public class Stack { * @param b the new stack element * @return the handle used to store the new element * @throws IOException + * @throws RowSpaceExceededException */ - protected synchronized void push(Entry e) throws IOException { + protected synchronized void push(Entry e) throws IOException, RowSpaceExceededException { this.stack.put(NaturalOrder.encodeLong(e.h, 8), e.b); } diff --git a/source/net/yacy/kelondro/blob/Stacks.java b/source/net/yacy/kelondro/blob/Stacks.java index ca8a0a425..e8dc9fdd3 100644 --- a/source/net/yacy/kelondro/blob/Stacks.java +++ b/source/net/yacy/kelondro/blob/Stacks.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; public class Stacks { @@ -160,8 +161,9 @@ public class Stacks { * @param b the new stack element * @return the handle used to store the new element * @throws IOException + * @throws RowSpaceExceededException */ - public long push(String stack, byte[] b) throws IOException { + public long push(String stack, byte[] b) throws IOException, RowSpaceExceededException { Stack s = getStack(stack); if (s == null) return -1; return s.push(b); @@ -175,8 +177,9 @@ public class Stacks { * @param b the new stack element * @return the handle used to store the new element * @throws IOException + * @throws RowSpaceExceededException */ - protected void push(String stack, Stack.Entry e) throws IOException { + protected void push(String stack, Stack.Entry e) throws IOException, RowSpaceExceededException { Stack s = getStack(stack); if (s == null) return; s.push(e); diff --git a/source/net/yacy/kelondro/index/Cache.java b/source/net/yacy/kelondro/index/Cache.java index e2fccb917..483400eeb 100644 --- a/source/net/yacy/kelondro/index/Cache.java +++ b/source/net/yacy/kelondro/index/Cache.java @@ -265,21 +265,25 @@ public final class Cache implements ObjectIndex, Iterable { entry = index.get(key); // learn from result if (entry == null) { - if (checkMissSpace()) { + if (checkMissSpace()) try { final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } return null; } - if (checkHitSpace()) { + if (checkHitSpace()) try { final Row.Entry dummy = readHitCache.replace(entry); if (dummy == null) this.writeUnique++; else this.writeDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } return entry; } - public final synchronized void put(final Row.Entry row) throws IOException { + public final synchronized void put(final Row.Entry row) throws IOException, RowSpaceExceededException { assert (row != null); assert (row.columns() == row().columns()); //assert (!(serverLog.allZero(row.getColBytes(index.primarykey())))); @@ -291,25 +295,26 @@ public final class Cache implements ObjectIndex, Iterable { if (readMissCache != null) { if (readMissCache.remove(key) != null) { this.hasnotHit++; - // the entry did not exist before - index.put(row); // write to back-end - if (checkHitSpace()) { - final Row.Entry dummy = readHitCache.replace(row); // learn that entry - if (dummy == null) this.writeUnique++; else this.writeDouble++; - } - return; } } // write to the back-end - index.put(row); - if (checkHitSpace()) { + try { + index.put(row); + } catch (RowSpaceExceededException e1) { + // flush all caches to get more memory + clearCache(); + index.put(row); // try again + } + if (checkHitSpace()) try { final Row.Entry dummy = readHitCache.replace(row); // overwrite old entry if (dummy == null) this.writeUnique++; else this.writeDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } } - public final synchronized Row.Entry replace(final Row.Entry row) throws IOException { + public final synchronized Row.Entry replace(final Row.Entry row) throws IOException, RowSpaceExceededException { assert (row != null); assert (row.columns() == row().columns()); //assert (!(serverLog.allZero(row.getColBytes(index.primarykey())))); @@ -322,27 +327,43 @@ public final class Cache implements ObjectIndex, Iterable { if (readMissCache.remove(key) != null) { this.hasnotHit++; // the entry does not exist before - index.put(row); // write to backend - if (checkHitSpace()) { + try { + index.put(row); + } catch (RowSpaceExceededException e1) { + // flush all caches to get more memory + clearCache(); + index.put(row); // try again + } + // write to backend + if (checkHitSpace()) try { final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } return null; } } - Row.Entry entry; - + Row.Entry entry = null; // write to the back-end - entry = index.replace(row); - if (checkHitSpace()) { + try { + entry = index.replace(row); + } catch (RowSpaceExceededException e1) { + // flush all caches to get more memory + clearCache(); + index.replace(row); // try again + } + if (checkHitSpace()) try { final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } return entry; } - public final synchronized void addUnique(final Row.Entry row) throws IOException { + public final synchronized void addUnique(final Row.Entry row) throws IOException, RowSpaceExceededException { assert (row != null); assert (row.columns() == row().columns()); //assert (!(serverLog.allZero(row.getColBytes(index.primarykey())))); @@ -355,23 +376,25 @@ public final class Cache implements ObjectIndex, Iterable { this.readMissCache.remove(key); this.hasnotDelete++; // the entry does not exist before - index.addUnique(row); // write to backend - if (checkHitSpace()) { - final Row.Entry dummy = readHitCache.replace(row); // learn that entry - if (dummy == null) this.writeUnique++; else this.writeDouble++; - } - return; } // the worst case: we must write to the back-end directly - index.addUnique(row); - if (checkHitSpace()) { + try { + index.addUnique(row); + } catch (RowSpaceExceededException e1) { + // flush all caches to get more memory + clearCache(); + index.addUnique(row); // try again + } + if (checkHitSpace()) try { final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } } - public final synchronized void addUnique(final Row.Entry row, final Date entryDate) throws IOException { + public final synchronized void addUnique(final Row.Entry row, final Date entryDate) throws IOException, RowSpaceExceededException { if (entryDate == null) { addUnique(row); return; @@ -390,19 +413,37 @@ public final class Cache implements ObjectIndex, Iterable { } // the worst case: we must write to the backend directly - index.addUnique(row); - if (checkHitSpace()) { + try { + index.addUnique(row); + } catch (RowSpaceExceededException e1) { + // flush all caches to get more memory + clearCache(); + index.addUnique(row); // try again + } + if (checkHitSpace()) try { final Row.Entry dummy = readHitCache.replace(row); // learn that entry if (dummy == null) this.writeUnique++; else this.writeDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } } - public final synchronized void addUnique(final List rows) throws IOException { + public final synchronized void addUnique(final List rows) throws IOException, RowSpaceExceededException { final Iterator i = rows.iterator(); - while (i.hasNext()) addUnique(i.next()); + Row.Entry r; + while (i.hasNext()) { + r = i.next(); + try { + addUnique(r); + } catch (RowSpaceExceededException e) { + // flush all caches to get more memory + clearCache(); + addUnique(r); // try again + } + } } - public final synchronized ArrayList removeDoubles() throws IOException { + public final synchronized ArrayList removeDoubles() throws IOException, RowSpaceExceededException { return index.removeDoubles(); // todo: remove reported entries from the cache!!! } @@ -411,7 +452,7 @@ public final class Cache implements ObjectIndex, Iterable { checkMissSpace(); // add entry to miss-cache - if (checkMissSpace()) { + if (checkMissSpace()) try { // set the miss cache; if there was already an entry we know that the return value must be null final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); if (dummy == null) { @@ -420,6 +461,8 @@ public final class Cache implements ObjectIndex, Iterable { this.hasnotHit++; this.hasnotDouble++; } + } catch (RowSpaceExceededException e) { + clearCache(); } // remove entry from hit-cache @@ -443,9 +486,11 @@ public final class Cache implements ObjectIndex, Iterable { final Row.Entry entry = index.removeOne(); if (entry == null) return null; final byte[] key = entry.getPrimaryKeyBytes(); - if (checkMissSpace()) { + if (checkMissSpace()) try { final Row.Entry dummy = readMissCache.replace(readMissCache.row().newEntry(key)); if (dummy == null) this.hasnotUnique++; else this.hasnotDouble++; + } catch (RowSpaceExceededException e) { + clearCache(); } if (readHitCache != null) { final Row.Entry dummy = readHitCache.remove(key); diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index a42743ad3..d9e256fe9 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -73,8 +73,9 @@ public final class HandleMap implements Iterable { * @param objectOrder * @param file * @throws IOException + * @throws RowSpaceExceededException */ - public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final File file, final int expectedspace) throws IOException { + public HandleMap(final int keylength, final ByteOrder objectOrder, int idxbytes, final File file, final int expectedspace) throws IOException, RowSpaceExceededException { this(keylength, objectOrder, idxbytes, (int) (file.length() / (keylength + idxbytes)), expectedspace); // read the index dump and fill the index InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); @@ -180,7 +181,7 @@ public final class HandleMap implements Iterable { return indexentry.getColLong(1); } - public final synchronized long put(final byte[] key, final long l) { + public final synchronized long put(final byte[] key, final long l) throws RowSpaceExceededException { assert l >= 0 : "l = " + l; assert (key != null); final Row.Entry newentry = index.row().newEntry(); @@ -191,7 +192,7 @@ public final class HandleMap implements Iterable { return oldentry.getColLong(1); } - public final synchronized void putUnique(final byte[] key, final long l) { + public final synchronized void putUnique(final byte[] key, final long l) throws RowSpaceExceededException { assert l >= 0 : "l = " + l; assert (key != null); final Row.Entry newentry = this.rowdef.newEntry(); @@ -200,7 +201,7 @@ public final class HandleMap implements Iterable { index.addUnique(newentry); } - public final synchronized long add(final byte[] key, long a) { + public final synchronized long add(final byte[] key, long a) throws RowSpaceExceededException { assert key != null; assert a > 0; // it does not make sense to add 0. If this occurres, it is a performance issue @@ -218,15 +219,15 @@ public final class HandleMap implements Iterable { return i; } - public final synchronized long inc(final byte[] key) { + public final synchronized long inc(final byte[] key) throws RowSpaceExceededException { return add(key, 1); } - public final synchronized long dec(final byte[] key) { + public final synchronized long dec(final byte[] key) throws RowSpaceExceededException { return add(key, -1); } - public final synchronized ArrayList removeDoubles() { + public final synchronized ArrayList removeDoubles() throws RowSpaceExceededException { final ArrayList report = new ArrayList(); Long[] is; int c; @@ -371,6 +372,8 @@ public final class HandleMap implements Iterable { } } catch (InterruptedException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } if (sortAtEnd) { map.index.finishInitialization(); diff --git a/source/net/yacy/kelondro/index/HandleSet.java b/source/net/yacy/kelondro/index/HandleSet.java index fa7b3f846..35afa69da 100644 --- a/source/net/yacy/kelondro/index/HandleSet.java +++ b/source/net/yacy/kelondro/index/HandleSet.java @@ -55,8 +55,9 @@ public final class HandleSet implements Iterable { * @param objectOrder * @param file * @throws IOException + * @throws RowSpaceExceededException */ - public HandleSet(final int keylength, final ByteOrder objectOrder, final File file, final int expectedspace) throws IOException { + public HandleSet(final int keylength, final ByteOrder objectOrder, final File file, final int expectedspace) throws IOException, RowSpaceExceededException { this(keylength, objectOrder, (int) (file.length() / (keylength + 8)), expectedspace); // read the index dump and fill the index InputStream is = new BufferedInputStream(new FileInputStream(file), 1024 * 1024); @@ -107,7 +108,7 @@ public final class HandleSet implements Iterable { return index.has(key); } - public final synchronized int put(final byte[] key) throws IOException { + public final synchronized int put(final byte[] key) throws IOException, RowSpaceExceededException { assert (key != null); final Row.Entry newentry = index.row().newEntry(); newentry.setCol(0, key); @@ -116,7 +117,7 @@ public final class HandleSet implements Iterable { return (int) oldentry.getColLong(1); } - public final synchronized void putUnique(final byte[] key) throws IOException { + public final synchronized void putUnique(final byte[] key) throws IOException, RowSpaceExceededException { assert (key != null); final Row.Entry newentry = this.rowdef.newEntry(); newentry.setCol(0, key); diff --git a/source/net/yacy/kelondro/index/IndexTest.java b/source/net/yacy/kelondro/index/IndexTest.java index 8890510cc..e0c65aba7 100644 --- a/source/net/yacy/kelondro/index/IndexTest.java +++ b/source/net/yacy/kelondro/index/IndexTest.java @@ -113,7 +113,12 @@ public class IndexTest { Runtime.getRuntime().gc(); long freeStartKelondro = MemoryControl.available(); HandleMap ii = new HandleMap(12, Base64Order.enhancedCoder, 4, count, count); - for (int i = 0; i < count; i++) ii.putUnique(tests[i], 1); + for (int i = 0; i < count; i++) + try { + ii.putUnique(tests[i], 1); + } catch (RowSpaceExceededException e) { + e.printStackTrace(); + } ii.get(randomHash(r)); // trigger sort long t6 = System.currentTimeMillis(); System.out.println("time for HandleMap generation: " + (t6 - t5)); diff --git a/source/net/yacy/kelondro/index/ObjectArrayCache.java b/source/net/yacy/kelondro/index/ObjectArrayCache.java index 30447ed0b..b65f38c75 100644 --- a/source/net/yacy/kelondro/index/ObjectArrayCache.java +++ b/source/net/yacy/kelondro/index/ObjectArrayCache.java @@ -34,8 +34,8 @@ import net.yacy.kelondro.order.NaturalOrder; public final class ObjectArrayCache { - // we use two indexes: one for initialization, and one for data aquired during runtime - // this has a gread advantage, if the setup-data is large. Then a re-organisation of + // we use two indexes: one for initialization, and one for data acquired during runtime + // this has a gread advantage, if the setup-data is large. Then a re-organization of // the run-time data does not need much memory and is done faster. // we distinguish two phases: the init phase where data can only be written // to index0 with addb, and a runtime-phase where data can only be written @@ -82,7 +82,7 @@ public final class ObjectArrayCache { return indexentry.getColBytes(1); } - public final byte[] putb(final int ii, final byte[] value) { + public final byte[] putb(final int ii, final byte[] value) throws RowSpaceExceededException { assert ii >= 0 : "i = " + ii; assert value != null; final byte[] key = NaturalOrder.encodeLong(ii, 4); @@ -114,7 +114,7 @@ public final class ObjectArrayCache { return oldentry.getColBytes(1); } - public final void addb(final int ii, final byte[] value) { + public final void addb(final int ii, final byte[] value) throws RowSpaceExceededException { assert index1 == null; // valid only in init-phase assert ii >= 0 : "i = " + ii; assert value != null; @@ -224,7 +224,11 @@ public final class ObjectArrayCache { r = Math.abs(random.nextLong() % 10000); //System.out.println("add " + r); jcontrol.add(Long.valueOf(r)); - kcontrol.putb((int) r, "x".getBytes()); + try { + kcontrol.putb((int) r, "x".getBytes()); + } catch (RowSpaceExceededException e) { + e.printStackTrace(); + } if (random.nextLong() % 5 == 0) ra.add(Long.valueOf(r)); if (!ra.isEmpty() && random.nextLong() % 7 == 0) { rc++; diff --git a/source/net/yacy/kelondro/index/ObjectIndex.java b/source/net/yacy/kelondro/index/ObjectIndex.java index d2bc5c685..857ff41e4 100644 --- a/source/net/yacy/kelondro/index/ObjectIndex.java +++ b/source/net/yacy/kelondro/index/ObjectIndex.java @@ -46,10 +46,10 @@ public interface ObjectIndex { public Row row(); public boolean has(byte[] key); // use this only if there is no get in case that has returns true public Row.Entry get(byte[] key) throws IOException; - public Row.Entry replace(Row.Entry row) throws IOException; - public void put(Row.Entry row) throws IOException; - public void addUnique(Row.Entry row) throws IOException; // no double-check - public ArrayList removeDoubles() throws IOException; // removes all elements that are double (to be used after all addUnique) + public Row.Entry replace(Row.Entry row) throws IOException, RowSpaceExceededException; + public void put(Row.Entry row) throws IOException, RowSpaceExceededException; + public void addUnique(Row.Entry row) throws IOException, RowSpaceExceededException; // no double-check + public ArrayList removeDoubles() throws IOException, RowSpaceExceededException; // removes all elements that are double (to be used after all addUnique) public Row.Entry remove(byte[] key) throws IOException; public Row.Entry removeOne() throws IOException; public CloneableIterator keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key diff --git a/source/net/yacy/kelondro/index/ObjectIndexCache.java b/source/net/yacy/kelondro/index/ObjectIndexCache.java index 3392c46d5..578919488 100644 --- a/source/net/yacy/kelondro/index/ObjectIndexCache.java +++ b/source/net/yacy/kelondro/index/ObjectIndexCache.java @@ -108,7 +108,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable return index1.has(key); } - public final synchronized Row.Entry replace(final Row.Entry entry) { + public final synchronized Row.Entry replace(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); finishInitialization(); // if the new entry is within the initialization part, just overwrite it @@ -122,7 +122,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable return index1.replace(entry); } - public final synchronized void put(final Row.Entry entry) { + public final synchronized void put(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); if (entry == null) return; finishInitialization(); @@ -137,7 +137,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable index1.put(entry); } - public final synchronized void addUnique(final Row.Entry entry) { + public final synchronized void addUnique(final Row.Entry entry) throws RowSpaceExceededException { assert (entry != null); if (entry == null) return; if (index1 == null) { @@ -149,12 +149,12 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable index1.addUnique(entry); } - public final void addUnique(final List rows) { + public final void addUnique(final List rows) throws RowSpaceExceededException { final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); } - public final synchronized long inc(final byte[] key, int col, long add, Row.Entry initrow) { + public final synchronized long inc(final byte[] key, int col, long add, Row.Entry initrow) throws RowSpaceExceededException { assert (key != null); finishInitialization(); assert index0.isSorted(); @@ -163,7 +163,7 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable return index1.inc(key, col, add, initrow); } - public final synchronized ArrayList removeDoubles() { + public final synchronized ArrayList removeDoubles() throws RowSpaceExceededException { // finish initialization phase explicitely index0.sort(); if (index1 == null) { diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 7b03cf022..bc5faeb8a 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -217,13 +217,18 @@ public class RowCollection implements Iterable { return needed; } - protected final void ensureSize(final int elements) { + protected final void ensureSize(final int elements) throws RowSpaceExceededException { long allocram = neededSpaceForEnsuredSize(elements, true); if (allocram == 0) return; assert allocram > chunkcache.length : "wrong alloc computation: allocram = " + allocram + ", chunkcache.length = " + chunkcache.length; - byte[] newChunkcache = new byte[(int) allocram]; // increase space - System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); - chunkcache = newChunkcache; + if (!MemoryControl.request(allocram, true)) throw new RowSpaceExceededException(allocram, "RowCollection grow"); + try { + byte[] newChunkcache = new byte[(int) allocram]; // increase space + System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); + chunkcache = newChunkcache; + } catch (OutOfMemoryError e) { + throw new RowSpaceExceededException(allocram, "RowCollection grow after OutOfMemoryError " + e.getMessage()); + } } /** @@ -285,7 +290,7 @@ public class RowCollection implements Iterable { return entry; } - public synchronized final void set(final int index, final Row.Entry a) { + public synchronized final void set(final int index, final Row.Entry a) throws RowSpaceExceededException { assert (index >= 0) : "set: access with index " + index + " is below zero"; ensureSize(index + 1); boolean sameKey = match(a.bytes(), 0, a.cellwidth(0), index); @@ -296,7 +301,7 @@ public class RowCollection implements Iterable { this.lastTimeWrote = System.currentTimeMillis(); } - public final void insertUnique(final int index, final Row.Entry a) { + public final void insertUnique(final int index, final Row.Entry a) throws RowSpaceExceededException { assert (a != null); if (index < chunkcount) { @@ -309,23 +314,23 @@ public class RowCollection implements Iterable { set(index, a); } - public synchronized void addUnique(final Row.Entry row) { + public synchronized void addUnique(final Row.Entry row) throws RowSpaceExceededException { final byte[] r = row.bytes(); addUnique(r, 0, r.length); } - public synchronized void addUnique(final List rows) { + public synchronized void addUnique(final List rows) throws RowSpaceExceededException { assert this.sortBound == 0 : "sortBound = " + this.sortBound + ", chunkcount = " + this.chunkcount; final Iterator i = rows.iterator(); while (i.hasNext()) addUnique(i.next()); } - public synchronized void add(final byte[] a) { + public synchronized void add(final byte[] a) throws RowSpaceExceededException { assert a.length == this.rowdef.objectsize : "a.length = " + a.length + ", objectsize = " + this.rowdef.objectsize; addUnique(a, 0, a.length); } - private final void addUnique(final byte[] a, final int astart, final int alength) { + private final void addUnique(final byte[] a, final int astart, final int alength) throws RowSpaceExceededException { assert (a != null); assert (astart >= 0) && (astart < a.length) : " astart = " + astart; assert (!(Log.allZero(a, astart, alength))) : "a = " + NaturalOrder.arrayList(a, astart, alength); @@ -349,7 +354,7 @@ public class RowCollection implements Iterable { this.lastTimeWrote = System.currentTimeMillis(); } - protected final void addSorted(final byte[] a, final int astart, final int alength) { + protected final void addSorted(final byte[] a, final int astart, final int alength) throws RowSpaceExceededException { assert (a != null); assert (astart >= 0) && (astart < a.length) : " astart = " + astart; assert (!(Log.allZero(a, astart, alength))) : "a = " + NaturalOrder.arrayList(a, astart, alength); @@ -364,7 +369,7 @@ public class RowCollection implements Iterable { this.lastTimeWrote = System.currentTimeMillis(); } - public synchronized final void addAllUnique(final RowCollection c) { + public synchronized final void addAllUnique(final RowCollection c) throws RowSpaceExceededException { if (c == null) return; assert(rowdef.objectsize == c.rowdef.objectsize); ensureSize(chunkcount + c.size()); @@ -838,7 +843,7 @@ public class RowCollection implements Iterable { } } - public synchronized ArrayList removeDoubles() { + public synchronized ArrayList removeDoubles() throws RowSpaceExceededException { assert (this.rowdef.objectOrder != null); // removes double-occurrences of chunks // in contrast to uniq() this removes also the remaining, non-double entry that had a double-occurrence to the others @@ -969,7 +974,7 @@ public class RowCollection implements Iterable { Base64Order.enhancedCoder.encodeLong(random.nextLong(), 4); } - public static void test(final int testsize) { + public static void test(final int testsize) throws RowSpaceExceededException { final Row r = new Row(new Column[]{ new Column("hash", Column.celltype_string, Column.encoder_bytes, 12, "hash")}, Base64Order.enhancedCoder); @@ -1075,7 +1080,11 @@ public class RowCollection implements Iterable { public static void main(final String[] args) { //test(1000); - test(50000); + try { + test(50000); + } catch (RowSpaceExceededException e) { + e.printStackTrace(); + } //test(100000); //test(1000000); diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index ff15706eb..b18b9feab 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -93,7 +93,7 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable= 0) { // the entry existed before @@ -371,8 +371,9 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable= c.size()) { @@ -399,8 +400,9 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable ii = d.iterator(); @@ -584,7 +596,11 @@ public class RowSet extends RowCollection implements ObjectIndex, Iterable { return r; } - public final void addUnique(Entry row) { + public final void addUnique(Entry row) throws RowSpaceExceededException { int i = indexFor(row); if (i < 0) return; accessArray(i).addUnique(row); } - public final void addUnique(List rows) { + public final void addUnique(List rows) throws RowSpaceExceededException { for (Entry row: rows) addUnique(row); } @@ -125,7 +125,7 @@ public final class RowSetArray implements ObjectIndex, Iterable { } } - public final void put(Entry row) { + public final void put(Entry row) throws RowSpaceExceededException { int i = indexFor(row); if (i < 0) return; accessArray(i).put(row); @@ -137,7 +137,7 @@ public final class RowSetArray implements ObjectIndex, Iterable { return accessArray(i).remove(key); } - public final ArrayList removeDoubles() { + public final ArrayList removeDoubles() throws RowSpaceExceededException { ArrayList col = new ArrayList(); synchronized (this.array) { for (int i = 0; i < this.array.length; i++) { @@ -163,7 +163,7 @@ public final class RowSetArray implements ObjectIndex, Iterable { return null; } - public final Entry replace(Entry row) { + public final Entry replace(Entry row) throws RowSpaceExceededException { int i = indexFor(row); if (i < 0) return null; return accessArray(i).replace(row); @@ -217,7 +217,7 @@ public final class RowSetArray implements ObjectIndex, Iterable { return this.rows(true, null); } - public final long inc(byte[] key, int col, long add, Entry initrow) { + public final long inc(byte[] key, int col, long add, Entry initrow) throws RowSpaceExceededException { int i = indexFor(key); if (i < 0) return -1; return accessArray(i).inc(key, col, add, initrow); diff --git a/source/net/yacy/kelondro/index/RowSpaceExceededException.java b/source/net/yacy/kelondro/index/RowSpaceExceededException.java new file mode 100644 index 000000000..f9002af49 --- /dev/null +++ b/source/net/yacy/kelondro/index/RowSpaceExceededException.java @@ -0,0 +1,64 @@ +// RowSpaceExceededException +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 06.12.2009 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.kelondro.index; + +import java.util.Date; + +import net.yacy.kelondro.util.MemoryControl; + +public class RowSpaceExceededException extends Exception { + + private static final long serialVersionUID = 9059516027929222151L; + + private String forUsage; + private long neededRAM, availableRAM, time; + + public RowSpaceExceededException(long neededRAM, String forUsage) { + super(Long.toString(neededRAM) + " bytes needed for " + forUsage + ": " + MemoryControl.available() + " free at " + (new Date()).toString()); + this.time = System.currentTimeMillis(); + this.availableRAM = MemoryControl.available(); + this.neededRAM = neededRAM; + this.forUsage = forUsage; + } + + public String getUsage() { + return forUsage; + } + + public long getNeededRAM() { + return neededRAM; + } + + public long getAvailableRAM() { + return availableRAM; + } + + public long getTime() { + return time; + } + +} diff --git a/source/net/yacy/kelondro/rwi/AbstractIndex.java b/source/net/yacy/kelondro/rwi/AbstractIndex.java index 428ade6a1..f9baa27c7 100644 --- a/source/net/yacy/kelondro/rwi/AbstractIndex.java +++ b/source/net/yacy/kelondro/rwi/AbstractIndex.java @@ -33,6 +33,7 @@ import java.util.Iterator; import java.util.Set; import java.util.TreeSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Order; @@ -133,8 +134,9 @@ public abstract class AbstractIndex implements * @param urlselection * @param maxDistance the maximum distance that the words in the result may have * @return ReferenceContainer the join result + * @throws RowSpaceExceededException */ - public ReferenceContainer searchJoin(final TreeSet wordHashes, final Set urlselection, int maxDistance) { + public ReferenceContainer searchJoin(final TreeSet wordHashes, final Set urlselection, int maxDistance) throws RowSpaceExceededException { // first check if there is any entry that has no match; // this uses only operations in ram for (byte[] wordHash: wordHashes) { @@ -170,7 +172,7 @@ public abstract class AbstractIndex implements final TreeSet excludeHashes, final Set urlselection, ReferenceFactory termFactory, - int maxDistance) { + int maxDistance) throws RowSpaceExceededException { return new TermSearch(this, queryHashes, excludeHashes, urlselection, termFactory, maxDistance); } diff --git a/source/net/yacy/kelondro/rwi/IODispatcher.java b/source/net/yacy/kelondro/rwi/IODispatcher.java index c6a3bc621..891968698 100644 --- a/source/net/yacy/kelondro/rwi/IODispatcher.java +++ b/source/net/yacy/kelondro/rwi/IODispatcher.java @@ -109,12 +109,8 @@ public class IODispatcher extends Thread { public synchronized void merge(File f1, File f2, ReferenceFactory factory, ArrayStack array, Row payloadrow, File newFile) { if (mergeQueue == null || controlQueue == null || !this.isAlive()) { - try { - Log.logWarning("IODispatcher", "emergency merge of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); - array.mergeMount(f1, f2, factory, payloadrow, newFile, (int) Math.min(MemoryControl.available() / 3, writeBufferSize)); - } catch (IOException e) { - Log.logSevere("IODispatcher", "emergency merge failed: " + e.getMessage(), e); - } + Log.logWarning("IODispatcher", "emergency merge of files " + f1.getName() + ", " + f2.getName() + " to " + newFile.getName()); + array.mergeMount(f1, f2, factory, payloadrow, newFile, (int) Math.min(MemoryControl.available() / 3, writeBufferSize)); } else { MergeJob job = new MergeJob(f1, f2, factory, array, payloadrow, newFile); try { @@ -128,11 +124,7 @@ public class IODispatcher extends Thread { } } catch (InterruptedException e) { Log.logWarning("IODispatcher", "interrupted: " + e.getMessage(), e); - try { - array.mergeMount(f1, f2, factory, payloadrow, newFile, (int) Math.min(MemoryControl.available() / 3, writeBufferSize)); - } catch (IOException ee) { - Log.logSevere("IODispatcher", "IO failed: " + e.getMessage(), ee); - } + array.mergeMount(f1, f2, factory, payloadrow, newFile, (int) Math.min(MemoryControl.available() / 3, writeBufferSize)); } } } @@ -255,12 +247,7 @@ public class IODispatcher extends Thread { Log.logWarning("IODispatcher", "merge of file (2) " + f2.getName() + " failed: file does not exists"); return null; } - try { - return array.mergeMount(f1, f2, factory, payloadrow, newFile, (int) Math.min(MemoryControl.available() / 3, writeBufferSize)); - } catch (IOException e) { - Log.logSevere("IODispatcher", "mergeMount failed: " + e.getMessage(), e); - } - return null; + return array.mergeMount(f1, f2, factory, payloadrow, newFile, (int) Math.min(MemoryControl.available() / 3, writeBufferSize)); } } diff --git a/source/net/yacy/kelondro/rwi/Index.java b/source/net/yacy/kelondro/rwi/Index.java index 096d45e90..72f10ac04 100644 --- a/source/net/yacy/kelondro/rwi/Index.java +++ b/source/net/yacy/kelondro/rwi/Index.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.Set; import java.util.TreeSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; @@ -46,8 +47,9 @@ public interface Index { * reference to be stored, then the old and the new references are merged * @param newEntries the References to be merged with existing references * @throws IOException + * @throws RowSpaceExceededException */ - public void add(ReferenceContainer newEntries) throws IOException; + public void add(ReferenceContainer newEntries) throws IOException, RowSpaceExceededException; /** * add a single reference to the reverse index @@ -57,8 +59,9 @@ public interface Index { * @param termHash * @param entry * @throws IOException + * @throws RowSpaceExceededException */ - public void add(final byte[] termHash, final ReferenceType entry) throws IOException; + public void add(final byte[] termHash, final ReferenceType entry) throws IOException, RowSpaceExceededException; /** * check if there are references stored to the given word hash diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index 21e24332e..ddb320e24 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -35,7 +35,9 @@ import de.anomic.yacy.graphics.ProfilingGraph; import net.yacy.kelondro.index.ARC; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.SimpleARC; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.MergeIterator; @@ -105,21 +107,35 @@ public final class IndexCell extends AbstractBu /** * add entries to the cell: this adds the new entries always to the RAM part, never to BLOBs * @throws IOException - * @throws IOException + * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ - public void add(ReferenceContainer newEntries) throws IOException { - this.ram.add(newEntries); - if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { + public void add(ReferenceContainer newEntries) throws IOException, RowSpaceExceededException { + try { + this.ram.add(newEntries); + if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { + EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime); + cleanCache(); + } + } catch (RowSpaceExceededException e) { EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime); cleanCache(); + this.ram.add(newEntries); } + } - public void add(byte[] termHash, ReferenceType entry) throws IOException { - this.ram.add(termHash, entry); - if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { + public void add(byte[] termHash, ReferenceType entry) throws IOException, RowSpaceExceededException { + try { + this.ram.add(termHash, entry); + if (this.ram.size() % 1000 == 0 || this.lastCleanup + cleanupCycle < System.currentTimeMillis()) { + EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime); + cleanCache(); + } + } catch (RowSpaceExceededException e) { EventTracker.update("wordcache", Long.valueOf(this.ram.size()), true, 30000, ProfilingGraph.maxTime); cleanCache(); + this.ram.add(termHash, entry); } } @@ -150,7 +166,8 @@ public final class IndexCell extends AbstractBu ReferenceContainer c1; try { c1 = this.array.get(termHash); - } catch (IOException e) { + } catch (Exception e) { + Log.logException(e); c1 = null; } countFile = (c1 == null) ? 0 : c1.size(); @@ -174,13 +191,29 @@ public final class IndexCell extends AbstractBu */ public ReferenceContainer get(byte[] termHash, Set urlselection) throws IOException { ReferenceContainer c0 = this.ram.get(termHash, null); - ReferenceContainer c1 = this.array.get(termHash); + ReferenceContainer c1 = null; + try { + c1 = this.array.get(termHash); + } catch (RowSpaceExceededException e2) { + Log.logException(e2); + } if (c1 == null) { if (c0 == null) return null; return c0; } if (c0 == null) return c1; - return c1.merge(c0); + try { + return c1.merge(c0); + } catch (RowSpaceExceededException e) { + // try to free some ram + countCache.clear(); + try { + return c1.merge(c0); + } catch (RowSpaceExceededException e1) { + // go silently over the problem + return (c1.size() > c0.size()) ? c1: c0; + } + } } /** @@ -189,7 +222,12 @@ public final class IndexCell extends AbstractBu * @throws IOException */ public ReferenceContainer delete(byte[] termHash) throws IOException { - ReferenceContainer c1 = this.array.get(termHash); + ReferenceContainer c1 = null; + try { + c1 = this.array.get(termHash); + } catch (RowSpaceExceededException e2) { + Log.logException(e2); + } if (c1 != null) { this.array.delete(termHash); this.countCache.remove(new ByteArray(termHash)); @@ -198,7 +236,18 @@ public final class IndexCell extends AbstractBu cleanCache(); if (c1 == null) return c0; if (c0 == null) return c1; - return c1.merge(c0); + try { + return c1.merge(c0); + } catch (RowSpaceExceededException e) { + // try to free some ram + countCache.clear(); + try { + return c1.merge(c0); + } catch (RowSpaceExceededException e1) { + // go silently over the problem + return (c1.size() > c0.size()) ? c1: c0; + } + } } /** diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainer.java b/source/net/yacy/kelondro/rwi/ReferenceContainer.java index 5c6b05c8d..0ce7da89d 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainer.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainer.java @@ -36,6 +36,7 @@ import java.util.TreeMap; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.ByteOrder; @@ -69,7 +70,7 @@ public class ReferenceContainer extends RowSet this.lastTimeWrote = 0; } - public ReferenceContainer topLevelClone() { + public ReferenceContainer topLevelClone() throws RowSpaceExceededException { final ReferenceContainer newContainer = new ReferenceContainer(this.factory, this.termHash, this.size()); newContainer.addAllUnique(this); return newContainer; @@ -97,29 +98,29 @@ public class ReferenceContainer extends RowSet return new String(termHash); } - public void add(final Reference entry) { + public void add(final Reference entry) throws RowSpaceExceededException { // add without double-occurrence test assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize; this.addUnique(entry.toKelondroEntry()); } - public ReferenceContainer merge(final ReferenceContainer c) { + public ReferenceContainer merge(final ReferenceContainer c) throws RowSpaceExceededException { return new ReferenceContainer(this.factory, this.termHash, super.merge(c)); } - public Reference replace(final Reference entry) { + public Reference replace(final Reference entry) throws RowSpaceExceededException { assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize; final Row.Entry r = super.replace(entry.toKelondroEntry()); if (r == null) return null; return factory.produceSlow(r); } - public void put(final Reference entry) { + public void put(final Reference entry) throws RowSpaceExceededException { assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize; super.put(entry.toKelondroEntry()); } - public boolean putRecent(final Reference entry) { + public boolean putRecent(final Reference entry) throws RowSpaceExceededException { assert entry.toKelondroEntry().objectsize() == super.rowdef.objectsize; // returns true if the new entry was added, false if it already existed final Row.Entry oldEntryRow = this.replace(entry.toKelondroEntry()); @@ -134,7 +135,7 @@ public class ReferenceContainer extends RowSet return true; } - public int putAllRecent(final ReferenceContainer c) { + public int putAllRecent(final ReferenceContainer c) throws RowSpaceExceededException { // adds all entries in c and checks every entry for double-occurrence // returns the number of new elements if (c == null) return 0; @@ -206,7 +207,7 @@ public class ReferenceContainer extends RowSet } - public static Object mergeUnique(final Object a, final Object b) { + public static Object mergeUnique(final Object a, final Object b) throws RowSpaceExceededException { final ReferenceContainer c = (ReferenceContainer) a; c.addAllUnique((ReferenceContainer) b); return c; @@ -233,7 +234,7 @@ public class ReferenceContainer extends RowSet final ReferenceFactory factory, final Collection> includeContainers, final Collection> excludeContainers, - final int maxDistance) { + final int maxDistance) throws RowSpaceExceededException { // join a search result and return the joincount (number of pages after join) // since this is a conjunction we return an empty entity if any word is not known @@ -250,7 +251,7 @@ public class ReferenceContainer extends RowSet public static ReferenceContainer joinContainers( final ReferenceFactory factory, final Collection> containers, - final int maxDistance) { + final int maxDistance) throws RowSpaceExceededException { // order entities by their size final TreeMap> map = new TreeMap>(); @@ -320,7 +321,7 @@ public class ReferenceContainer extends RowSet final ReferenceFactory factory, final ReferenceContainer i1, final ReferenceContainer i2, - final int maxDistance) { + final int maxDistance) throws RowSpaceExceededException { if ((i1 == null) || (i2 == null)) return null; if (i1.isEmpty() || i2.isEmpty()) return null; @@ -344,7 +345,7 @@ public class ReferenceContainer extends RowSet final ReferenceFactory factory, final ReferenceContainer small, final ReferenceContainer large, - final int maxDistance) { + final int maxDistance) throws RowSpaceExceededException { //System.out.println("DEBUG: JOIN METHOD BY TEST, maxdistance = " + maxDistance); assert small.rowdef.equals(large.rowdef) : "small = " + small.rowdef.toString() + "; large = " + large.rowdef.toString(); final int keylength = small.rowdef.width(0); @@ -373,7 +374,7 @@ public class ReferenceContainer extends RowSet final ReferenceFactory factory, final ReferenceContainer i1, final ReferenceContainer i2, - final int maxDistance) { + final int maxDistance) throws RowSpaceExceededException { //System.out.println("DEBUG: JOIN METHOD BY ENUMERATION, maxdistance = " + maxDistance); assert i1.rowdef.equals(i2.rowdef) : "i1 = " + i1.rowdef.toString() + "; i2 = " + i2.rowdef.toString(); final int keylength = i1.rowdef.width(0); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index d9ad04787..503766696 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -34,6 +34,7 @@ import net.yacy.kelondro.blob.BLOB; import net.yacy.kelondro.index.HandleMap; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; @@ -154,20 +155,23 @@ public final class ReferenceContainerArray { } public ReferenceContainer next() { - try { - if (iterator.hasNext()) { - return get(iterator.next()); - } - // rotation iteration - if (!rot) { - return null; - } - iterator = array.keys(true, null); - return get(iterator.next()); - } catch (IOException e) { + if (iterator.hasNext()) try { + return get(iterator.next()); + } catch (Exception e) { Log.logException(e); - return null; - } + return null; + } + // rotation iteration + if (!rot) { + return null; + } + try { + iterator = array.keys(true, null); + return get(iterator.next()); + } catch (Exception e) { + Log.logException(e); + return null; + } } public void remove() { @@ -184,7 +188,7 @@ public final class ReferenceContainerArray { * test if a given key is in the heap * this works with heaps in write- and read-mode * @param key - * @return true, if the key is used in the heap; false othervise + * @return true, if the key is used in the heap; false otherwise * @throws IOException */ public synchronized boolean has(final byte[] termHash) { @@ -196,8 +200,9 @@ public final class ReferenceContainerArray { * @param key * @return the indexContainer if one exist, null otherwise * @throws IOException + * @throws RowSpaceExceededException */ - public ReferenceContainer get(final byte[] termHash) throws IOException { + public ReferenceContainer get(final byte[] termHash) throws IOException, RowSpaceExceededException { long timeout = System.currentTimeMillis() + 3000; Iterator entries = this.array.getAll(termHash).iterator(); if (entries == null || !entries.hasNext()) return null; @@ -312,7 +317,7 @@ public final class ReferenceContainerArray { final File heapLocation, final ReferenceFactory factory, final ByteOrder termOrder, - final Row payloadrow) throws IOException { + final Row payloadrow) throws IOException, RowSpaceExceededException { System.out.println("CELL REFERENCE COLLECTION startup"); HandleMap references = new HandleMap(payloadrow.primaryKeyLength, termOrder, 4, 0, 1000000); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index 788fc0b90..95819ce1e 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.kelondro.blob.HeapWriter; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.ByteOrder; @@ -114,6 +115,8 @@ public final class ReferenceContainerCache exte dump.add(wordHash, container.exportCollection()); } catch (IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } urlcount += container.size(); } @@ -195,6 +198,7 @@ public final class ReferenceContainerCache exte this.rot = rot; if (startWordHash != null && startWordHash.length == 0) startWordHash = null; this.cachecopy = sortedClone(); + assert this.cachecopy != null; this.p = 0; if (startWordHash != null) { while ( (this.p < this.cachecopy.length) && @@ -218,7 +222,12 @@ public final class ReferenceContainerCache exte if (this.p < this.cachecopy.length) { ReferenceContainer c = this.cachecopy[this.p++]; this.latestTermHash = c.getTermHash(); - return c.topLevelClone(); + try { + return c.topLevelClone(); + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; + } } // rotation iteration if (!rot) { @@ -228,7 +237,12 @@ public final class ReferenceContainerCache exte p = 0; ReferenceContainer c = this.cachecopy[this.p++]; this.latestTermHash = c.getTermHash(); - return c.topLevelClone(); + try { + return c.topLevelClone(); + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; + } } public void remove() { @@ -246,7 +260,7 @@ public final class ReferenceContainerCache exte * test if a given key is in the heap * this works with heaps in write- and read-mode * @param key - * @return true, if the key is used in the heap; false othervise + * @return true, if the key is used in the heap; false otherwise */ public boolean has(final byte[] key) { return this.cache.containsKey(new ByteArray(key)); @@ -267,7 +281,12 @@ public final class ReferenceContainerCache exte ReferenceType ee; while (e.hasNext()) { ee = e.next(); - if (urlselection.contains(ee.metadataHash())) c1.add(ee); + if (urlselection.contains(ee.metadataHash())) try { + c1.add(ee); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + break; + } } return c1; } @@ -286,7 +305,7 @@ public final class ReferenceContainerCache exte /** * delete a indexContainer from the heap cache. This can only be used for write-enabled heaps * @param wordHash - * @return the indexContainer if the cache contained the container, null othervise + * @return the indexContainer if the cache contained the container, null otherwise */ public ReferenceContainer delete(final byte[] termHash) { // returns the index that had been deleted @@ -332,7 +351,7 @@ public final class ReferenceContainerCache exte return 0; } - public void add(final ReferenceContainer container) { + public void add(final ReferenceContainer container) throws RowSpaceExceededException { // this puts the entries into the cache assert this.cache != null; if (this.cache == null || container == null || container.isEmpty()) return; @@ -356,7 +375,7 @@ public final class ReferenceContainerCache exte } } - public void add(final byte[] termHash, final ReferenceType newEntry) { + public void add(final byte[] termHash, final ReferenceType newEntry) throws RowSpaceExceededException { assert this.cache != null; ByteArray tha = new ByteArray(termHash); diff --git a/source/net/yacy/kelondro/rwi/TermSearch.java b/source/net/yacy/kelondro/rwi/TermSearch.java index 1d44f2c3c..a43da6216 100644 --- a/source/net/yacy/kelondro/rwi/TermSearch.java +++ b/source/net/yacy/kelondro/rwi/TermSearch.java @@ -31,6 +31,8 @@ import java.util.HashMap; import java.util.Set; import java.util.TreeSet; +import net.yacy.kelondro.index.RowSpaceExceededException; + public class TermSearch { @@ -43,7 +45,7 @@ public class TermSearch { final TreeSet excludeHashes, final Set urlselection, ReferenceFactory termFactory, - int maxDistance) { + int maxDistance) throws RowSpaceExceededException { this.inclusionContainers = (queryHashes.isEmpty()) ? diff --git a/source/net/yacy/kelondro/table/Records.java b/source/net/yacy/kelondro/table/Records.java index 0508225ff..267f54f23 100644 --- a/source/net/yacy/kelondro/table/Records.java +++ b/source/net/yacy/kelondro/table/Records.java @@ -147,7 +147,7 @@ public class Records { } synchronized void writeused(final boolean finalwrite) throws IOException { - // we write only at close time, not in between. othervise, the read/write head + // we write only at close time, not in between. Otherwise, the read/write head // needs to run up and own all the way between the beginning and the end of the // file for each record. We check consistency beteen file size and if (finalwrite) synchronized (entryFile) { @@ -1055,7 +1055,7 @@ public class Records { public Node(final Handle handle, final byte[] bulkchunk, final int offset) throws IOException { // this initializer is used to create nodes from bulk-read byte arrays // if write is true, then the chunk in bulkchunk is written to the file - // othervise it is considered equal to what is stored in the file + // otherwise it is considered equal to what is stored in the file // (that is ensured during pre-loaded enumeration) this.handle = handle; boolean changed; diff --git a/source/net/yacy/kelondro/table/Relations.java b/source/net/yacy/kelondro/table/Relations.java index d120b96f0..50356ed38 100755 --- a/source/net/yacy/kelondro/table/Relations.java +++ b/source/net/yacy/kelondro/table/Relations.java @@ -32,6 +32,7 @@ import java.util.HashMap; import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.NaturalOrder; @@ -78,7 +79,7 @@ public class Relations { return tablename + "-" + keysize + "-" + payloadsize + ".eco"; } - public void declareRelation(final String name, final int keysize, final int payloadsize) { + public void declareRelation(final String name, final int keysize, final int payloadsize) throws RowSpaceExceededException { // try to get the relation from the relation-cache final ObjectIndex relation = relations.get(name); if (relation != null) return; @@ -90,18 +91,28 @@ public class Relations { if (!list[i].equals(targetfilename)) continue; final Row row = rowdef(list[i]); if (row.primaryKeyLength != keysize || row.column(1).cellwidth != payloadsize) continue; // a wrong table - final ObjectIndex table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); + ObjectIndex table; + try { + table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); + } catch (RowSpaceExceededException e) { + table = new Table(new File(baseDir, list[i]), row, 0, 0, false, this.exceed134217727); + } relations.put(name, table); return; } } // the relation does not exist, create it final Row row = rowdef(keysize, payloadsize); - final ObjectIndex table = new Table(new File(baseDir, targetfilename), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); + ObjectIndex table; + try { + table = new Table(new File(baseDir, targetfilename), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); + } catch (RowSpaceExceededException e) { + table = new Table(new File(baseDir, targetfilename), row, 0, 0, false, this.exceed134217727); + } relations.put(name, table); } - public ObjectIndex getRelation(final String name) { + public ObjectIndex getRelation(final String name) throws RowSpaceExceededException { // try to get the relation from the relation-cache final ObjectIndex relation = relations.get(name); if (relation != null) return relation; @@ -110,7 +121,12 @@ public class Relations { for (int i = 0; i < list.length; i++) { if (list[i].startsWith(name)) { final Row row = rowdef(list[i]); - final ObjectIndex table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); + ObjectIndex table; + try { + table = new Table(new File(baseDir, list[i]), row, 1024*1024, 0, this.useTailCache, this.exceed134217727); + } catch (RowSpaceExceededException e) { + table = new Table(new File(baseDir, list[i]), row, 0, 0, false, this.exceed134217727); + } relations.put(name, table); return table; } @@ -119,13 +135,13 @@ public class Relations { return null; } - public String putRelation(final String name, final String key, final String value) throws IOException { + public String putRelation(final String name, final String key, final String value) throws IOException, RowSpaceExceededException { final byte[] r = putRelation(name, key.getBytes(), value.getBytes()); if (r == null) return null; return new String(r); } - public byte[] putRelation(final String name, final byte[] key, final byte[] value) throws IOException { + public byte[] putRelation(final String name, final byte[] key, final byte[] value) throws IOException, RowSpaceExceededException { final ObjectIndex table = getRelation(name); if (table == null) return null; final Row.Entry entry = table.row().newEntry(); @@ -138,13 +154,13 @@ public class Relations { return oldentry.getColBytes(3); } - public String getRelation(final String name, final String key) throws IOException { + public String getRelation(final String name, final String key) throws IOException, RowSpaceExceededException { final byte[] r = getRelation(name, key.getBytes()); if (r == null) return null; return new String(r); } - public byte[] getRelation(final String name, final byte[] key) throws IOException { + public byte[] getRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException { final ObjectIndex table = getRelation(name); if (table == null) return null; final Row.Entry entry = table.get(key); @@ -152,13 +168,13 @@ public class Relations { return entry.getColBytes(3); } - public boolean hasRelation(final String name, final byte[] key) { + public boolean hasRelation(final String name, final byte[] key) throws RowSpaceExceededException { final ObjectIndex table = getRelation(name); if (table == null) return false; return table.has(key); } - public byte[] removeRelation(final String name, final byte[] key) throws IOException { + public byte[] removeRelation(final String name, final byte[] key) throws IOException, RowSpaceExceededException { final ObjectIndex table = getRelation(name); if (table == null) return null; final Row.Entry entry = table.remove(key); @@ -175,6 +191,8 @@ public class Relations { r.putRelation(table1, "abcdefg", "eineintrag"); } catch (final IOException e) { Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); } } diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index fe0b42be0..ee3ce883e 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -48,6 +48,7 @@ import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.ObjectIndexCache; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowCollection; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.CloneableIterator; @@ -90,7 +91,7 @@ public class SplitTable implements ObjectIndex, Iterable { final String tablename, final Row rowdef, final boolean useTailCache, - final boolean exceed134217727) { + final boolean exceed134217727) throws RowSpaceExceededException { this(path, tablename, rowdef, ArrayStack.oneMonth, (long) Integer.MAX_VALUE, useTailCache, exceed134217727); } @@ -101,7 +102,7 @@ public class SplitTable implements ObjectIndex, Iterable { final long fileAgeLimit, final long fileSizeLimit, final boolean useTailCache, - final boolean exceed134217727) { + final boolean exceed134217727) throws RowSpaceExceededException { this.path = path; this.prefix = tablename; this.rowdef = rowdef; @@ -125,7 +126,7 @@ public class SplitTable implements ObjectIndex, Iterable { return prefix + "." + DateFormatter.formatShortMilliSecond(new Date()) + ".table"; } - public void init() { + public void init() throws RowSpaceExceededException { current = null; // initialized tables map @@ -205,7 +206,11 @@ public class SplitTable implements ObjectIndex, Iterable { if (maxf != null) { f = new File(path, maxf); Log.logInfo("kelondroSplitTable", "opening partial eco table " + f); - table = new Table(f, rowdef, EcoFSBufferSize, 0, this.useTailCache, this.exceed134217727); + try { + table = new Table(f, rowdef, EcoFSBufferSize, 0, this.useTailCache, this.exceed134217727); + } catch (RowSpaceExceededException e) { + table = new Table(f, rowdef, 0, 0, false, this.exceed134217727); + } tables.put(maxf, table); } } @@ -230,7 +235,16 @@ public class SplitTable implements ObjectIndex, Iterable { if (f.isDirectory()) delete(path, l[i]); else FileUtils.deletedelete(f); } } - init(); + try { + init(); + } catch (RowSpaceExceededException e) { + useTailCache = false; + try { + init(); + } catch (RowSpaceExceededException e1) { + throw new IOException(e1); + } + } } public static void delete(final File path, final String tablename) { @@ -291,7 +305,16 @@ public class SplitTable implements ObjectIndex, Iterable { private ObjectIndex newTable() { this.current = newFilename(); final File f = new File(path, this.current); - Table table = new Table(f, rowdef, EcoFSBufferSize, 0, this.useTailCache, this.exceed134217727); + Table table = null; + try { + table = new Table(f, rowdef, EcoFSBufferSize, 0, this.useTailCache, this.exceed134217727); + } catch (RowSpaceExceededException e) { + try { + table = new Table(f, rowdef, 0, 0, false, this.exceed134217727); + } catch (RowSpaceExceededException e1) { + Log.logException(e1); + } + } tables.put(this.current, table); return table; } @@ -314,7 +337,7 @@ public class SplitTable implements ObjectIndex, Iterable { return table; } - public synchronized Row.Entry replace(final Row.Entry row) throws IOException { + public synchronized Row.Entry replace(final Row.Entry row) throws IOException, RowSpaceExceededException { assert row.objectsize() <= this.rowdef.objectsize; ObjectIndex keeper = keeperOf(row.getColBytes(0)); if (keeper != null) return keeper.replace(row); @@ -323,7 +346,7 @@ public class SplitTable implements ObjectIndex, Iterable { return null; } - public synchronized void put(final Row.Entry row) throws IOException { + public synchronized void put(final Row.Entry row) throws IOException, RowSpaceExceededException { assert row.objectsize() <= this.rowdef.objectsize; ObjectIndex keeper = keeperOf(row.getColBytes(0)); if (keeper != null) {keeper.put(row); return;} @@ -551,14 +574,14 @@ public class SplitTable implements ObjectIndex, Iterable { return null; } */ - public synchronized void addUnique(final Row.Entry row) throws IOException { + public synchronized void addUnique(final Row.Entry row) throws IOException, RowSpaceExceededException { assert row.objectsize() <= this.rowdef.objectsize; ObjectIndex table = (this.current == null) ? null : tables.get(this.current); if (table == null) table = newTable(); else table = checkTable(table); table.addUnique(row); } - public ArrayList removeDoubles() throws IOException { + public ArrayList removeDoubles() throws IOException, RowSpaceExceededException { final Iterator i = tables.values().iterator(); final ArrayList report = new ArrayList(); while (i.hasNext()) { diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index ff7ea8bdd..86085aaa0 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -44,6 +44,7 @@ import net.yacy.kelondro.index.ObjectIndex; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowCollection; import net.yacy.kelondro.index.RowSet; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.io.BufferedRecords; import net.yacy.kelondro.io.Records; @@ -56,10 +57,10 @@ import net.yacy.kelondro.util.kelondroException; /* - * The EcoIndex builds upon the EcoFS and tries to reduce the number of IO requests that the + * The Table builds upon the EcoFS and tries to reduce the number of IO requests that the * EcoFS must do to a minimum. In best cases, no IO has to be done for read operations (complete database shadow in RAM) * and a rare number of write IO operations must be done for a large number of table-writings (using the write buffer of EcoFS) - * To make the EcoIndex scalable in question of available RAM, there are two elements that must be scalable: + * To make the Table scalable in question of available RAM, there are two elements that must be scalable: * - the access index can be either completely in RAM (kelondroRAMIndex) or it is file-based (kelondroTree) * - the content cache can be either a complete RAM-based shadow of the File, or empty. * The content cache can also be deleted during run-time, if the available RAM gets too low. @@ -87,7 +88,7 @@ public class Table implements ObjectIndex, Iterable { final int buffersize, final int initialSpace, final boolean useTailCache, - final boolean exceed134217727) { + final boolean exceed134217727) throws RowSpaceExceededException { this.tablefile = tablefile; this.rowdef = rowdef; this.buffersize = buffersize; @@ -169,7 +170,12 @@ public class Table implements ObjectIndex, Iterable { if (rowdef.objectOrder.wellformed(key)) { index.putUnique(key, i++); // write the tail into the table - table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); + try { + table.addUnique(taildef.newEntry(record, rowdef.primaryKeyLength, true)); + } catch (RowSpaceExceededException e) { + table = null; + break; + } if (abandonTable()) { table = null; break; @@ -282,35 +288,53 @@ public class Table implements ObjectIndex, Iterable { return (int) (((long)(rowdef.primaryKeyLength + 4)) * tableSize(f, rowdef.objectsize) * RowCollection.growfactorLarge100 / 100L); } - public synchronized void addUnique(final Entry row) throws IOException { + public synchronized void addUnique(final Entry row) throws IOException, RowSpaceExceededException { assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); final int i = (int) file.size(); - index.putUnique(row.getPrimaryKeyBytes(), i); + try { + index.putUnique(row.getPrimaryKeyBytes(), i); + } catch (RowSpaceExceededException e) { + if (table == null) throw e; // in case the table is not used, there is no help here + table = null; + // try again with less memory + index.putUnique(row.getPrimaryKeyBytes(), i); + } if (table != null) { assert table.size() == i; - table.addUnique(taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + try { + table.addUnique(taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + } catch (RowSpaceExceededException e) { + table = null; + } if (abandonTable()) table = null; } file.add(row.bytes(), 0); assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); } - public synchronized void addUnique(final List rows) throws IOException { + public synchronized void addUnique(final List rows) throws IOException, RowSpaceExceededException { assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); - final Iterator i = rows.iterator(); - while (i.hasNext()) { - addUnique(i.next()); + for (Entry entry: rows) { + try { + addUnique(entry); + } catch (RowSpaceExceededException e) { + if (this.table == null) throw e; + table = null; + addUnique(entry); + } } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); } /** + * @throws RowSpaceExceededException * remove double-entries from the table * this process calls the underlying removeDoubles() method from the table index * and + * @throws */ - public synchronized ArrayList removeDoubles() throws IOException { + public synchronized ArrayList removeDoubles() throws IOException, RowSpaceExceededException { assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); final ArrayList report = new ArrayList(); RowSet rows; @@ -320,7 +344,15 @@ public class Table implements ObjectIndex, Iterable { Row.Entry inconsistentEntry; // iterate over all entries that have inconsistent index references long lastlog = System.currentTimeMillis(); - for (final Long[] is: index.removeDoubles()) { + ArrayList doubles; + try { + doubles = index.removeDoubles(); + } catch (RowSpaceExceededException e) { + if (this.table == null) throw e; + table = null; + doubles = index.removeDoubles(); + } + for (final Long[] is: doubles) { // 'is' is the set of all indexes, that have the same reference // we collect that entries now here rows = new RowSet(this.rowdef, is.length); @@ -331,7 +363,13 @@ public class Table implements ObjectIndex, Iterable { if (L.intValue() >= file.size()) continue; // prevent IndexOutOfBoundsException file.get(L.intValue(), b, 0); // TODO: fix IndexOutOfBoundsException here inconsistentEntry = rowdef.newEntry(b); - rows.addUnique(inconsistentEntry); + try { + rows.addUnique(inconsistentEntry); + } catch (RowSpaceExceededException e) { + if (this.table == null) throw e; + this.table = null; + rows.addUnique(inconsistentEntry); + } } report.add(rows); } @@ -401,7 +439,7 @@ public class Table implements ObjectIndex, Iterable { return index.keys(up, firstKey); } - public synchronized Entry replace(final Entry row) throws IOException { + public synchronized Entry replace(final Entry row) throws IOException, RowSpaceExceededException { assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); assert row != null; @@ -409,7 +447,13 @@ public class Table implements ObjectIndex, Iterable { if ((row == null) || (row.bytes() == null)) return null; final int i = (int) index.get(row.getPrimaryKeyBytes()); if (i == -1) { - addUnique(row); + try { + addUnique(row); + } catch (RowSpaceExceededException e) { + if (this.table == null) throw e; + this.table = null; + addUnique(row); + } return null; } @@ -426,7 +470,11 @@ public class Table implements ObjectIndex, Iterable { System.arraycopy(row.getPrimaryKeyBytes(), 0, b, 0, rowdef.primaryKeyLength); System.arraycopy(v.bytes(), 0, b, rowdef.primaryKeyLength, rowdef.objectsize - rowdef.primaryKeyLength); // write new value - table.set(i, taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + try { + table.set(i, taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + } catch (RowSpaceExceededException e) { + table = null; + } file.put(i, row.bytes(), 0); } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); @@ -435,7 +483,7 @@ public class Table implements ObjectIndex, Iterable { return rowdef.newEntry(b); } - public synchronized void put(final Entry row) throws IOException { + public synchronized void put(final Entry row) throws IOException, RowSpaceExceededException { assert file == null || file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); assert row != null; @@ -443,7 +491,13 @@ public class Table implements ObjectIndex, Iterable { if (file == null || row == null || row.bytes() == null) return; final int i = (int) index.get(row.getPrimaryKeyBytes()); if (i == -1) { - addUnique(row); + try { + addUnique(row); + } catch (RowSpaceExceededException e) { + if (this.table == null) throw e; + this.table = null; + addUnique(row); + } return; } @@ -452,14 +506,18 @@ public class Table implements ObjectIndex, Iterable { file.put(i, row.bytes(), 0); } else { // write new value - table.set(i, taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + try { + table.set(i, taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); + } catch (RowSpaceExceededException e) { + table = null; + } file.put(i, row.bytes(), 0); } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); } - public synchronized Entry put(final Entry row, final Date entryDate) throws IOException { + public synchronized Entry put(final Entry row, final Date entryDate) throws IOException, RowSpaceExceededException { return replace(row); } @@ -467,8 +525,9 @@ public class Table implements ObjectIndex, Iterable { * remove one entry from the file * @param i an index position within the file (not a byte position) * @throws IOException + * @throws RowSpaceExceededException */ - private void removeInFile(final int i) throws IOException { + private void removeInFile(final int i) throws IOException, RowSpaceExceededException { assert i >= 0; final byte[] p = new byte[rowdef.objectsize]; @@ -490,7 +549,11 @@ public class Table implements ObjectIndex, Iterable { } else { // switch values final Row.Entry te = table.removeOne(); - table.set(i, te); + try { + table.set(i, te); + } catch (RowSpaceExceededException e) { + table = null; + } file.cleanLast(p, 0); file.put(i, p, 0); @@ -533,7 +596,12 @@ public class Table implements ObjectIndex, Iterable { file.put(i, p, 0); final byte[] k = new byte[rowdef.primaryKeyLength]; System.arraycopy(p, 0, k, 0, rowdef.primaryKeyLength); - index.put(k, i); + try { + index.put(k, i); + } catch (RowSpaceExceededException e) { + Log.logException(e); + throw new IOException("RowSpaceExceededException: " + e.getMessage()); + } } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); } else { @@ -557,14 +625,24 @@ public class Table implements ObjectIndex, Iterable { // remove last entry from the file copy to fill it in the gap final Row.Entry te = table.removeOne(); // fill the gap in file copy - table.set(i, te); + try { + table.set(i, te); + } catch (RowSpaceExceededException e) { + Log.logException(e); + table = null; + } // move entry from last entry in file to gap position file.cleanLast(p, 0); file.put(i, p, 0); // set new index for moved entry in index final Row.Entry lr = rowdef.newEntry(p); - index.put(lr.getPrimaryKeyBytes(), i); + try { + index.put(lr.getPrimaryKeyBytes(), i); + } catch (RowSpaceExceededException e) { + table = null; + throw new IOException("RowSpaceExceededException: " + e.getMessage()); + } } assert file.size() == index.size() : "file.size() = " + file.size() + ", index.size() = " + index.size(); assert table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size(); @@ -760,7 +838,7 @@ public class Table implements ObjectIndex, Iterable { return result; } - private static Table testTable(final File f, final String testentities, final boolean useTailCache, final boolean exceed134217727) throws IOException { + private static Table testTable(final File f, final String testentities, final boolean useTailCache, final boolean exceed134217727) throws IOException, RowSpaceExceededException { if (f.exists()) FileUtils.deletedelete(f); final Row rowdef = new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder); final Table tt = new Table(f, rowdef, 100, 0, useTailCache, exceed134217727); diff --git a/source/net/yacy/kelondro/util/AttrSeq.java b/source/net/yacy/kelondro/util/AttrSeq.java index 35f160cb0..cc9f169fe 100644 --- a/source/net/yacy/kelondro/util/AttrSeq.java +++ b/source/net/yacy/kelondro/util/AttrSeq.java @@ -46,6 +46,7 @@ import java.util.zip.GZIPInputStream; import net.yacy.kelondro.index.Column; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowCollection; +import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; @@ -390,7 +391,7 @@ public class AttrSeq { return seq; } - public RowCollection getSeqCollection() { + public RowCollection getSeqCollection() throws RowSpaceExceededException { final RowCollection collection = new RowCollection(structure.seqrow, seq.size()); final Iterator i = seq.iterator(); while (i.hasNext()) { diff --git a/source/net/yacy/kelondro/util/ByteBuffer.java b/source/net/yacy/kelondro/util/ByteBuffer.java index 03b502bb0..b535754d4 100644 --- a/source/net/yacy/kelondro/util/ByteBuffer.java +++ b/source/net/yacy/kelondro/util/ByteBuffer.java @@ -318,7 +318,7 @@ public final class ByteBuffer extends OutputStream { // - first byte begins with 110, the following byte begins with 10 // - first byte begins with 1110, the following two bytes begin with 10 // - First byte begins with 11110, the following three bytes begin with 10 - // if an utf-8 sequence is detected, the length of the sequence is returned. -1 othervise + // if an utf-8 sequence is detected, the length of the sequence is returned. -1 otherwise if ((start < length) && ((buffer[offset + start] & 0x80) != 0)) return 1; if ((start < length - 1) && diff --git a/source/net/yacy/kelondro/util/DateFormatter.java b/source/net/yacy/kelondro/util/DateFormatter.java index c1316d9e2..eaf77eca0 100644 --- a/source/net/yacy/kelondro/util/DateFormatter.java +++ b/source/net/yacy/kelondro/util/DateFormatter.java @@ -559,7 +559,7 @@ public final class DateFormatter { } public DateFormatter(final String datestring) throws java.text.ParseException { - // parse a date string; othervise throw a java.text.ParseException + // parse a date string; otherwise throw a java.text.ParseException if ((datestring.length() == 14) || (datestring.length() == 17)) { // parse a ShortString try {years = Integer.parseInt(datestring.substring(0, 4)) - 1970;} catch (final NumberFormatException e) { diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 1588801bb..95ec2c68d 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -103,7 +103,7 @@ import de.anomic.yacy.yacyVersion; * * On termination, the following must be done: *
    -*
  • stop feeding of the crawling process because it othervise fills the +*
  • stop feeding of the crawling process because it otherwise fills the * indexing queue. *
  • say goodbye to connected peers and disable new connections. Don't wait for * success.