diff --git a/source/de/anomic/crawler/CrawlStacker.java b/source/de/anomic/crawler/CrawlStacker.java index a3c298112..b64375dda 100644 --- a/source/de/anomic/crawler/CrawlStacker.java +++ b/source/de/anomic/crawler/CrawlStacker.java @@ -286,27 +286,27 @@ public final class CrawlStacker { // it may be possible that global == true and local == true, so do not check an error case against it if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); - int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT); + //int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT); nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_LIMIT, entry); - assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT); + //assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT); this.log.logInfo("stacked/global: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT)); } else if (local) { if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); - int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); + //int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry); - assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); + //assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); this.log.logInfo("stacked/local: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE)); } else if (proxy) { if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle()); - int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); + //int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry); - assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); + //assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE); this.log.logInfo("stacked/proxy: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE)); } else if (remote) { - int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE); + //int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE); nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_REMOTE, entry); - assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE); + //assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE); this.log.logInfo("stacked/remote: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE)); } diff --git a/source/de/anomic/kelondro/blob/Gap.java b/source/de/anomic/kelondro/blob/Gap.java index aae05f045..8f7746216 100644 --- a/source/de/anomic/kelondro/blob/Gap.java +++ b/source/de/anomic/kelondro/blob/Gap.java @@ -74,6 +74,7 @@ public class Gap extends TreeMap { } } is.close(); + is = null; } /** diff --git a/source/de/anomic/kelondro/blob/HeapReader.java b/source/de/anomic/kelondro/blob/HeapReader.java index 8d55c1b45..982fa3d86 100644 --- a/source/de/anomic/kelondro/blob/HeapReader.java +++ b/source/de/anomic/kelondro/blob/HeapReader.java @@ -344,6 +344,9 @@ public class HeapReader { } public CloneableIterator> clone(Object modifier) { + // if the entries iterator is cloned, close the file! + if (is != null) try { is.close(); } catch (final IOException e) {} + is = null; try { return new entries(blobFile, keylen); } catch (IOException e) { @@ -353,7 +356,10 @@ public class HeapReader { } public boolean hasNext() { - return this.nextEntry != null; + if (is == null) return false; + if (this.nextEntry != null) return true; + close(); + return false; } private Map.Entry next0() { diff --git a/source/de/anomic/kelondro/index/LongHandleIndex.java b/source/de/anomic/kelondro/index/LongHandleIndex.java index a494dbb6d..14563afed 100644 --- a/source/de/anomic/kelondro/index/LongHandleIndex.java +++ b/source/de/anomic/kelondro/index/LongHandleIndex.java @@ -83,6 +83,7 @@ public class LongHandleIndex { this.index.addUnique(this.rowdef.newEntry(a)); } is.close(); + is = null; assert this.index.size() == file.length() / (keylength + 8); } diff --git a/source/de/anomic/kelondro/io/CachedRandomAccess.java b/source/de/anomic/kelondro/io/CachedRandomAccess.java index 6dbc23f7f..3fd65b1b6 100644 --- a/source/de/anomic/kelondro/io/CachedRandomAccess.java +++ b/source/de/anomic/kelondro/io/CachedRandomAccess.java @@ -127,7 +127,10 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra public synchronized void close() { if (RAFile != null) try { + RAFile.getChannel().close(); + //System.out.println("***DEBUG*** closed file " + this.file + ", FD is " + ((RAFile.getFD().valid()) ? "VALID" : "VOID") + ", channel is " + ((RAFile.getChannel().isOpen()) ? "OPEN" : "CLOSE")); RAFile.close(); + //System.out.println("***DEBUG*** closed file " + this.file + ", FD is " + ((RAFile.getFD().valid()) ? "VALID" : "VOID") + ", channel is " + ((RAFile.getChannel().isOpen()) ? "OPEN" : "CLOSE")); } catch (IOException e) { e.printStackTrace(); } @@ -137,9 +140,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra } protected void finalize() throws Throwable { - if (RAFile != null) { - this.close(); - } + this.close(); super.finalize(); } diff --git a/source/de/anomic/kelondro/order/Digest.java b/source/de/anomic/kelondro/order/Digest.java index 6d6a074ad..2406f90a9 100644 --- a/source/de/anomic/kelondro/order/Digest.java +++ b/source/de/anomic/kelondro/order/Digest.java @@ -260,7 +260,6 @@ public class Digest { final int mb = 16 * 1024; final long fl = file.length(); if (fl <= 2 * mb) return encodeMD5Raw(file); - RandomAccessFile raf = new RandomAccessFile(file, "r"); MessageDigest digest; try { digest = MessageDigest.getInstance("MD5"); @@ -268,15 +267,21 @@ public class Digest { e.printStackTrace(); return null; } - raf.seek(0); + RandomAccessFile raf = new RandomAccessFile(file, "r"); byte[] a = new byte[mb]; - raf.readFully(a, 0, mb - 1); - digest.update(a, 0, mb); - raf.seek(fl - mb); - raf.readFully(a, 0, mb - 1); - digest.update(a, 0, mb); - digest.update(NaturalOrder.encodeLong(fl, 8), 0, 8); - if (includeDate) digest.update(NaturalOrder.encodeLong(file.lastModified(), 8), 0, 8); + try { + raf.seek(0); + raf.readFully(a, 0, mb - 1); + digest.update(a, 0, mb); + raf.seek(fl - mb); + raf.readFully(a, 0, mb - 1); + digest.update(a, 0, mb); + digest.update(NaturalOrder.encodeLong(fl, 8), 0, 8); + if (includeDate) digest.update(NaturalOrder.encodeLong(file.lastModified(), 8), 0, 8); + } finally { + raf.close(); + raf.getChannel().close(); + } return digest.digest(); } diff --git a/source/de/anomic/kelondro/table/Stack.java b/source/de/anomic/kelondro/table/Stack.java index 0a4836730..e0f91b68e 100644 --- a/source/de/anomic/kelondro/table/Stack.java +++ b/source/de/anomic/kelondro/table/Stack.java @@ -277,7 +277,7 @@ public final class Stack extends FullRecords { } return recs; } finally { - if (f!=null) try{f.close();}catch(final Exception e){} + if (f!=null) try{f.close(); f.getChannel().close();}catch(final Exception e){} } } diff --git a/source/de/anomic/kelondro/text/ReferenceContainerCache.java b/source/de/anomic/kelondro/text/ReferenceContainerCache.java index 78086d7dc..1370eb3bd 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerCache.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerCache.java @@ -153,7 +153,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde * static iterator of BLOBHeap files: is used to import heap dumps into a write-enabled index heap */ public static class blobFileEntries implements CloneableIterator, Iterable { - Iterator> blobs; + HeapReader.entries blobs; Row payloadrow; File blobFile; @@ -164,7 +164,10 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde } public boolean hasNext() { - return blobs.hasNext(); + if (blobs == null) return false; + if (blobs.hasNext()) return true; + close(); + return false; } /** @@ -186,6 +189,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde } public void close() { + if (blobs != null) this.blobs.close(); blobs = null; } @@ -194,6 +198,8 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde } public CloneableIterator clone(Object modifier) { + if (blobs != null) this.blobs.close(); + blobs = null; try { return new blobFileEntries(this.blobFile, this.payloadrow); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/util/FileUtils.java b/source/de/anomic/kelondro/util/FileUtils.java index be9c3dc69..13e21ea49 100644 --- a/source/de/anomic/kelondro/util/FileUtils.java +++ b/source/de/anomic/kelondro/util/FileUtils.java @@ -733,12 +733,26 @@ public final class FileUtils { } } - if (!path.delete()) path.deleteOnExit(); + int c = 0; + while (c++ < 20) { + if (!path.exists()) break; + if (path.delete()) break; + // some OS may be slow when giving up file pointer + //System.runFinalization(); + //System.gc(); + try { Thread.sleep(200); } catch (InterruptedException e) { break; } + } if (path.exists()) { + path.deleteOnExit(); + String p = ""; + try { + p = path.getCanonicalPath(); + } catch (IOException e1) { + e1.printStackTrace(); + } if (System.getProperties().getProperty("os.name","").toLowerCase().startsWith("windows")) { // deleting files on windows sometimes does not work with java try { - String p = path.getCanonicalPath(); String command = "cmd /C del /F /Q " + p; Process r = Runtime.getRuntime().exec(command); if (r == null) { @@ -747,11 +761,11 @@ public final class FileUtils { byte[] response = read(r.getInputStream()); Log.logInfo("FileUtils", "deletedelete: " + new String(response)); } - if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p); } catch (IOException e) { e.printStackTrace(); } } + if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p); } } } diff --git a/source/de/anomic/tools/mediawikiIndex.java b/source/de/anomic/tools/mediawikiIndex.java index d9b41f6b2..2c124dcdc 100644 --- a/source/de/anomic/tools/mediawikiIndex.java +++ b/source/de/anomic/tools/mediawikiIndex.java @@ -309,17 +309,22 @@ public class mediawikiIndex { } public static byte[] read(File f, long start, int len) { + byte[] b = new byte[len]; + RandomAccessFile raf = null; try { - RandomAccessFile raf = new RandomAccessFile(f, "r"); - byte[] b = new byte[len]; + raf = new RandomAccessFile(f, "r"); raf.seek(start); raf.read(b); - raf.close(); - return b; } catch (IOException e) { e.printStackTrace(); + return null; + } finally { + if (raf != null) try { + raf.close(); + raf.getChannel().close(); + } catch (IOException e) { } } - return null; + return b; } public static wikirecord find(String title, File f) throws IOException {