boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more care about open file handles. Now files also close on windows and can be deleted afterwards.

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5760 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 029495e64d
commit d39a5b42ca
10 changed files with 71 additions and 32 deletions
Show Stats Download Patch File Download Diff File

16
source/de/anomic/crawler/CrawlStacker.java
Unescape View File

@ -286,27 +286,27 @@ public final class CrawlStacker {
// it may be possible that global == true and local == true, so do not check an error case against it
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: global = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_LIMIT, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT);
this.log.logInfo("stacked/global: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_LIMIT));
} else if (local) {
if (proxy) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, proxy = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: local = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
this.log.logInfo("stacked/local: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
} else if (proxy) {
if (remote) this.log.logWarning("URL '" + entry.url().toString() + "' has conflicting initiator properties: proxy = true, remote = true, initiator = " + entry.initiator() + ", profile.handle = " + profile.handle());
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_CORE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE);
this.log.logInfo("stacked/proxy: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_CORE));
} else if (remote) {
int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
//int b = nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
nextQueue.noticeURL.push(NoticedURL.STACK_TYPE_REMOTE, entry);
assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
//assert b < nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE);
this.log.logInfo("stacked/remote: " + entry.url().toString() + ", stacksize = " + nextQueue.noticeURL.stackSize(NoticedURL.STACK_TYPE_REMOTE));
}

1
source/de/anomic/kelondro/blob/Gap.java
Unescape View File

@ -74,6 +74,7 @@ public class Gap extends TreeMap<Long, Integer> {
}
}
is.close();
is = null;
}
/**

8
source/de/anomic/kelondro/blob/HeapReader.java
Unescape View File

@ -344,6 +344,9 @@ public class HeapReader {
}
public CloneableIterator<Entry<String, byte[]>> clone(Object modifier) {
// if the entries iterator is cloned, close the file!
if (is != null) try { is.close(); } catch (final IOException e) {}
is = null;
try {
return new entries(blobFile, keylen);
} catch (IOException e) {
@ -353,7 +356,10 @@ public class HeapReader {
}
public boolean hasNext() {
return this.nextEntry != null;
if (is == null) return false;
if (this.nextEntry != null) return true;
close();
return false;
}
private Map.Entry<String, byte[]> next0() {

1
source/de/anomic/kelondro/index/LongHandleIndex.java
Unescape View File

@ -83,6 +83,7 @@ public class LongHandleIndex {
this.index.addUnique(this.rowdef.newEntry(a));
}
is.close();
is = null;
assert this.index.size() == file.length() / (keylength + 8);
}

7
source/de/anomic/kelondro/io/CachedRandomAccess.java
Unescape View File

@ -127,7 +127,10 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
public synchronized void close() {
if (RAFile != null) try {
RAFile.getChannel().close();
//System.out.println("***DEBUG*** closed file " + this.file + ", FD is " + ((RAFile.getFD().valid()) ? "VALID" : "VOID") + ", channel is " + ((RAFile.getChannel().isOpen()) ? "OPEN" : "CLOSE"));
RAFile.close();
//System.out.println("***DEBUG*** closed file " + this.file + ", FD is " + ((RAFile.getFD().valid()) ? "VALID" : "VOID") + ", channel is " + ((RAFile.getChannel().isOpen()) ? "OPEN" : "CLOSE"));
} catch (IOException e) {
e.printStackTrace();
}
@ -137,9 +140,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
}
protected void finalize() throws Throwable {
if (RAFile != null) {
this.close();
}
this.close();
super.finalize();
}

23
source/de/anomic/kelondro/order/Digest.java
Unescape View File

@ -260,7 +260,6 @@ public class Digest {
final int mb = 16 * 1024;
final long fl = file.length();
if (fl <= 2 * mb) return encodeMD5Raw(file);
RandomAccessFile raf = new RandomAccessFile(file, "r");
MessageDigest digest;
try {
digest = MessageDigest.getInstance("MD5");
@ -268,15 +267,21 @@ public class Digest {
e.printStackTrace();
return null;
}
raf.seek(0);
RandomAccessFile raf = new RandomAccessFile(file, "r");
byte[] a = new byte[mb];
raf.readFully(a, 0, mb - 1);
digest.update(a, 0, mb);
raf.seek(fl - mb);
raf.readFully(a, 0, mb - 1);
digest.update(a, 0, mb);
digest.update(NaturalOrder.encodeLong(fl, 8), 0, 8);
if (includeDate) digest.update(NaturalOrder.encodeLong(file.lastModified(), 8), 0, 8);
try {
raf.seek(0);
raf.readFully(a, 0, mb - 1);
digest.update(a, 0, mb);
raf.seek(fl - mb);
raf.readFully(a, 0, mb - 1);
digest.update(a, 0, mb);
digest.update(NaturalOrder.encodeLong(fl, 8), 0, 8);
if (includeDate) digest.update(NaturalOrder.encodeLong(file.lastModified(), 8), 0, 8);
} finally {
raf.close();
raf.getChannel().close();
}
return digest.digest();
}

2
source/de/anomic/kelondro/table/Stack.java
Unescape View File

@ -277,7 +277,7 @@ public final class Stack extends FullRecords {
}
return recs;
} finally {
if (f!=null) try{f.close();}catch(final Exception e){}
if (f!=null) try{f.close(); f.getChannel().close();}catch(final Exception e){}
}
}

10
source/de/anomic/kelondro/text/ReferenceContainerCache.java
Unescape View File

@ -153,7 +153,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
* static iterator of BLOBHeap files: is used to import heap dumps into a write-enabled index heap
*/
public static class blobFileEntries implements CloneableIterator<ReferenceContainer>, Iterable<ReferenceContainer> {
Iterator<Map.Entry<String, byte[]>> blobs;
HeapReader.entries blobs;
Row payloadrow;
File blobFile;
@ -164,7 +164,10 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
}
public boolean hasNext() {
return blobs.hasNext();
if (blobs == null) return false;
if (blobs.hasNext()) return true;
close();
return false;
}
/**
@ -186,6 +189,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
}
public void close() {
if (blobs != null) this.blobs.close();
blobs = null;
}
@ -194,6 +198,8 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
}
public CloneableIterator<ReferenceContainer> clone(Object modifier) {
if (blobs != null) this.blobs.close();
blobs = null;
try {
return new blobFileEntries(this.blobFile, this.payloadrow);
} catch (IOException e) {

20
source/de/anomic/kelondro/util/FileUtils.java
Unescape View File

@ -733,12 +733,26 @@ public final class FileUtils {
}
}
if (!path.delete()) path.deleteOnExit();
int c = 0;
while (c++ < 20) {
if (!path.exists()) break;
if (path.delete()) break;
// some OS may be slow when giving up file pointer
//System.runFinalization();
//System.gc();
try { Thread.sleep(200); } catch (InterruptedException e) { break; }
}
if (path.exists()) {
path.deleteOnExit();
String p = "";
try {
p = path.getCanonicalPath();
} catch (IOException e1) {
e1.printStackTrace();
}
if (System.getProperties().getProperty("os.name","").toLowerCase().startsWith("windows")) {
// deleting files on windows sometimes does not work with java
try {
String p = path.getCanonicalPath();
String command = "cmd /C del /F /Q " + p;
Process r = Runtime.getRuntime().exec(command);
if (r == null) {
@ -747,11 +761,11 @@ public final class FileUtils {
byte[] response = read(r.getInputStream());
Log.logInfo("FileUtils", "deletedelete: " + new String(response));
}
if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p);
} catch (IOException e) {
e.printStackTrace();
}
}
if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p);
}
}
}

15
source/de/anomic/tools/mediawikiIndex.java
Unescape View File

@ -309,17 +309,22 @@ public class mediawikiIndex {
}
public static byte[] read(File f, long start, int len) {
byte[] b = new byte[len];
RandomAccessFile raf = null;
try {
RandomAccessFile raf = new RandomAccessFile(f, "r");
byte[] b = new byte[len];
raf = new RandomAccessFile(f, "r");
raf.seek(start);
raf.read(b);
raf.close();
return b;
} catch (IOException e) {
e.printStackTrace();
return null;
} finally {
if (raf != null) try {
raf.close();
raf.getChannel().close();
} catch (IOException e) { }
}
return null;
return b;
}
public static wikirecord find(String title, File f) throws IOException {

Write Preview
Loading…
Cancel
Save