From 96eaecda3e4ded11f0969532341d031cfda4b255 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 30 Mar 2009 15:31:25 +0000 Subject: [PATCH] - added migration class to go from index collections to the index cell data structure. - added better control over file deletion, because this sometimes fails, especially on windows git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5756 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ConfigAppearance_p.java | 2 +- htroot/ConfigLanguage_p.java | 2 +- htroot/ConfigUpdate_p.java | 3 +- source/de/anomic/crawler/CrawlProfile.java | 3 +- source/de/anomic/crawler/CrawlQueues.java | 3 +- .../de/anomic/crawler/NoticeURLImporter.java | 3 +- source/de/anomic/crawler/RobotsTxt.java | 2 +- source/de/anomic/crawler/ZURL.java | 3 +- source/de/anomic/data/userDB.java | 3 +- source/de/anomic/kelondro/blob/BLOBArray.java | 11 +- source/de/anomic/kelondro/blob/BLOBHeap.java | 1 + .../kelondro/blob/BLOBHeapModifier.java | 16 +- source/de/anomic/kelondro/blob/BLOBTree.java | 6 +- .../de/anomic/kelondro/blob/HeapReader.java | 14 +- .../de/anomic/kelondro/blob/HeapWriter.java | 3 +- source/de/anomic/kelondro/blob/MapView.java | 3 +- .../kelondro/io/CachedRandomAccess.java | 8 +- source/de/anomic/kelondro/io/EcoFS.java | 3 +- .../kelondro/table/AbstractRecords.java | 3 +- source/de/anomic/kelondro/table/EcoTable.java | 11 +- .../kelondro/table/FixedWidthArray.java | 3 +- .../de/anomic/kelondro/table/FlexTable.java | 3 +- .../anomic/kelondro/table/FlexWidthArray.java | 7 +- .../de/anomic/kelondro/table/SplitTable.java | 3 +- source/de/anomic/kelondro/table/Stack.java | 5 +- source/de/anomic/kelondro/table/Tree.java | 11 +- .../text/BufferedIndexCollection.java | 10 +- .../de/anomic/kelondro/text/IODispatcher.java | 15 +- source/de/anomic/kelondro/text/IndexCell.java | 10 + .../anomic/kelondro/text/IndexCollection.java | 6 +- .../text/IndexCollectionMigration.java | 331 ++++++++++++++++++ .../text/ReferenceContainerArray.java | 7 +- .../text/ReferenceContainerCache.java | 3 +- source/de/anomic/kelondro/util/FileUtils.java | 51 ++- source/de/anomic/kelondro/util/XMLTables.java | 2 +- .../anomic/plasma/parser/bzip/bzipParser.java | 3 +- .../anomic/plasma/parser/gzip/gzipParser.java | 3 +- .../parser/mimeType/mimeTypeParser.java | 2 +- .../plasma/parser/mimeType/odtDetector.java | 2 +- .../anomic/plasma/parser/odt/odtParser.java | 4 +- .../anomic/plasma/parser/pdf/pdfParser.java | 2 +- .../de/anomic/plasma/parser/ps/psParser.java | 4 +- .../anomic/plasma/parser/rpm/rpmParser.java | 2 +- .../anomic/plasma/parser/tar/tarParser.java | 4 +- .../anomic/plasma/parser/zip/zipParser.java | 4 +- source/de/anomic/plasma/plasmaHTCache.java | 4 +- .../anomic/plasma/plasmaParserDocument.java | 2 +- .../anomic/plasma/plasmaRankingCRProcess.java | 6 +- .../plasma/plasmaRankingDistribution.java | 2 +- source/de/anomic/plasma/plasmaWordIndex.java | 65 ++-- source/de/anomic/server/serverSystem.java | 2 +- source/de/anomic/tools/enumerateFiles.java | 4 +- source/de/anomic/yacy/yacyNewsDB.java | 3 +- source/de/anomic/yacy/yacyNewsQueue.java | 3 +- source/de/anomic/yacy/yacySeedDB.java | 7 +- source/de/anomic/yacy/yacyVersion.java | 9 +- 56 files changed, 559 insertions(+), 148 deletions(-) create mode 100644 source/de/anomic/kelondro/text/IndexCollectionMigration.java diff --git a/htroot/ConfigAppearance_p.java b/htroot/ConfigAppearance_p.java index 520f6f4a7..82429f029 100644 --- a/htroot/ConfigAppearance_p.java +++ b/htroot/ConfigAppearance_p.java @@ -81,7 +81,7 @@ public class ConfigAppearance_p { if (post.containsKey("delete_button")) { // delete skin final File skinfile = new File(skinPath, post.get("skin")); - skinfile.delete(); + FileUtils.deletedelete(skinfile); } if (post.containsKey("install_button")) { diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index 734035014..9529079ba 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -71,7 +71,7 @@ public class ConfigLanguage_p { //delete language file }else if(post.containsKey("delete")){ final File langfile= new File(langPath, post.get("language")); - langfile.delete(); + FileUtils.deletedelete(langfile); //load language file from URL } else if (post.containsKey("url")){ diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index b0a2da4fb..0afafd5f9 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -31,6 +31,7 @@ import java.util.Iterator; import java.util.TreeSet; import de.anomic.http.httpRequestHeader; +import de.anomic.kelondro.util.FileUtils; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -77,7 +78,7 @@ public class ConfigUpdate_p { final String release = post.get("releaseinstall", ""); if(release.length() > 0) { try { - new File(sb.releasePath, release).delete(); + FileUtils.deletedelete(new File(sb.releasePath, release)); } catch (final NullPointerException e) { sb.getLog().logSevere("AUTO-UPDATE: could not delete release " + release + ": " + e.getMessage()); } diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index 6c67a1bef..8fd5d9a60 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -38,6 +38,7 @@ import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacyURL; @@ -73,7 +74,7 @@ public class CrawlProfile { public void clear() { // deletes the profile database and creates a new one if (profileTable != null) profileTable.close(); - if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); + FileUtils.deletedelete(profileTableFile); profileTableFile.getParentFile().mkdirs(); BLOB dyn = null; try { diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index ccf9c303c..77b5bb110 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentHashMap; import de.anomic.kelondro.table.FlexWidthArray; import de.anomic.kelondro.text.Document; import de.anomic.kelondro.util.DateFormatter; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaSwitchboard; @@ -77,7 +78,7 @@ public class CrawlQueues { if (errorDBFile.exists()) { // delete the error db to get a fresh each time on startup // this is useful because there is currently no re-use of the data in this table. - if (errorDBFile.isDirectory()) FlexWidthArray.delete(plasmaPath, "urlError2.db"); else errorDBFile.delete(); + if (errorDBFile.isDirectory()) FlexWidthArray.delete(plasmaPath, "urlError2.db"); else FileUtils.deletedelete(errorDBFile); } errorURL = new ZURL(plasmaPath, "urlError3.db", false); delegatedURL = new ZURL(plasmaPath, "urlDelegated3.db", true); diff --git a/source/de/anomic/crawler/NoticeURLImporter.java b/source/de/anomic/crawler/NoticeURLImporter.java index 7bfe433d6..adfe7d2c1 100644 --- a/source/de/anomic/crawler/NoticeURLImporter.java +++ b/source/de/anomic/crawler/NoticeURLImporter.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import de.anomic.kelondro.util.FileUtils; import de.anomic.plasma.plasmaWordIndex; public class NoticeURLImporter extends AbstractImporter implements Importer { @@ -76,7 +77,7 @@ public class NoticeURLImporter extends AbstractImporter implements Importer { try { this.importProfileDB = new CrawlProfile(profileDbFile); } catch (IOException e) { - profileDbFile.delete(); + FileUtils.deletedelete(profileDbFile); try { this.importProfileDB = new CrawlProfile(profileDbFile); } catch (IOException e1) { diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index 516e1b05d..480bfafbc 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -84,7 +84,7 @@ public class RobotsTxt { private void resetDatabase() { // deletes the robots.txt database and creates a new one if (robotsTable != null) robotsTable.close(); - if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); + FileUtils.deletedelete(robotsTableFile); robotsTableFile.getParentFile().mkdirs(); BLOB blob = null; try { diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 656492ba3..f74d141e4 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -38,6 +38,7 @@ import de.anomic.kelondro.index.ObjectIndex; import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.table.EcoTable; import de.anomic.kelondro.table.FlexWidthArray; +import de.anomic.kelondro.util.FileUtils; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacyURL; @@ -66,7 +67,7 @@ public class ZURL { final File f = new File(cachePath, tablename); if (startWithEmptyFile) { if (f.exists()) { - if (f.isDirectory()) FlexWidthArray.delete(cachePath, tablename); else f.delete(); + if (f.isDirectory()) FlexWidthArray.delete(cachePath, tablename); else FileUtils.deletedelete(f); } } this.urlIndex = new EcoTable(f, rowdef, EcoTable.tailCacheDenyUsage, EcoFSBufferSize, 0); diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index df9871894..5cbf00469 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -42,6 +42,7 @@ import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.Digest; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; public final class userDB { @@ -63,7 +64,7 @@ public final class userDB { void resetDatabase() { // deletes the database and creates a new one if (userTable != null) userTable.close(); - if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); + FileUtils.deletedelete(userTableFile); userTableFile.getParentFile().mkdirs(); userTable = new MapView(new BLOBTree(userTableFile, true, true, 256, 512, '_', NaturalOrder.naturalOrder, true, false, false), 10); } diff --git a/source/de/anomic/kelondro/blob/BLOBArray.java b/source/de/anomic/kelondro/blob/BLOBArray.java index 4eb433d6f..9f4e5eef2 100755 --- a/source/de/anomic/kelondro/blob/BLOBArray.java +++ b/source/de/anomic/kelondro/blob/BLOBArray.java @@ -41,6 +41,7 @@ import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.MergeIterator; import de.anomic.kelondro.util.DateFormatter; +import de.anomic.kelondro.util.FileUtils; public class BLOBArray implements BLOB { @@ -172,6 +173,7 @@ public class BLOBArray implements BLOB { } blobItem b = this.blobs.remove(bestIndex); b.blob.close(false); + b.blob = null; return b.location; } @@ -181,6 +183,7 @@ public class BLOBArray implements BLOB { if (smallestFromFirst2 && this.blobs.get(1).location.length() < this.blobs.get(0).location.length()) idx = 1; blobItem b = this.blobs.remove(idx); b.blob.close(false); + b.blob = null; return b.location; } @@ -201,6 +204,7 @@ public class BLOBArray implements BLOB { } b = this.blobs.remove(bestIndex); b.blob.close(false); + b.blob = null; return b.location; } @@ -245,7 +249,8 @@ public class BLOBArray implements BLOB { // too old blobItem oldestBLOB = blobs.remove(0); oldestBLOB.blob.close(false); - if (!oldestBLOB.location.delete()) oldestBLOB.location.deleteOnExit(); + oldestBLOB.blob = null; + FileUtils.deletedelete(oldestBLOB.location); } // size limit @@ -253,7 +258,7 @@ public class BLOBArray implements BLOB { // too large blobItem oldestBLOB = blobs.remove(0); oldestBLOB.blob.close(false); - if (!oldestBLOB.location.delete()) oldestBLOB.location.deleteOnExit(); + FileUtils.deletedelete(oldestBLOB.location); } } @@ -303,7 +308,7 @@ public class BLOBArray implements BLOB { for (blobItem bi: blobs) { bi.blob.clear(); bi.blob.close(false); - bi.location.delete(); + FileUtils.deletedelete(bi.location); } blobs.clear(); } diff --git a/source/de/anomic/kelondro/blob/BLOBHeap.java b/source/de/anomic/kelondro/blob/BLOBHeap.java index 3179a37a7..8faa25a92 100755 --- a/source/de/anomic/kelondro/blob/BLOBHeap.java +++ b/source/de/anomic/kelondro/blob/BLOBHeap.java @@ -241,6 +241,7 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB { } this.buffer = null; super.close(); + assert file == null; } /** diff --git a/source/de/anomic/kelondro/blob/BLOBHeapModifier.java b/source/de/anomic/kelondro/blob/BLOBHeapModifier.java index ef1fc2982..56ad8c047 100644 --- a/source/de/anomic/kelondro/blob/BLOBHeapModifier.java +++ b/source/de/anomic/kelondro/blob/BLOBHeapModifier.java @@ -32,6 +32,7 @@ import java.util.SortedMap; import de.anomic.kelondro.io.CachedRandomAccess; import de.anomic.kelondro.order.ByteOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.Log; @@ -90,12 +91,9 @@ public class BLOBHeapModifier extends HeapReader implements BLOB { public synchronized void clear() throws IOException { this.index.clear(); this.free.clear(); - try { - this.file.close(); - } catch (final IOException e) { - e.printStackTrace(); - } - this.heapFile.delete(); + this.file.close(); + this.file = null; + FileUtils.deletedelete(this.heapFile); this.file = new CachedRandomAccess(heapFile); } @@ -105,11 +103,7 @@ public class BLOBHeapModifier extends HeapReader implements BLOB { public synchronized void close(boolean writeIDX) { shrinkWithGapsAtEnd(); if (file != null) { - try { - file.close(); - } catch (final IOException e) { - e.printStackTrace(); - } + file.close(); } file = null; diff --git a/source/de/anomic/kelondro/blob/BLOBTree.java b/source/de/anomic/kelondro/blob/BLOBTree.java index d9bee0e91..8ddea1ada 100644 --- a/source/de/anomic/kelondro/blob/BLOBTree.java +++ b/source/de/anomic/kelondro/blob/BLOBTree.java @@ -54,6 +54,7 @@ import de.anomic.kelondro.table.EcoTable; import de.anomic.kelondro.table.FlexTable; import de.anomic.kelondro.table.FlexWidthArray; import de.anomic.kelondro.table.Tree; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; public class BLOBTree implements BLOB { @@ -86,7 +87,7 @@ public class BLOBTree implements BLOB { } catch (final IOException e) { e.printStackTrace(); if (resetOnFail) { - file.delete(); + FileUtils.deletedelete(file); try { fbi = new Tree(file, useNodeCache, -1, rowdef, 1, 8); } catch (final IOException e1) { @@ -145,8 +146,7 @@ public class BLOBTree implements BLOB { public static final void delete(final File file) { if (file.isFile()) { - file.delete(); - if (file.exists()) file.deleteOnExit(); + FileUtils.deletedelete(file); } else { FlexWidthArray.delete(file.getParentFile(), file.getName()); } diff --git a/source/de/anomic/kelondro/blob/HeapReader.java b/source/de/anomic/kelondro/blob/HeapReader.java index d3f054d4b..e0305fb71 100644 --- a/source/de/anomic/kelondro/blob/HeapReader.java +++ b/source/de/anomic/kelondro/blob/HeapReader.java @@ -39,13 +39,14 @@ import de.anomic.kelondro.io.CachedRandomAccess; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.RotateIterator; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.Log; public class HeapReader { protected int keylength; // the length of the primary key - protected LongHandleIndex index; // key/seek relation for used records + protected LongHandleIndex index; // key/seek relation for used records protected Gap free; // set of {seek, size} pairs denoting space and position of free records protected final File heapFile; // the file of the heap protected final ByteOrder ordering; // the ordering on keys @@ -112,7 +113,7 @@ public class HeapReader { return false; } // an index file is a one-time throw-away object, so just delete it now - fif.delete(); + FileUtils.deletedelete(fif); // read the gap file: try { @@ -122,7 +123,7 @@ public class HeapReader { return false; } // same with gap file - fgf.delete(); + FileUtils.deletedelete(fgf); // everything is fine now return this.index.size() > 0; @@ -277,13 +278,8 @@ public class HeapReader { * close the BLOB table */ public synchronized void close() { - if (file != null) try { - file.close(); - } catch (final IOException e) { - e.printStackTrace(); - } + if (file != null) file.close(); file = null; - free.clear(); free = null; index.close(); diff --git a/source/de/anomic/kelondro/blob/HeapWriter.java b/source/de/anomic/kelondro/blob/HeapWriter.java index 1f3f11607..a244ea811 100644 --- a/source/de/anomic/kelondro/blob/HeapWriter.java +++ b/source/de/anomic/kelondro/blob/HeapWriter.java @@ -33,6 +33,7 @@ import java.io.IOException; import de.anomic.kelondro.index.LongHandleIndex; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.Digest; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; public final class HeapWriter { @@ -118,7 +119,7 @@ public final class HeapWriter { String n = f.getName(); String[] l = d.list(); for (int i = 0; i < l.length; i++) { - if (l[i].startsWith(n) && (l[i].endsWith(".idx") || l[i].endsWith(".gap"))) new File(d, l[i]).delete(); + if (l[i].startsWith(n) && (l[i].endsWith(".idx") || l[i].endsWith(".gap"))) FileUtils.deletedelete(new File(d, l[i])); } } diff --git a/source/de/anomic/kelondro/blob/MapView.java b/source/de/anomic/kelondro/blob/MapView.java index 65ac30835..093e6aefa 100644 --- a/source/de/anomic/kelondro/blob/MapView.java +++ b/source/de/anomic/kelondro/blob/MapView.java @@ -40,6 +40,7 @@ import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.RotateIterator; import de.anomic.kelondro.util.DateFormatter; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.ScoreCluster; import de.anomic.kelondro.util.kelondroException; @@ -339,7 +340,7 @@ public class MapView { public static void main(String[] args) { // test the class File f = new File("maptest"); - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); try { // make a blob BLOB blob = new BLOBHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024); diff --git a/source/de/anomic/kelondro/io/CachedRandomAccess.java b/source/de/anomic/kelondro/io/CachedRandomAccess.java index c55e72a7f..a10af403d 100644 --- a/source/de/anomic/kelondro/io/CachedRandomAccess.java +++ b/source/de/anomic/kelondro/io/CachedRandomAccess.java @@ -125,8 +125,12 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra RAFile.seek(pos); } - public synchronized void close() throws IOException { - if (RAFile != null) RAFile.close(); + public synchronized void close() { + if (RAFile != null) try { + RAFile.close(); + } catch (IOException e) { + e.printStackTrace(); + } this.cache = null; this.RAFile = null; } diff --git a/source/de/anomic/kelondro/io/EcoFS.java b/source/de/anomic/kelondro/io/EcoFS.java index cdfa49384..33a65c82f 100644 --- a/source/de/anomic/kelondro/io/EcoFS.java +++ b/source/de/anomic/kelondro/io/EcoFS.java @@ -30,6 +30,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; /** @@ -573,7 +574,7 @@ public class EcoFS { public static void main(final String[] args) { // open a file, add one entry and exit final File f = new File(args[0]); - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); try { final EcoFS t = new EcoFS(f, 8); final byte[] b = new byte[8]; diff --git a/source/de/anomic/kelondro/table/AbstractRecords.java b/source/de/anomic/kelondro/table/AbstractRecords.java index b1cce1a5f..8ec80d14d 100644 --- a/source/de/anomic/kelondro/table/AbstractRecords.java +++ b/source/de/anomic/kelondro/table/AbstractRecords.java @@ -49,6 +49,7 @@ import de.anomic.kelondro.io.RandomAccessIOChunks; import de.anomic.kelondro.io.RandomAccessRecords; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.Log; @@ -440,7 +441,7 @@ public abstract class AbstractRecords implements RandomAccessRecords { final File f = ra.file(); assert f != null; this.entryFile.close(); - f.delete(); + FileUtils.deletedelete(f); ra = (useChannel) ? new ChannelRandomAccess(f) : new FileRandomAccess(f); initNewFile(ra, this.HANDLES.length, this.TXTPROPS.length); } diff --git a/source/de/anomic/kelondro/table/EcoTable.java b/source/de/anomic/kelondro/table/EcoTable.java index cec6b03e9..bacdab27c 100644 --- a/source/de/anomic/kelondro/table/EcoTable.java +++ b/source/de/anomic/kelondro/table/EcoTable.java @@ -49,6 +49,7 @@ import de.anomic.kelondro.io.BufferedEcoFS; import de.anomic.kelondro.io.EcoFS; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.Log; @@ -416,11 +417,11 @@ public class EcoTable implements ObjectIndex { } public synchronized void put(final Entry row) throws IOException { - assert file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); - assert ((table == null) || (table.size() == index.size())); + assert file == null || file.size() == index.size() + fail : "file.size() = " + file.size() + ", index.size() = " + index.size(); + assert table == null || table.size() == index.size(); assert row != null; assert row.bytes() != null; - if ((row == null) || (row.bytes() == null)) return; + if (file == null || table == null || row == null || row.bytes() == null) return; final int i = index.get(row.getPrimaryKeyBytes()); if (i == -1) { addUnique(row); @@ -562,7 +563,7 @@ public class EcoTable implements ObjectIndex { public void clear() throws IOException { final File f = file.filename(); file.close(); - f.delete(); + FileUtils.deletedelete(f); // make new file FileOutputStream fos = null; @@ -694,7 +695,7 @@ public class EcoTable implements ObjectIndex { } public static ObjectIndex testTable(final File f, final String testentities, final int testcase) throws IOException { - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); final Row rowdef = new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder); final ObjectIndex tt = new EcoTable(f, rowdef, testcase, 100, 0); byte[] b; diff --git a/source/de/anomic/kelondro/table/FixedWidthArray.java b/source/de/anomic/kelondro/table/FixedWidthArray.java index 104672b46..4fac433c1 100644 --- a/source/de/anomic/kelondro/table/FixedWidthArray.java +++ b/source/de/anomic/kelondro/table/FixedWidthArray.java @@ -37,6 +37,7 @@ import de.anomic.kelondro.index.ObjectArray; import de.anomic.kelondro.index.Row; import de.anomic.kelondro.io.RandomAccessInterface; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; public class FixedWidthArray extends FullRecords implements ObjectArray { @@ -76,7 +77,7 @@ public class FixedWidthArray extends FullRecords implements ObjectArray { try { return new FixedWidthArray(file, rowdef, intprops); } catch (final IOException e) { - file.delete(); + FileUtils.deletedelete(file); try { return new FixedWidthArray(file, rowdef, intprops); } catch (final IOException ee) { diff --git a/source/de/anomic/kelondro/table/FlexTable.java b/source/de/anomic/kelondro/table/FlexTable.java index 502ba4b21..92017f895 100644 --- a/source/de/anomic/kelondro/table/FlexTable.java +++ b/source/de/anomic/kelondro/table/FlexTable.java @@ -42,6 +42,7 @@ import de.anomic.kelondro.index.RowSet; import de.anomic.kelondro.index.ObjectIndex; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.Log; @@ -88,7 +89,7 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex { if (indexfile.exists()) { // delete existing index file System.out.println("*** Delete File index " + indexfile); - indexfile.delete(); + FileUtils.deletedelete(indexfile); } // fill the index diff --git a/source/de/anomic/kelondro/table/FlexWidthArray.java b/source/de/anomic/kelondro/table/FlexWidthArray.java index 65e636897..1f78eed87 100644 --- a/source/de/anomic/kelondro/table/FlexWidthArray.java +++ b/source/de/anomic/kelondro/table/FlexWidthArray.java @@ -36,6 +36,7 @@ import de.anomic.kelondro.index.ObjectArray; import de.anomic.kelondro.index.Column; import de.anomic.kelondro.index.Row; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.Log; @@ -174,16 +175,16 @@ public class FlexWidthArray implements ObjectArray { final File tabledir = new File(path, tablename); if (!(tabledir.exists())) return; if ((!(tabledir.isDirectory()))) { - tabledir.delete(); + FileUtils.deletedelete(tabledir); return; } final String[] files = tabledir.list(); for (int i = 0; i < files.length; i++) { - new File(tabledir, files[i]).delete(); + FileUtils.deletedelete(new File(tabledir, files[i])); } - tabledir.delete(); + FileUtils.deletedelete(tabledir); } public void reset() throws IOException { diff --git a/source/de/anomic/kelondro/table/SplitTable.java b/source/de/anomic/kelondro/table/SplitTable.java index ab53c6ed7..68afa85c9 100644 --- a/source/de/anomic/kelondro/table/SplitTable.java +++ b/source/de/anomic/kelondro/table/SplitTable.java @@ -57,6 +57,7 @@ import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.order.MergeIterator; import de.anomic.kelondro.order.Order; import de.anomic.kelondro.order.StackIterator; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; import de.anomic.kelondro.util.NamePrefixThreadFactory; @@ -166,7 +167,7 @@ public class SplitTable implements ObjectIndex { for (int i = 0; i < l.length; i++) { if (l[i].startsWith(tablename)) { final File f = new File(path, l[i]); - if (f.isDirectory()) FlexWidthArray.delete(path, l[i]); else f.delete(); + if (f.isDirectory()) FlexWidthArray.delete(path, l[i]); else FileUtils.deletedelete(f); } } init(true); diff --git a/source/de/anomic/kelondro/table/Stack.java b/source/de/anomic/kelondro/table/Stack.java index 8d3a0725d..0a4836730 100644 --- a/source/de/anomic/kelondro/table/Stack.java +++ b/source/de/anomic/kelondro/table/Stack.java @@ -36,6 +36,7 @@ import java.util.StringTokenizer; import de.anomic.kelondro.index.Row; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; public final class Stack extends FullRecords { @@ -66,7 +67,7 @@ public final class Stack extends FullRecords { try { return new Stack(file, rowdef); } catch (final IOException e) { - file.delete(); + FileUtils.deletedelete(file); try { return new Stack(file, rowdef); } catch (final IOException ee) { @@ -358,7 +359,7 @@ public final class Stack extends FullRecords { if (args[0].equals("-c")) { // create final File f = new File(args[3]); - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); final Stack fm = new Stack(f, lens); fm.close(); } else if (args[0].equals("-p")) { diff --git a/source/de/anomic/kelondro/table/Tree.java b/source/de/anomic/kelondro/table/Tree.java index 28afcd33c..af3d63ce9 100644 --- a/source/de/anomic/kelondro/table/Tree.java +++ b/source/de/anomic/kelondro/table/Tree.java @@ -51,6 +51,7 @@ import de.anomic.kelondro.index.Row.Entry; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.NaturalOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.Log; @@ -109,7 +110,7 @@ public class Tree extends CachedRecords implements ObjectIndex { try { return new Tree(file, useNodeCache, preloadTime, rowdef, txtProps, txtPropsWidth); } catch (final IOException e) { - file.delete(); + FileUtils.deletedelete(file); try { return new Tree(file, useNodeCache, preloadTime, rowdef, txtProps, txtPropsWidth); } catch (final IOException ee) { @@ -1354,7 +1355,7 @@ public class Tree extends CachedRecords implements ObjectIndex { if (args[0].equals("-c")) { // create final File f = new File(args[3]); - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); final Row lens = new Row("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]), NaturalOrder.naturalOrder); final Tree fm = new Tree(f, true, 10, lens); fm.close(); @@ -1417,7 +1418,7 @@ public class Tree extends CachedRecords implements ObjectIndex { try { int steps = 0; while (true) { - if (testFile.exists()) testFile.delete(); + if (testFile.exists()) FileUtils.deletedelete(testFile); tt = new Tree(testFile, true, 10, new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder)); steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11); t = s; @@ -1482,7 +1483,7 @@ public class Tree extends CachedRecords implements ObjectIndex { public static void smalltest() { final File f = new File("test.db"); - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); try { final Tree tt = new Tree(f, true, 10, new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder)); byte[] b; @@ -1548,7 +1549,7 @@ public class Tree extends CachedRecords implements ObjectIndex { */ public static Tree testTree(final File f, final String testentities) throws IOException { - if (f.exists()) f.delete(); + if (f.exists()) FileUtils.deletedelete(f); final Tree tt = new Tree(f, false, 10, new Row("byte[] a-4, byte[] b-4", NaturalOrder.naturalOrder)); byte[] b; for (int i = 0; i < testentities.length(); i++) { diff --git a/source/de/anomic/kelondro/text/BufferedIndexCollection.java b/source/de/anomic/kelondro/text/BufferedIndexCollection.java index 3e7bd3f1e..bc2ca4484 100644 --- a/source/de/anomic/kelondro/text/BufferedIndexCollection.java +++ b/source/de/anomic/kelondro/text/BufferedIndexCollection.java @@ -45,6 +45,7 @@ import de.anomic.kelondro.text.IndexCollection; import de.anomic.kelondro.text.ReferenceContainer; import de.anomic.kelondro.text.ReferenceContainerOrder; import de.anomic.kelondro.text.ReferenceRow; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.MemoryControl; import de.anomic.kelondro.util.Log; import de.anomic.server.serverProfiling; @@ -78,7 +79,7 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme for (ReferenceContainer c: dhtInCache) { this.buffer.add(c); } - new File(textindexcache, "index.dhtin.blob").delete(); + FileUtils.deletedelete(new File(textindexcache, "index.dhtin.blob")); } else { // read in new BLOB this.buffer = new IndexBuffer(textindexcache, wordOrdering, payloadrow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log); @@ -328,16 +329,11 @@ public final class BufferedIndexCollection extends AbstractBufferedIndex impleme public CloneableIterator references(String startWordHash, boolean rot) { final Order containerOrder = new ReferenceContainerOrder(this.buffer.ordering().clone()); return new MergeIterator( - this.buffer.references(startWordHash, rot), - new MergeIterator( this.buffer.references(startWordHash, false), this.collections.references(startWordHash, false), containerOrder, ReferenceContainer.containerMergeMethod, - true), - containerOrder, - ReferenceContainer.containerMergeMethod, - true); + true); } } diff --git a/source/de/anomic/kelondro/text/IODispatcher.java b/source/de/anomic/kelondro/text/IODispatcher.java index 122367cc4..0d7e4f9b5 100644 --- a/source/de/anomic/kelondro/text/IODispatcher.java +++ b/source/de/anomic/kelondro/text/IODispatcher.java @@ -34,6 +34,7 @@ import de.anomic.kelondro.index.Row; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.text.ReferenceContainerCache.blobFileEntries; +import de.anomic.kelondro.util.FileUtils; /** * merger class for files from ReferenceContainerArray. @@ -173,7 +174,7 @@ public class IODispatcher extends Thread { public void dump() { try { cache.dump(file, true); - array.mountBLOBContainer(file); + array.mountBLOBFile(file); } catch (IOException e) { e.printStackTrace(); } @@ -223,16 +224,16 @@ public class IODispatcher extends Thread { CloneableIterator i2 = new blobFileEntries(f2, payloadrow); if (!i1.hasNext()) { if (i2.hasNext()) { - if (!f1.delete()) f1.deleteOnExit(); + FileUtils.deletedelete(f1); if (f2.renameTo(newFile)) return newFile; return f2; } else { - if (!f1.delete()) f1.deleteOnExit(); - if (!f2.delete()) f2.deleteOnExit(); + FileUtils.deletedelete(f1); + FileUtils.deletedelete(f2); return null; } } else if (!i2.hasNext()) { - if (!f2.delete()) f2.deleteOnExit(); + FileUtils.deletedelete(f2); if (f1.renameTo(newFile)) return newFile; return f1; } @@ -242,8 +243,8 @@ public class IODispatcher extends Thread { merge(i1, i2, array.ordering(), writer); writer.close(true); // we don't need the old files any more - if (!f1.delete()) f1.deleteOnExit(); - if (!f2.delete()) f2.deleteOnExit(); + FileUtils.deletedelete(f1); + FileUtils.deletedelete(f2); return newFile; } diff --git a/source/de/anomic/kelondro/text/IndexCell.java b/source/de/anomic/kelondro/text/IndexCell.java index b8fdc2e7a..3e9c2f9d7 100644 --- a/source/de/anomic/kelondro/text/IndexCell.java +++ b/source/de/anomic/kelondro/text/IndexCell.java @@ -287,6 +287,16 @@ public final class IndexCell extends AbstractBufferedIndex implements BufferedIn } } + public File newContainerBLOBFile() { + // for migration of cache files + return this.array.newContainerBLOBFile(); + } + + public void mountBLOBFile(File blobFile) throws IOException { + // for migration of cache files + this.array.mountBLOBFile(blobFile); + } + public void cleanupBuffer(int time) { // do nothing } diff --git a/source/de/anomic/kelondro/text/IndexCollection.java b/source/de/anomic/kelondro/text/IndexCollection.java index 57252018b..6321141c3 100644 --- a/source/de/anomic/kelondro/text/IndexCollection.java +++ b/source/de/anomic/kelondro/text/IndexCollection.java @@ -106,7 +106,7 @@ public class IndexCollection extends AbstractIndex implements Index { File cop = new File(path, filenameStub + "." + fillZ(Integer.toHexString(payloadrow.objectsize).toUpperCase(), 4) + ".commons"); this.commonsPath = (useCommons) ? cop : null; if (this.commonsPath == null) { - FileUtils.deleteDirectory(cop); + FileUtils.deletedelete(cop); } else { this.commonsPath.mkdirs(); } @@ -220,7 +220,7 @@ public class IndexCollection extends AbstractIndex implements Index { return null; } } - + public boolean remove(final String wordHash, final String urlHash) { final HashSet hs = new HashSet(); hs.add(urlHash); @@ -1041,8 +1041,10 @@ public class IndexCollection extends AbstractIndex implements Index { public synchronized void close() { this.index.close(); + this.index = null; final Iterator i = arrays.values().iterator(); while (i.hasNext()) i.next().close(); + this.arrays = null; } public static void main(final String[] args) { diff --git a/source/de/anomic/kelondro/text/IndexCollectionMigration.java b/source/de/anomic/kelondro/text/IndexCollectionMigration.java new file mode 100644 index 000000000..172bad75d --- /dev/null +++ b/source/de/anomic/kelondro/text/IndexCollectionMigration.java @@ -0,0 +1,331 @@ +// IndexCollectionMigration.java +// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 30.03.2009 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2009-03-13 11:34:51 +0100 (Fr, 13 Mrz 2009) $ +// $LastChangedRevision: 5709 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro.text; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import de.anomic.kelondro.index.Row; +import de.anomic.kelondro.order.Base64Order; +import de.anomic.kelondro.order.ByteOrder; +import de.anomic.kelondro.order.CloneableIterator; +import de.anomic.kelondro.order.MergeIterator; +import de.anomic.kelondro.order.Order; +import de.anomic.kelondro.order.RotateIterator; +import de.anomic.kelondro.text.Index; +import de.anomic.kelondro.text.IndexCollection; +import de.anomic.kelondro.text.ReferenceContainer; +import de.anomic.kelondro.text.ReferenceContainerOrder; +import de.anomic.kelondro.text.ReferenceRow; +import de.anomic.kelondro.util.FileUtils; +import de.anomic.kelondro.util.Log; + +public final class IndexCollectionMigration extends AbstractBufferedIndex implements Index, BufferedIndex { + + private final IndexCell cell; + private IndexCollection collections; + private final IODispatcher merger; + + public IndexCollectionMigration ( + final File indexPrimaryTextLocation, + final ByteOrder wordOrdering, + final Row payloadrow, + final int entityCacheMaxSize, + final int maxCellArrayFiles, + final IODispatcher merger, + final Log log) throws IOException { + + this.merger = merger; + final File celldir = new File(indexPrimaryTextLocation, "RICELL"); + this.cell = new IndexCell( + celldir, + wordOrdering, + ReferenceRow.urlEntryRow, + entityCacheMaxSize, maxCellArrayFiles, this.merger); + final File textindexcache = new File(indexPrimaryTextLocation, "RICACHE"); + if (textindexcache.exists()) { + // migrate the "index.dhtout.blob" into RICELL directory + File f = new File(textindexcache, "index.dhtout.blob"); + if (f.exists()) { + File n = this.cell.newContainerBLOBFile(); + f.renameTo(n); + this.cell.mountBLOBFile(n); + } + f = new File(textindexcache, "index.dhtin.blob"); + if (f.exists()) { + File n = this.cell.newContainerBLOBFile(); + f.renameTo(n); + this.cell.mountBLOBFile(n); + } + // delete everything else + String[] l = textindexcache.list(); + for (String s: l) { + f = new File(textindexcache, s); + FileUtils.deletedelete(f); + } + FileUtils.deletedelete(textindexcache); + } + + // open collections, this is for migration only. + final File textindexcollections = new File(indexPrimaryTextLocation, "RICOLLECTION"); + if (textindexcollections.exists()) { + this.collections = new IndexCollection( + textindexcollections, + "collection", + 12, + Base64Order.enhancedCoder, + BufferedIndexCollection.maxCollectionPartition, + ReferenceRow.urlEntryRow, + false); + if (this.collections.size() == 0) { + // delete everything here + this.collections.close(); + this.collections = null; + String[] l = textindexcollections.list(); + File f; + for (String s: l) { + f = new File(textindexcollections, s); + FileUtils.deletedelete(f); + } + FileUtils.deletedelete(textindexcollections); + } + } else { + this.collections = null; + } + } + + /* methods for interface Index */ + + public void add(final ReferenceContainer entries) throws IOException { + assert (entries.row().objectsize == ReferenceRow.urlEntryRow.objectsize); + + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(entries.getWordHash()); + if (e != null) { + e.merge(entries); + cell.add(e); + } else { + cell.add(entries); + } + } else { + cell.add(entries); + } + } + + public void add(final String wordHash, final ReferenceRow entry) throws IOException { + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(wordHash); + if (e != null) { + e.add(entry); + cell.add(e); + } else { + cell.add(wordHash, entry); + } + } else { + cell.add(wordHash, entry); + } + } + + public boolean has(final String wordHash) { + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(wordHash); + if (e != null) { + try { + cell.add(e); + } catch (IOException e1) { + e1.printStackTrace(); + } + return true; + } else { + return cell.has(wordHash); + } + } else { + return cell.has(wordHash); + } + } + + public int count(String wordHash) { + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(wordHash); + if (e != null) { + try { + cell.add(e); + } catch (IOException e1) { + e1.printStackTrace(); + } + return cell.count(wordHash); + } else { + return cell.count(wordHash); + } + } else { + return cell.count(wordHash); + } + } + + public ReferenceContainer get(final String wordHash, final Set urlselection) throws IOException { + if (wordHash == null) { + // wrong input + return null; + } + + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(wordHash); + if (e != null) cell.add(e); + } + + return this.cell.get(wordHash, urlselection); + } + + public ReferenceContainer delete(final String wordHash) throws IOException { + final ReferenceContainer c = new ReferenceContainer( + wordHash, + ReferenceRow.urlEntryRow, + cell.count(wordHash)); + c.addAllUnique(cell.delete(wordHash)); + if (this.collections != null) c.merge(collections.delete(wordHash)); + return c; + } + + public boolean remove(final String wordHash, final String urlHash) throws IOException { + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(wordHash); + if (e != null) cell.add(e); + } + return cell.remove(wordHash, urlHash); + } + + public int remove(final String wordHash, final Set urlHashes) throws IOException { + if (this.collections != null) { + ReferenceContainer e = this.collections.delete(wordHash); + if (e != null) cell.add(e); + } + return cell.remove(wordHash, urlHashes); + } + + public synchronized CloneableIterator references(final String startHash, final boolean rot, final boolean ram) throws IOException { + final CloneableIterator i = wordContainers(startHash, ram); + if (rot) { + return new RotateIterator(i, new String(Base64Order.zero(startHash.length())), cell.size() + ((ram) ? 0 : collections.size())); + } + return i; + } + + private synchronized CloneableIterator wordContainers(final String startWordHash, final boolean ram) throws IOException { + final Order containerOrder = new ReferenceContainerOrder(cell.ordering().clone()); + containerOrder.rotate(ReferenceContainer.emptyContainer(startWordHash, 0)); + if (ram) { + return cell.references(startWordHash, true); + } + if (collections == null) return cell.references(startWordHash, false); + return new MergeIterator( + cell.references(startWordHash, false), + collections.references(startWordHash, false), + containerOrder, + ReferenceContainer.containerMergeMethod, + true); + } + + public void clear() { + try { + cell.clear(); + } catch (IOException e1) { + e1.printStackTrace(); + } + if (collections != null) try { + collections.clear(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void close() { + cell.close(); + if (collections != null) collections.close(); + } + + public int size() { + return (collections == null) ? cell.size() : java.lang.Math.max(collections.size(), cell.size()); + } + + public int minMem() { + return 1024*1024 /* indexing overhead */ + cell.minMem() + ((collections == null) ? 0 : collections.minMem()); + } + + + /* + * methods for cache management + */ + + public int getBufferMaxReferences() { + return cell.getBufferMaxReferences(); + } + + public long getBufferMinAge() { + return cell.getBufferMinAge(); + } + + public long getBufferMaxAge() { + return cell.getBufferMaxAge(); + } + + public long getBufferSizeBytes() { + return cell.getBufferSizeBytes(); + } + + public void setBufferMaxWordCount(final int maxWords) { + cell.setBufferMaxWordCount(maxWords); + } + + public int getBackendSize() { + return (collections == null) ? cell.getBackendSize() : collections.size(); + } + + public int getBufferSize() { + return cell.getBufferSize(); + } + + public ByteOrder ordering() { + return cell.ordering(); + } + + public CloneableIterator references(String startWordHash, boolean rot) { + final Order containerOrder = new ReferenceContainerOrder(this.cell.ordering().clone()); + if (this.collections == null) return this.cell.references(startWordHash, rot); + //else + return new MergeIterator( + this.cell.references(startWordHash, false), + this.collections.references(startWordHash, false), + containerOrder, + ReferenceContainer.containerMergeMethod, + true); + } + + public void cleanupBuffer(int time) { + this.cell.cleanupBuffer(time); + } +} diff --git a/source/de/anomic/kelondro/text/ReferenceContainerArray.java b/source/de/anomic/kelondro/text/ReferenceContainerArray.java index 4dff5f9ea..0e27c2576 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerArray.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerArray.java @@ -36,6 +36,7 @@ import de.anomic.kelondro.index.Row; import de.anomic.kelondro.index.RowSet; import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.order.CloneableIterator; +import de.anomic.kelondro.util.FileUtils; public final class ReferenceContainerArray { @@ -88,7 +89,7 @@ public final class ReferenceContainerArray { return this.array.newBLOB(new Date()); } - public void mountBLOBContainer(File location) throws IOException { + public void mountBLOBFile(File location) throws IOException { this.array.mountBLOB(location); } @@ -247,6 +248,10 @@ public final class ReferenceContainerArray { if (this.array.entries() < 2) return false; if (this.merger.queueLength() > 0) return false; File f1 = this.array.unmountOldestBLOB(similar); + if (f1.length() == 0) { + FileUtils.deletedelete(f1); + return true; + } File f2 = (similar) ? this.array.unmountSimilarSizeBLOB(f1.length()) : this.array.unmountOldestBLOB(false); merger.merge(f1, f2, this.array, this.payloadrow, newContainerBLOBFile()); return true; diff --git a/source/de/anomic/kelondro/text/ReferenceContainerCache.java b/source/de/anomic/kelondro/text/ReferenceContainerCache.java index b77078d84..78086d7dc 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerCache.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerCache.java @@ -41,6 +41,7 @@ import de.anomic.kelondro.blob.HeapWriter; import de.anomic.kelondro.order.CloneableIterator; import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.order.ByteOrder; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.Log; import de.anomic.kelondro.index.Row; import de.anomic.kelondro.index.RowSet; @@ -112,7 +113,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde public void dump(final File heapFile, boolean writeIDX) throws IOException { assert this.cache != null; Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + cache.size() + " rwi's"); - if (heapFile.exists()) heapFile.delete(); + if (heapFile.exists()) FileUtils.deletedelete(heapFile); HeapWriter dump = new HeapWriter(heapFile, payloadrow.primaryKeyLength, Base64Order.enhancedCoder); final long startTime = System.currentTimeMillis(); long wordcount = 0, urlcount = 0; diff --git a/source/de/anomic/kelondro/util/FileUtils.java b/source/de/anomic/kelondro/util/FileUtils.java index 4cd9da7cb..be9c3dc69 100644 --- a/source/de/anomic/kelondro/util/FileUtils.java +++ b/source/de/anomic/kelondro/util/FileUtils.java @@ -584,8 +584,7 @@ public final class FileUtils { if(!(to.delete() && from.renameTo(to))) { // do it manually copy(from, to); - if(!from.delete()) - from.deleteOnExit(); + FileUtils.deletedelete(from); } } @@ -718,23 +717,41 @@ public final class FileUtils { } /** - * delete a directory - * if the directory is not empty, delete also everything inside - * @param directory + * delete files and directories + * if a directory is not empty, delete also everything inside + * because deletion sometimes fails on windows, there is also a windows exec included + * @param path */ - public static void deleteDirectory(final File directory) { - final String[] list = directory.list(); - if (list != null) { - File object; - for (int i = list.length - 1; i >= 0; i--) { - object = new File(directory, list[i]); - if (object.isFile()) { - object.delete(); - } else { - deleteDirectory(object); - } + public static void deletedelete(final File path) { + if (!path.exists()) return; + + // empty the directory first + if (path.isDirectory()) { + final String[] list = path.list(); + if (list != null) { + for (String s: list) deletedelete(new File(path, s)); + } + } + + if (!path.delete()) path.deleteOnExit(); + if (path.exists()) { + if (System.getProperties().getProperty("os.name","").toLowerCase().startsWith("windows")) { + // deleting files on windows sometimes does not work with java + try { + String p = path.getCanonicalPath(); + String command = "cmd /C del /F /Q " + p; + Process r = Runtime.getRuntime().exec(command); + if (r == null) { + Log.logSevere("FileUtils", "cannot execute command: " + command); + } else { + byte[] response = read(r.getInputStream()); + Log.logInfo("FileUtils", "deletedelete: " + new String(response)); + } + if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p); + } catch (IOException e) { + e.printStackTrace(); + } } } - directory.delete(); } } diff --git a/source/de/anomic/kelondro/util/XMLTables.java b/source/de/anomic/kelondro/util/XMLTables.java index 40b9e2b37..100b81d32 100644 --- a/source/de/anomic/kelondro/util/XMLTables.java +++ b/source/de/anomic/kelondro/util/XMLTables.java @@ -88,7 +88,7 @@ public class XMLTables { xmlenc.close(); // delete old file and rename tmp-file to old file's name - this.propFile.delete(); + FileUtils.deletedelete(this.propFile); tmpFile.renameTo(this.propFile); // set the new time stamp diff --git a/source/de/anomic/plasma/parser/bzip/bzipParser.java b/source/de/anomic/plasma/parser/bzip/bzipParser.java index 88d657cdf..73b899218 100644 --- a/source/de/anomic/plasma/parser/bzip/bzipParser.java +++ b/source/de/anomic/plasma/parser/bzip/bzipParser.java @@ -31,6 +31,7 @@ import java.util.Hashtable; import org.apache.tools.bzip2.CBZip2InputStream; +import de.anomic.kelondro.util.FileUtils; import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.parser.AbstractParser; @@ -115,7 +116,7 @@ public class bzipParser extends AbstractParser implements Parser { throw new ParserException("Unexpected error while parsing bzip file. " + e.getMessage(),location); } finally { - if (tempFile != null) tempFile.delete(); + if (tempFile != null) FileUtils.deletedelete(tempFile); } } diff --git a/source/de/anomic/plasma/parser/gzip/gzipParser.java b/source/de/anomic/plasma/parser/gzip/gzipParser.java index 9fe2fef6d..b91a460f2 100644 --- a/source/de/anomic/plasma/parser/gzip/gzipParser.java +++ b/source/de/anomic/plasma/parser/gzip/gzipParser.java @@ -30,6 +30,7 @@ import java.io.InputStream; import java.util.Hashtable; import java.util.zip.GZIPInputStream; +import de.anomic.kelondro.util.FileUtils; import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.parser.AbstractParser; @@ -98,7 +99,7 @@ public class gzipParser extends AbstractParser implements Parser { throw new ParserException("Unexpected error while parsing gzip file. " + e.getMessage(),location); } finally { - if (tempFile != null) tempFile.delete(); + if (tempFile != null) FileUtils.deletedelete(tempFile); } } diff --git a/source/de/anomic/plasma/parser/mimeType/mimeTypeParser.java b/source/de/anomic/plasma/parser/mimeType/mimeTypeParser.java index bfabde8fc..73b1b9abc 100644 --- a/source/de/anomic/plasma/parser/mimeType/mimeTypeParser.java +++ b/source/de/anomic/plasma/parser/mimeType/mimeTypeParser.java @@ -178,7 +178,7 @@ public class mimeTypeParser extends AbstractParser implements Parser { } catch (final IOException e) { throw new ParserException("Unexpected error while detect mimetype of resource. " + e.getMessage(),location); } finally { - if (dstFile != null) {dstFile.delete();} + if (dstFile != null) FileUtils.deletedelete(dstFile); } } diff --git a/source/de/anomic/plasma/parser/mimeType/odtDetector.java b/source/de/anomic/plasma/parser/mimeType/odtDetector.java index c010f5a7a..1b8a6f661 100644 --- a/source/de/anomic/plasma/parser/mimeType/odtDetector.java +++ b/source/de/anomic/plasma/parser/mimeType/odtDetector.java @@ -66,7 +66,7 @@ public class odtDetector implements MagicDetector { } catch (final IOException e) { return null; } finally { - if (dstFile != null) {dstFile.delete();} + if (dstFile != null) FileUtils.deletedelete(dstFile); } } diff --git a/source/de/anomic/plasma/parser/odt/odtParser.java b/source/de/anomic/plasma/parser/odt/odtParser.java index cf8df5006..03f13dc0d 100644 --- a/source/de/anomic/plasma/parser/odt/odtParser.java +++ b/source/de/anomic/plasma/parser/odt/odtParser.java @@ -197,7 +197,7 @@ public class odtParser extends AbstractParser implements Parser { if (writer != null) try { writer.close(); } catch (final Exception ex) {/* ignore this */} // delete the file - if (writerFile != null) try { writerFile.delete(); } catch (final Exception ex) {/* ignore this */} + if (writerFile != null) FileUtils.deletedelete(writerFile); throw new ParserException("Unexpected error while parsing odt file. " + e.getMessage(),location); } @@ -221,7 +221,7 @@ public class odtParser extends AbstractParser implements Parser { throw new ParserException("Unexpected error while parsing odt file. " + e.getMessage(),location); } finally { - if (dest != null) try { dest.delete(); } catch (final Exception e){/* ignore this */} + if (dest != null) FileUtils.deletedelete(dest); } } diff --git a/source/de/anomic/plasma/parser/pdf/pdfParser.java b/source/de/anomic/plasma/parser/pdf/pdfParser.java index 490f74b1d..5566443cc 100644 --- a/source/de/anomic/plasma/parser/pdf/pdfParser.java +++ b/source/de/anomic/plasma/parser/pdf/pdfParser.java @@ -177,7 +177,7 @@ public class pdfParser extends AbstractParser implements Parser { if (writer != null) try { writer.close(); } catch (final Exception ex) {/* ignore this */} // delete the file - if (writerFile != null) try { writerFile.delete(); } catch (final Exception ex) {/* ignore this */} + if (writerFile != null) FileUtils.deletedelete(writerFile); e.printStackTrace(); throw new ParserException("Unexpected error while parsing pdf file. " + e.getMessage(),location); diff --git a/source/de/anomic/plasma/parser/ps/psParser.java b/source/de/anomic/plasma/parser/ps/psParser.java index d6f4f0362..39fc80d2b 100644 --- a/source/de/anomic/plasma/parser/ps/psParser.java +++ b/source/de/anomic/plasma/parser/ps/psParser.java @@ -131,7 +131,7 @@ public class psParser extends AbstractParser implements Parser { if (e instanceof ParserException) throw (ParserException) e; // delete temp file - if (outputFile != null) outputFile.delete(); + if (outputFile != null) FileUtils.deletedelete(outputFile); // throw exception throw new ParserException("Unexpected error while parsing ps file. " + e.getMessage(),location); @@ -285,7 +285,7 @@ public class psParser extends AbstractParser implements Parser { throw new ParserException("Unable to parse the ps file. " + e.getMessage(),location, e); } finally { - if (tempFile != null) try{ tempFile.delete(); }catch(final Exception e) {/* */} + if (tempFile != null) FileUtils.deletedelete(tempFile); } } diff --git a/source/de/anomic/plasma/parser/rpm/rpmParser.java b/source/de/anomic/plasma/parser/rpm/rpmParser.java index 542a76cf8..401a67ae6 100644 --- a/source/de/anomic/plasma/parser/rpm/rpmParser.java +++ b/source/de/anomic/plasma/parser/rpm/rpmParser.java @@ -85,7 +85,7 @@ public class rpmParser extends AbstractParser implements Parser { } catch (final Exception e) { return null; } finally { - if (dstFile != null) {dstFile.delete();} + if (dstFile != null) FileUtils.deletedelete(dstFile); } } diff --git a/source/de/anomic/plasma/parser/tar/tarParser.java b/source/de/anomic/plasma/parser/tar/tarParser.java index e56e1688c..96df00b32 100644 --- a/source/de/anomic/plasma/parser/tar/tarParser.java +++ b/source/de/anomic/plasma/parser/tar/tarParser.java @@ -152,7 +152,7 @@ public class tarParser extends AbstractParser implements Parser { } catch (final ParserException e) { this.theLogger.logInfo("Unable to parse tar file entry '" + entryName + "'. " + e.getMessage()); } finally { - if (subDocTempFile != null) try {subDocTempFile.delete(); } catch(final Exception ex){/* ignore this */} + if (subDocTempFile != null) FileUtils.deletedelete(subDocTempFile); } if (subDoc == null) continue; @@ -224,7 +224,7 @@ public class tarParser extends AbstractParser implements Parser { if (docText != null) try { docText.close(); } catch (final Exception ex) {/* ignore this */} // delete the file - if (outputFile != null) try { outputFile.delete(); } catch (final Exception ex) {/* ignore this */} + if (outputFile != null) FileUtils.deletedelete(outputFile); throw new ParserException("Unexpected error while parsing tar resource. " + e.getMessage(),location); } diff --git a/source/de/anomic/plasma/parser/zip/zipParser.java b/source/de/anomic/plasma/parser/zip/zipParser.java index e0e8f9c2c..72b530f79 100644 --- a/source/de/anomic/plasma/parser/zip/zipParser.java +++ b/source/de/anomic/plasma/parser/zip/zipParser.java @@ -135,7 +135,7 @@ public class zipParser extends AbstractParser implements Parser { } catch (final ParserException e) { this.theLogger.logInfo("Unable to parse zip file entry '" + entryName + "'. " + e.getMessage()); } finally { - if (subDocTempFile != null) try {subDocTempFile.delete(); } catch(final Exception ex){/* ignore this */} + if (subDocTempFile != null) FileUtils.deletedelete(subDocTempFile); } if (subDoc == null) continue; @@ -208,7 +208,7 @@ public class zipParser extends AbstractParser implements Parser { if (docText != null) try { docText.close(); } catch (final Exception ex) {/* ignore this */} // delete the file - if (outputFile != null) try { outputFile.delete(); } catch (final Exception ex) {/* ignore this */} + if (outputFile != null) FileUtils.deletedelete(outputFile); throw new ParserException("Unexpected error while parsing zip resource. " + e.getClass().getName() + ": "+ e.getMessage(),location); } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 292bc3bef..6bae99bb9 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -98,7 +98,7 @@ public final class plasmaHTCache { object.getName().equals("yacy") || object.getName().equals("https") || object.getName().equals("ftp")) { - FileUtils.deleteDirectory(cachePath); + FileUtils.deletedelete(cachePath); } } } @@ -121,7 +121,7 @@ public final class plasmaHTCache { log.logFine("reset responseHeader DB with "+ responseHeaderDB.size() +" entries"); if (responseHeaderDB != null) responseHeaderDB.close(); final File dbfile = new File(cachePath, RESPONSE_HEADER_DB_NAME); - if (dbfile.exists()) dbfile.delete(); + if (dbfile.exists()) FileUtils.deletedelete(dbfile); openDB(); } diff --git a/source/de/anomic/plasma/plasmaParserDocument.java b/source/de/anomic/plasma/plasmaParserDocument.java index 4db737a1c..8b0dacb9c 100644 --- a/source/de/anomic/plasma/plasmaParserDocument.java +++ b/source/de/anomic/plasma/plasmaParserDocument.java @@ -470,7 +470,7 @@ dc_rights // delete the temp file if ((this.text != null) && (this.text instanceof File)) { try { - ((File)this.text).delete(); + FileUtils.deletedelete((File) this.text); } catch (final Exception e) { /* ignore this */ } finally { diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index a9648b052..509d29fc9 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -298,7 +298,7 @@ public class plasmaRankingCRProcess { // store the file acc.toFile(tmp_file); // since this was successful, we remove the old file and move the new file to it - to_file.delete(); + FileUtils.deletedelete(to_file); tmp_file.renameTo(to_file); } FileUtils.moveAll(tmp_dir, bkp_dir); @@ -313,7 +313,7 @@ public class plasmaRankingCRProcess { public static int genrci(File cr_in, final File rci_out) throws IOException { if (!(cr_in.exists())) return 0; AttrSeq cr = new AttrSeq(cr_in, false); - //if (rci_out.exists()) rci_out.delete(); // we want only fresh rci here (during testing) + //if (rci_out.exists()) FileUtils.deletedelete(rci_out); // we want only fresh rci here (during testing) if (!(rci_out.exists())) { final AttrSeq rcix = new AttrSeq("Global Ranking Reverse Citation Index", ",'='," + @@ -478,7 +478,7 @@ public class plasmaRankingCRProcess { } catch (final IOException e) { // there is something wrong with this file; delete it System.out.println("file " + f.getName() + " is corrupted and deleted"); - f.delete(); + FileUtils.deletedelete(f); } } final long seconds = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000); diff --git a/source/de/anomic/plasma/plasmaRankingDistribution.java b/source/de/anomic/plasma/plasmaRankingDistribution.java index 759311205..30e6458b2 100644 --- a/source/de/anomic/plasma/plasmaRankingDistribution.java +++ b/source/de/anomic/plasma/plasmaRankingDistribution.java @@ -179,7 +179,7 @@ public final class plasmaRankingDistribution { result = yacyClient.transfer(address, crfile.getName(), b); if (result == null) { log.logInfo("RankingDistribution - transmitted file " + crfile + " to " + address + " successfully in " + ((System.currentTimeMillis() - starttime) / 1000) + " seconds"); - crfile.delete(); // the file is not needed any more locally + FileUtils.deletedelete(crfile); // the file is not needed any more locally } else { log.logInfo("RankingDistribution - error transmitting file " + crfile + " to " + address + ": " + result); } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index d0745fb2e..cf66657df 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -45,6 +45,7 @@ import de.anomic.kelondro.order.ByteOrder; import de.anomic.kelondro.text.BufferedIndex; import de.anomic.kelondro.text.BufferedIndexCollection; import de.anomic.kelondro.text.IndexCell; +import de.anomic.kelondro.text.IndexCollectionMigration; import de.anomic.kelondro.text.MetadataRowContainer; import de.anomic.kelondro.text.ReferenceContainer; import de.anomic.kelondro.text.IODispatcher; @@ -52,6 +53,7 @@ import de.anomic.kelondro.text.ReferenceRow; import de.anomic.kelondro.text.MetadataRepository; import de.anomic.kelondro.text.Word; import de.anomic.kelondro.text.Blacklist; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; import de.anomic.kelondro.util.Log; import de.anomic.tools.iso639; @@ -63,10 +65,11 @@ import de.anomic.yacy.yacyURL; public final class plasmaWordIndex { // environment constants - public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes - public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash - public static final int lowcachedivisor = 900; - public static final int maxCollectionPartition = 7; // should be 7 + public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes + public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash + public static final int lowcachedivisor = 900; + public static final int maxCollectionPartition = 7; // should be 7 + public static final int maxCellArrayFiles = 10; public static final String CRAWL_PROFILE_PROXY = "proxy"; public static final String CRAWL_PROFILE_REMOTE = "remote"; @@ -126,7 +129,7 @@ public final class plasmaWordIndex { assert !indexPrimaryTextLocation.exists(); indexPrimaryTextLocation.mkdirs(); if (oldPrimaryTextLocation.renameTo(indexPrimaryTextLocation)) { - if (!oldPrimaryPath.delete()) oldPrimaryPath.deleteOnExit(); + FileUtils.deletedelete(oldPrimaryPath); } else { indexPrimaryTextLocation = oldPrimaryTextLocation; // emergency case: stay with old directory } @@ -134,20 +137,38 @@ public final class plasmaWordIndex { } this.merger = (useCell) ? new IODispatcher(1, 1) : null; if (this.merger != null) this.merger.start(); - this.index = (useCell) ? - new IndexCell( - new File(indexPrimaryTextLocation, "RICELL"), - wordOrder, - ReferenceRow.urlEntryRow, - entityCacheMaxSize, 10, this.merger) : - new BufferedIndexCollection( - indexPrimaryTextLocation, - wordOrder, - ReferenceRow.urlEntryRow, - entityCacheMaxSize, - useCommons, - redundancy, - log); + + // check if the peer has migrated the index + if (new File(indexPrimaryTextLocation, "RICOLLECTION").exists()) { + this.index = (useCell) ? + new IndexCollectionMigration( + indexPrimaryTextLocation, + wordOrder, + ReferenceRow.urlEntryRow, + entityCacheMaxSize, + maxCellArrayFiles, + this.merger, + log) + : + new BufferedIndexCollection( + indexPrimaryTextLocation, + wordOrder, + ReferenceRow.urlEntryRow, + entityCacheMaxSize, + useCommons, + redundancy, + log); + } else { + this.index = new IndexCell( + new File(indexPrimaryTextLocation, "RICELL"), + wordOrder, + ReferenceRow.urlEntryRow, + entityCacheMaxSize, + maxCellArrayFiles, + this.merger); + } + + // migrate LURL-db files into new subdirectory METADATA File textdir = new File(this.secondaryRoot, "TEXT"); @@ -173,7 +194,7 @@ public final class plasmaWordIndex { try { this.profilesActiveCrawls = new CrawlProfile(profilesActiveFile); } catch (IOException e) { - profilesActiveFile.delete(); + FileUtils.deletedelete(profilesActiveFile); try { this.profilesActiveCrawls = new CrawlProfile(profilesActiveFile); } catch (IOException e1) { @@ -194,7 +215,7 @@ public final class plasmaWordIndex { try { this.profilesPassiveCrawls = new CrawlProfile(profilesPassiveFile); } catch (IOException e) { - profilesPassiveFile.delete(); + FileUtils.deletedelete(profilesPassiveFile); try { this.profilesPassiveCrawls = new CrawlProfile(profilesPassiveFile); } catch (IOException e1) { @@ -323,7 +344,7 @@ public final class plasmaWordIndex { private void resetProfiles() { final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES); - if (pdb.exists()) pdb.delete(); + if (pdb.exists()) FileUtils.deletedelete(pdb); try { profilesActiveCrawls = new CrawlProfile(pdb); } catch (IOException e) { diff --git a/source/de/anomic/server/serverSystem.java b/source/de/anomic/server/serverSystem.java index 961868884..2576f5e65 100644 --- a/source/de/anomic/server/serverSystem.java +++ b/source/de/anomic/server/serverSystem.java @@ -387,7 +387,7 @@ public final class serverSystem { } catch (final InterruptedException e) { throw new IOException(e.getMessage()); } - starterFile.delete(); + FileUtils.deletedelete(starterFile); } public static Vector execSynchronous(final String command) throws IOException { diff --git a/source/de/anomic/tools/enumerateFiles.java b/source/de/anomic/tools/enumerateFiles.java index 08251b907..77f106e20 100644 --- a/source/de/anomic/tools/enumerateFiles.java +++ b/source/de/anomic/tools/enumerateFiles.java @@ -26,6 +26,8 @@ import java.util.ArrayList; import java.util.Enumeration; import java.util.TreeSet; +import de.anomic.kelondro.util.FileUtils; + public class enumerateFiles implements Enumeration { // implements iterative search through recursively defined subdirectories @@ -83,7 +85,7 @@ public class enumerateFiles implements Enumeration { } else { if (l.length == 0) { if (delete_emptyFolders) { - f.delete(); + FileUtils.deletedelete(f); f = null; } else { if (!(return_folders)) f = null; diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 056e23b27..03d003abe 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -54,6 +54,7 @@ import de.anomic.kelondro.index.ObjectIndex; import de.anomic.kelondro.order.Base64Order; import de.anomic.kelondro.table.EcoTable; import de.anomic.kelondro.util.DateFormatter; +import de.anomic.kelondro.util.FileUtils; import de.anomic.kelondro.util.kelondroException; import de.anomic.server.serverCodings; @@ -70,7 +71,7 @@ public class yacyNewsDB { private void resetDB() { try {close();} catch (final Exception e) {} - if (path.exists()) path.delete(); + if (path.exists()) FileUtils.deletedelete(path); this.news = new EcoTable(path, yacyNewsRecord.rowdef, EcoTable.tailCacheUsageAuto, 10, 0); } diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 04955b51c..df2f1b107 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -55,6 +55,7 @@ import de.anomic.kelondro.index.Row; import de.anomic.kelondro.order.NaturalOrder; import de.anomic.kelondro.table.Stack; import de.anomic.kelondro.util.DateFormatter; +import de.anomic.kelondro.util.FileUtils; public class yacyNewsQueue { @@ -77,7 +78,7 @@ public class yacyNewsQueue { private void resetDB() { try {close();} catch (final Exception e) {} - if (path.exists()) path.delete(); + if (path.exists()) FileUtils.deletedelete(path); queueStack = Stack.open(path, rowdef); } diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index fe0919627..a2c792635 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -267,7 +267,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { return new MapDataMining(new BLOBHeap(seedDBFile, commonHashLength, Base64Order.enhancedCoder, 1024 * 512), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); } catch (final Exception e) { // try again - seedDBFile.delete(); + FileUtils.deletedelete(seedDBFile); try { return new MapDataMining(new BLOBHeap(seedDBFile, commonHashLength, Base64Order.enhancedCoder, 1024 * 512), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this); } catch (IOException e1) { @@ -283,7 +283,8 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { // seed.db is detected yacyCore.log.logWarning("seed-db " + seedDBFile.toString() + " reset (on-the-fly)"); seedDB.close(); - if(!seedDBFile.delete()) + FileUtils.deletedelete(seedDBFile); + if (seedDBFile.exists()) Log.logWarning("yacySeedDB", "could not delete file "+ seedDBFile); // create new seed database seedDB = openSeedTable(seedDBFile); @@ -810,7 +811,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { } finally { if (seedFile != null) try { - seedFile.delete(); + FileUtils.deletedelete(seedFile); } catch (final Exception e) { /* ignore this */ } diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index 455652538..524c2fa53 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -369,9 +369,11 @@ public final class yacyVersion implements Comparator, Comparable, Comparable deleteAfterMillis) { // delete file - if (!downloadedFile.delete()) { + FileUtils.deletedelete(downloadedFile); + if (downloadedFile.exists()) { Log.logWarning("STARTUP", "cannot delete old release " + downloadedFile.getAbsolutePath()); } }