From 25192e0d369eca26708aac65eef49489740ee296 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 24 May 2008 12:30:50 +0000 Subject: [PATCH] added a deletion button to indexControlRWIs that deletes the complete web index git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4847 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/IndexControlRWIs_p.html | 10 ++++++++++ htroot/IndexControlRWIs_p.java | 8 +++++++- source/de/anomic/crawler/CrawlProfile.java | 16 ++++++++-------- source/de/anomic/index/indexCollectionRI.java | 8 ++++++++ source/de/anomic/index/indexContainerHeap.java | 6 ++++++ source/de/anomic/index/indexRAMRI.java | 18 ++++++++++++++++++ source/de/anomic/index/indexRI.java | 1 + .../anomic/index/indexRepositoryReference.java | 5 +++++ .../kelondro/kelondroAbstractRecords.java | 2 +- .../anomic/kelondro/kelondroBytesLongMap.java | 4 ++++ source/de/anomic/kelondro/kelondroCache.java | 4 ++-- .../kelondro/kelondroCollectionIndex.java | 7 +++++++ source/de/anomic/kelondro/kelondroDyn.java | 4 ++-- .../de/anomic/kelondro/kelondroEcoTable.java | 2 +- .../de/anomic/kelondro/kelondroFlexTable.java | 2 +- source/de/anomic/kelondro/kelondroIndex.java | 2 +- .../anomic/kelondro/kelondroMScoreCluster.java | 7 +++++++ .../de/anomic/kelondro/kelondroMapObjects.java | 4 ++-- source/de/anomic/kelondro/kelondroObjects.java | 4 ++-- .../de/anomic/kelondro/kelondroRAMIndex.java | 2 +- .../de/anomic/kelondro/kelondroSQLTable.java | 2 +- .../de/anomic/kelondro/kelondroSplitTable.java | 2 +- source/de/anomic/kelondro/kelondroTree.java | 4 ++-- .../LanguageStatisticsHolder.java | 2 ++ source/de/anomic/plasma/plasmaWordIndex.java | 15 +++++++++++++-- source/de/anomic/yacy/yacySeedDB.java | 2 +- 26 files changed, 114 insertions(+), 29 deletions(-) diff --git a/htroot/IndexControlRWIs_p.html b/htroot/IndexControlRWIs_p.html index 845735c4e..8ea175c90 100644 --- a/htroot/IndexControlRWIs_p.html +++ b/htroot/IndexControlRWIs_p.html @@ -24,6 +24,16 @@ + + +
+
Cleanup +
+
Index Deletion
(deletes all words and all urls)
+
+
+
+
#(searchresult)#:: diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index a274bb6a2..b93629f20 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -104,7 +104,13 @@ public class IndexControlRWIs_p { prop.put("searchresult_wordhash", keyhash); } } - + + // delete everything + if (post.containsKey("deletecomplete")) { + sb.webIndex.clear(); + post.remove("deletecomplete"); + } + // delete word if (post.containsKey("keyhashdeleteall")) { if (delurl || delurlref) { diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index a23bba5eb..52fa0c161 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -72,7 +72,7 @@ public class CrawlProfile { profileTable = new kelondroMapObjects(dyn, 500); } - public void resetDatabase() { + public void clear() { // deletes the profile database and creates a new one if (profileTable != null) profileTable.close(); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); @@ -110,7 +110,7 @@ public class CrawlProfile { try { return handleIterator.hasNext(); } catch (kelondroException e) { - resetDatabase(); + clear(); return false; } } @@ -119,7 +119,7 @@ public class CrawlProfile { lastkey = (String) handleIterator.next(); return getEntry(lastkey); } catch (kelondroException e) { - resetDatabase(); + clear(); return null; } } @@ -127,7 +127,7 @@ public class CrawlProfile { if (lastkey != null) try { removeEntry(lastkey); } catch (kelondroException e) { - resetDatabase(); + clear(); } } } @@ -143,7 +143,7 @@ public class CrawlProfile { try { profileTable.set(ne.handle(), ne.map()); } catch (kelondroException e) { - resetDatabase(); + clear(); try { profileTable.set(ne.handle(), ne.map()); } catch (IOException ee) { @@ -151,7 +151,7 @@ public class CrawlProfile { System.exit(0); } } catch (IOException e) { - resetDatabase(); + clear(); try { profileTable.set(ne.handle(), ne.map()); } catch (IOException ee) { @@ -182,7 +182,7 @@ public class CrawlProfile { try { profileTable.set(ne.handle(), ne.map()); } catch (kelondroException e) { - resetDatabase(); + clear(); try { profileTable.set(ne.handle(), ne.map()); } catch (IOException ee) { @@ -190,7 +190,7 @@ public class CrawlProfile { System.exit(0); } } catch (IOException e) { - resetDatabase(); + clear(); try { profileTable.set(ne.handle(), ne.map()); } catch (IOException ee) { diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java index db3b64822..5bbb82f57 100644 --- a/source/de/anomic/index/indexCollectionRI.java +++ b/source/de/anomic/index/indexCollectionRI.java @@ -59,6 +59,14 @@ public class indexCollectionRI implements indexRI { } } + public void clear() { + try { + collectionIndex.clear(); + } catch (IOException e) { + e.printStackTrace(); + } + } + public long getUpdateTime(String wordHash) { indexContainer entries = getContainer(wordHash, null); if (entries == null) return 0; diff --git a/source/de/anomic/index/indexContainerHeap.java b/source/de/anomic/index/indexContainerHeap.java index 0d21558b3..64890a4f4 100755 --- a/source/de/anomic/index/indexContainerHeap.java +++ b/source/de/anomic/index/indexContainerHeap.java @@ -88,6 +88,12 @@ public final class indexContainerHeap { this.readOnlyMode = false; } + public void clear() throws IOException { + if (index != null) index.clear(); + if (cache != null) cache.clear(); + initWriteMode(); + } + /** * initializes the heap in read/write mode without reading of a dump first * another dump reading afterwards is not possible diff --git a/source/de/anomic/index/indexRAMRI.java b/source/de/anomic/index/indexRAMRI.java index 4b63170ff..8d8538cc6 100644 --- a/source/de/anomic/index/indexRAMRI.java +++ b/source/de/anomic/index/indexRAMRI.java @@ -79,6 +79,21 @@ public final class indexRAMRI implements indexRI, indexRIReader { heap.initWriteMode(); } } + + /** + * clear the content + * @throws IOException + */ + public void clear() { + hashScore.clear(); + hashDate.clear(); + initTime = System.currentTimeMillis(); + try { + heap.clear(); + } catch (IOException e) { + e.printStackTrace(); + } + } public int minMem() { // there is no specific large array that needs to be maintained @@ -280,5 +295,8 @@ public final class indexRAMRI implements indexRI, indexRIReader { } catch (IOException e){ log.logSevere("unable to dump cache: " + e.getMessage(), e); } + heap = null; + hashScore.clear(); + hashDate.clear(); } } diff --git a/source/de/anomic/index/indexRI.java b/source/de/anomic/index/indexRI.java index 4a9b7c109..7952efdd8 100644 --- a/source/de/anomic/index/indexRI.java +++ b/source/de/anomic/index/indexRI.java @@ -45,6 +45,7 @@ public interface indexRI { public boolean removeEntry(String wordHash, String urlHash); public int removeEntries(String wordHash, Set urlHashes); public void addEntries(indexContainer newEntries); + public void clear(); public void close(); } diff --git a/source/de/anomic/index/indexRepositoryReference.java b/source/de/anomic/index/indexRepositoryReference.java index de5686ae9..29e905c69 100644 --- a/source/de/anomic/index/indexRepositoryReference.java +++ b/source/de/anomic/index/indexRepositoryReference.java @@ -64,6 +64,11 @@ public final class indexRepositoryReference { urlIndexFile = new kelondroSplitTable(this.location, "urls", indexURLReference.rowdef, false); } + public void clear() throws IOException { + if (exportthread != null) exportthread.interrupt(); + urlIndexFile.clear(); + } + public int size() { return urlIndexFile.size(); } diff --git a/source/de/anomic/kelondro/kelondroAbstractRecords.java b/source/de/anomic/kelondro/kelondroAbstractRecords.java index abffcf60e..3a4c75209 100644 --- a/source/de/anomic/kelondro/kelondroAbstractRecords.java +++ b/source/de/anomic/kelondro/kelondroAbstractRecords.java @@ -403,7 +403,7 @@ public abstract class kelondroAbstractRecords implements kelondroRecords { writeOrderType(); } - public void reset() throws IOException { + public void clear() throws IOException { kelondroRA ra = this.entryFile.getRA(); File f = new File(ra.name()); this.entryFile.close(); diff --git a/source/de/anomic/kelondro/kelondroBytesLongMap.java b/source/de/anomic/kelondro/kelondroBytesLongMap.java index 5bc7f2843..4b671ec2e 100644 --- a/source/de/anomic/kelondro/kelondroBytesLongMap.java +++ b/source/de/anomic/kelondro/kelondroBytesLongMap.java @@ -49,6 +49,10 @@ public class kelondroBytesLongMap { return index.row(); } + public void clear() throws IOException { + index.clear(); + } + public synchronized long getl(byte[] key) throws IOException { assert (key != null); kelondroRow.Entry indexentry = index.get(key); diff --git a/source/de/anomic/kelondro/kelondroCache.java b/source/de/anomic/kelondro/kelondroCache.java index 8967b7995..fb8661f41 100644 --- a/source/de/anomic/kelondro/kelondroCache.java +++ b/source/de/anomic/kelondro/kelondroCache.java @@ -431,8 +431,8 @@ public class kelondroCache implements kelondroIndex { return index.filename(); } - public void reset() throws IOException { - this.index.reset(); + public void clear() throws IOException { + this.index.clear(); init(); } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index bac921118..f9a73bf5b 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -170,6 +170,13 @@ public class kelondroCollectionIndex { } } + public void clear() throws IOException { + index.clear(); + for (kelondroFixedWidthArray array: arrays.values()) { + array.clear(); + } + } + private void openAllArrayFiles(boolean indexGeneration, kelondroByteOrder indexOrder) throws IOException { String[] list = this.path.list(); diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index ca91a55d5..015544bc4 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -123,9 +123,9 @@ public class kelondroDyn { } } - public void reset() throws IOException { + public void clear() throws IOException { String name = this.index.filename(); - this.index.reset(); + this.index.clear(); this.buffer = new kelondroObjectBuffer(name); } diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index 6836f4a96..af354d2c6 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -516,7 +516,7 @@ public class kelondroEcoTable implements kelondroIndex { return lr; } - public void reset() throws IOException { + public void clear() throws IOException { File f = file.filename(); file.close(); f.delete(); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 7385a048e..d4254e53f 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -129,7 +129,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr } } - public void reset() throws IOException { + public void clear() throws IOException { super.reset(); RAMIndex = true; index = new kelondroBytesIntMap(super.row().column(0).cellwidth, super.rowdef.objectOrder, 0); diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java index 37e29e153..4dae69e41 100644 --- a/source/de/anomic/kelondro/kelondroIndex.java +++ b/source/de/anomic/kelondro/kelondroIndex.java @@ -73,6 +73,6 @@ public interface kelondroIndex { public kelondroRow.Entry removeOne() throws IOException; public kelondroCloneableIterator keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key public kelondroCloneableIterator rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row + public void clear() throws IOException; public void close(); - public void reset() throws IOException; } diff --git a/source/de/anomic/kelondro/kelondroMScoreCluster.java b/source/de/anomic/kelondro/kelondroMScoreCluster.java index 4ed56af98..406f3af6b 100644 --- a/source/de/anomic/kelondro/kelondroMScoreCluster.java +++ b/source/de/anomic/kelondro/kelondroMScoreCluster.java @@ -62,6 +62,13 @@ public final class kelondroMScoreCluster { encnt = 0; } + public void clear() { + refkeyDB.clear(); + keyrefDB.clear(); + gcount = 0; + encnt = 0; + } + public static final String shortDateFormatString = "yyyyMMddHHmmss"; public static final SimpleDateFormat shortFormatter = new SimpleDateFormat(shortDateFormatString); public static final long minutemillis = 60000; diff --git a/source/de/anomic/kelondro/kelondroMapObjects.java b/source/de/anomic/kelondro/kelondroMapObjects.java index 650e5b8af..56b799abb 100644 --- a/source/de/anomic/kelondro/kelondroMapObjects.java +++ b/source/de/anomic/kelondro/kelondroMapObjects.java @@ -145,8 +145,8 @@ public class kelondroMapObjects extends kelondroObjects { if (doubleaccfields != null) for (int i = 0; i < doubleaccfields.length; i++) accMap.put(doubleaccfields[i], doubleaccumulator[i]); } - public void reset() throws IOException { - super.reset(); + public void clear() throws IOException { + super.clear(); if (sortfields == null) sortClusterMap = null; else { sortClusterMap = new HashMap>(); for (int i = 0; i < sortfields.length; i++) { diff --git a/source/de/anomic/kelondro/kelondroObjects.java b/source/de/anomic/kelondro/kelondroObjects.java index 4b50ab558..932406d2b 100644 --- a/source/de/anomic/kelondro/kelondroObjects.java +++ b/source/de/anomic/kelondro/kelondroObjects.java @@ -48,8 +48,8 @@ public class kelondroObjects { this.cachesize = cachesize; } - public void reset() throws IOException { - this.dyn.reset(); + public void clear() throws IOException { + this.dyn.clear(); this.cache = new HashMap(); this.cacheScore = new kelondroMScoreCluster(); } diff --git a/source/de/anomic/kelondro/kelondroRAMIndex.java b/source/de/anomic/kelondro/kelondroRAMIndex.java index db8a0eef7..000662e76 100644 --- a/source/de/anomic/kelondro/kelondroRAMIndex.java +++ b/source/de/anomic/kelondro/kelondroRAMIndex.java @@ -43,7 +43,7 @@ public class kelondroRAMIndex implements kelondroIndex { reset(initialspace); } - public void reset() { + public void clear() { reset(0); } diff --git a/source/de/anomic/kelondro/kelondroSQLTable.java b/source/de/anomic/kelondro/kelondroSQLTable.java index 6e0fe784e..dfb028d80 100644 --- a/source/de/anomic/kelondro/kelondroSQLTable.java +++ b/source/de/anomic/kelondro/kelondroSQLTable.java @@ -297,7 +297,7 @@ public class kelondroSQLTable implements kelondroIndex { return new int[]{0,0,0,0,0,0,0,0,0,0}; } - public void reset() { + public void clear() { // TODO Auto-generated method stub } diff --git a/source/de/anomic/kelondro/kelondroSplitTable.java b/source/de/anomic/kelondro/kelondroSplitTable.java index d93929f5d..7e4cb280e 100644 --- a/source/de/anomic/kelondro/kelondroSplitTable.java +++ b/source/de/anomic/kelondro/kelondroSplitTable.java @@ -143,7 +143,7 @@ public class kelondroSplitTable implements kelondroIndex { } } - public void reset() throws IOException { + public void clear() throws IOException { this.close(); String[] l = path.list(); for (int i = 0; i < l.length; i++) { diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 8b636fe01..b91b6c72e 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -159,8 +159,8 @@ public class kelondroTree extends kelondroCachedRecords implements kelondroIndex super.setLogger(log); } - public void reset() throws IOException { - super.reset(); + public void clear() throws IOException { + super.clear(); setHandle(root, null); } diff --git a/source/de/anomic/language/identification/LanguageStatisticsHolder.java b/source/de/anomic/language/identification/LanguageStatisticsHolder.java index d9d9b408d..2f5695ab1 100644 --- a/source/de/anomic/language/identification/LanguageStatisticsHolder.java +++ b/source/de/anomic/language/identification/LanguageStatisticsHolder.java @@ -54,6 +54,8 @@ import java.util.Vector; */ public class LanguageStatisticsHolder extends Vector { + private static final long serialVersionUID = -887517724227204705L; + private String languageDir = "langstats"; // directory that contains language files private static LanguageStatisticsHolder instance; diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 76f9f7c84..d82f0e000 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -190,7 +190,18 @@ public final class plasmaWordIndex implements indexRI { newsPool = new yacyNewsPool(networkRoot); } - + public void clear() { + dhtInCache.clear(); + dhtOutCache.clear(); + collections.clear(); + try { + referenceURL.clear(); + } catch (IOException e) { + e.printStackTrace(); + } + queuePreStack.clear(); + } + private void initActiveCrawlProfiles() { this.defaultProxyProfile = null; this.defaultRemoteProfile = null; @@ -213,7 +224,7 @@ public final class plasmaWordIndex implements indexRI { if (name.equals(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA)) this.defaultMediaSnippetGlobalProfile = profile; } } catch (Exception e) { - this.profilesActiveCrawls.resetDatabase(); + this.profilesActiveCrawls.clear(); this.defaultProxyProfile = null; this.defaultRemoteProfile = null; this.defaultTextSnippetLocalProfile = null; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 53ff67fd8..499424c6a 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -1031,7 +1031,7 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { // eergency reset yacyCore.log.logSevere("seed-db emergency reset", e); try { - database.reset(); + database.clear(); nextSeed = null; return null; } catch (IOException e1) {