You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
270 lines
11 KiB
270 lines
11 KiB
// plasmaWordIndexFileCache.java
|
|
// -----------------------------
|
|
// part of YACY
|
|
// (C) by Michael Peter Christen; mc@anomic.de
|
|
// first published on http://www.anomic.de
|
|
// Frankfurt, Germany, 2004
|
|
// last major change: 22.01.2004
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
//
|
|
// Using this software in any meaning (reading, learning, copying, compiling,
|
|
// running) means that you agree that the Author(s) is (are) not responsible
|
|
// for cost, loss of data or any harm that may be caused directly or indirectly
|
|
// by usage of this softare or this documentation. The usage of this software
|
|
// is on your own risk. The installation and usage (starting/running) of this
|
|
// software may allow other people or application to access your computer and
|
|
// any attached devices and is highly dependent on the configuration of the
|
|
// software which must be done by the user of the software; the author(s) is
|
|
// (are) also not responsible for proper configuration and usage of the
|
|
// software, even if provoked by documentation provided together with
|
|
// the software.
|
|
//
|
|
// Any changes to this file according to the GPL as documented in the file
|
|
// gpl.txt aside this file in the shipment you received can be done to the
|
|
// lines that follows this copyright notice here, but changes must not be
|
|
// done inside the copyright notive above. A re-distribution must contain
|
|
// the intact and unchanged copyright notice.
|
|
// Contributions and changes to the program code must be marked as such.
|
|
|
|
|
|
/*
|
|
The plasmaIndexCache manages a database table with a list of
|
|
indexEntries in it. This is done in a completely different fashion
|
|
as organized by the plasmaIndex tables. The entries are not
|
|
sorted and just stored in a buffer.
|
|
Whenever during a seach an index is retrieved, first it's buffer
|
|
is flushed into the corresponding index table, so that it can be
|
|
sorted into the remaining index entry elements.
|
|
The cache database consist of
|
|
- the word hash as primary key
|
|
- one column with a one-byte counter
|
|
- a number of more columns with indexEntry elements
|
|
*/
|
|
|
|
|
|
// compile with
|
|
// javac -classpath classes -sourcepath source -d classes -g source/de/anomic/plasma/*.java
|
|
|
|
package de.anomic.plasma;
|
|
|
|
import java.io.*;
|
|
import java.util.*;
|
|
import de.anomic.server.*;
|
|
import de.anomic.kelondro.*;
|
|
|
|
public class plasmaWordIndexFileCache {
|
|
|
|
private static final String indexCacheFileName = "indexCache.db";
|
|
private static final int buffers = 50; // number of buffered entries per word
|
|
|
|
// class variables
|
|
private File databaseRoot;
|
|
private kelondroTree indexCache;
|
|
private int bufferkb;
|
|
|
|
public plasmaWordIndexFileCache(File databaseRoot, int bufferkb) throws IOException {
|
|
this.databaseRoot = databaseRoot;
|
|
this.bufferkb = bufferkb;
|
|
File indexCacheFile = new File(databaseRoot, indexCacheFileName);
|
|
if (indexCacheFile.exists()) {
|
|
// simply open the file
|
|
indexCache = new kelondroTree(indexCacheFile, bufferkb * 0x400);
|
|
} else {
|
|
createCacheFile(indexCacheFile);
|
|
}
|
|
}
|
|
|
|
private void resetCacheFile() {
|
|
// this has to be used in emergencies only
|
|
// it can happen that there is a serious db inconsistency; in that case we re-create the indexCache
|
|
try { indexCache.close(); } catch (IOException e) {}
|
|
File indexCacheFile = new File(databaseRoot, indexCacheFileName);
|
|
if (indexCacheFile.exists()) indexCacheFile.delete();
|
|
try {
|
|
createCacheFile(indexCacheFile);
|
|
} catch (IOException e) {
|
|
de.anomic.server.serverLog.logError("PLASMA", "plasmaWordIndexFileCache.resetCacheFile: serious failure creating the cache file: " + e.getMessage());
|
|
indexCache = null;
|
|
}
|
|
}
|
|
|
|
private void createCacheFile(File indexCacheFile) throws IOException {
|
|
// create a new file
|
|
int[] columns = new int[buffers + 2];
|
|
columns[0] = plasmaWordIndexEntry.wordHashLength;
|
|
columns[1] = 1;
|
|
for (int i = 0; i < buffers; i++) columns[i + 2] = plasmaCrawlLURL.urlHashLength + plasmaWordIndexEntry.attrSpaceShort;
|
|
indexCache = new kelondroTree(indexCacheFile, bufferkb * 0x400, columns);
|
|
}
|
|
|
|
protected void close() throws IOException {
|
|
indexCache.close();
|
|
indexCache = null;
|
|
}
|
|
|
|
private byte[][] getCache(String wordHash) throws IOException {
|
|
// read one line from the cache; if none exists: construct one
|
|
byte[][] row;
|
|
try {
|
|
row = indexCache.get(wordHash.getBytes());
|
|
} catch (Exception e) {
|
|
// we had some negativeSeekOffsetExceptions here; in that case the indexCache is corrupt
|
|
System.out.println("Error in plasmaWordINdexFileCache.getCache: index for hash " + wordHash + " is corrupt:" + e.toString());
|
|
e.printStackTrace();
|
|
row = null;
|
|
}
|
|
if (row == null) {
|
|
row = new byte[indexCache.columns()][];
|
|
row[0] = wordHash.getBytes();
|
|
row[1] = new byte[1];
|
|
row[1][0] = (byte) 0;
|
|
}
|
|
return row;
|
|
}
|
|
|
|
|
|
protected Iterator wordHashes(String wordHash, boolean up) throws IOException {
|
|
try {
|
|
return indexCache.rows(up, false, (wordHash == null) ? null : wordHash.getBytes());
|
|
} catch (kelondroException e) {
|
|
de.anomic.server.serverLog.logError("PLASMA", "kelondro error in plasmaWordIndexFileCache: " + e.getMessage() + "; deleting index for " + wordHash);
|
|
deleteComplete(wordHash);
|
|
return new HashSet().iterator();
|
|
}
|
|
}
|
|
|
|
protected plasmaWordIndexEntity getIndex(String wordHash, boolean deleteIfEmpty) throws IOException {
|
|
// first flush the index cache, if there is any for that word hash
|
|
byte[][] row = indexCache.get(wordHash.getBytes());
|
|
if (row != null) {
|
|
int entries = (int) row[1][0];
|
|
if (entries != 0) flushCache(row, null); // if the cache has entries, flush it
|
|
indexCache.remove(wordHash.getBytes()); // delete the cache index row; suppose to be empty now
|
|
}
|
|
// then return the index from the uncached file (with new entries)
|
|
return new plasmaWordIndexEntity(databaseRoot, wordHash, deleteIfEmpty);
|
|
}
|
|
|
|
protected void addEntriesToIndex(String wordHash, Vector /* of plasmaIndexEntry */ newEntries) throws IOException {
|
|
//System.out.println("* adding cached word index: " + wordHash + "=" + word + ":" + entry.toEncodedForm()); // debug
|
|
// fetch the index cache
|
|
if (newEntries.size() == 0) return;
|
|
byte[][] row = getCache(wordHash);
|
|
int entries = (int) row[1][0];
|
|
// check if the index cache is full
|
|
if (entries + 2 + newEntries.size() >= indexCache.columns()) {
|
|
flushCache(row, newEntries); // and put in new values
|
|
entries = 0;
|
|
row[1][0] = (byte) 0; // set number of entries to zero
|
|
} else {
|
|
// put in the new values
|
|
String newEntry;
|
|
for (int i = 0; i < newEntries.size(); i++) {
|
|
newEntry = ((plasmaWordIndexEntry) newEntries.elementAt(i)).getUrlHash() + ((plasmaWordIndexEntry) newEntries.elementAt(i)).toEncodedForm(false);
|
|
row[entries + 2] = newEntry.getBytes();
|
|
entries++;
|
|
}
|
|
row[1][0] = (byte) entries;
|
|
try {
|
|
indexCache.put(row);
|
|
} catch (kelondroException e) {
|
|
// this is a very bad case; a database inconsistency occurred
|
|
serverLog.logError("PLASMA", "fatal error in plasmaWordIndexFileCache.addEntriesToIndex: write of " + wordHash + " to index cache failed - " + e.getMessage() + " - indexCache.db deleted");
|
|
resetCacheFile();
|
|
} catch (IOException e) {
|
|
// this is a very bad case; a database inconsistency occurred
|
|
serverLog.logError("PLASMA", "fatal error in plasmaWordIndexFileCache.addEntriesToIndex: write of " + wordHash + " to index cache failed - " + e.getMessage() + " - indexCache.db deleted");
|
|
resetCacheFile();
|
|
}
|
|
}
|
|
// finished!
|
|
}
|
|
|
|
protected void deleteComplete(String wordHash) throws IOException {
|
|
plasmaWordIndexEntity.removePlasmaIndex(databaseRoot, wordHash);
|
|
indexCache.remove(wordHash.getBytes());
|
|
}
|
|
|
|
protected int removeEntries(String wordHash, String[] urlHashes, boolean deleteComplete) throws IOException {
|
|
// removes all given url hashes from a single word index. Returns number of deletions.
|
|
plasmaWordIndexEntity pi = getIndex(wordHash, true);
|
|
int count = 0;
|
|
for (int i = 0; i < urlHashes.length; i++) if (pi.removeEntry(urlHashes[i], deleteComplete)) count++;
|
|
int size = pi.size();
|
|
pi.close(); pi = null;
|
|
// check if we can remove the index completely
|
|
if ((deleteComplete) && (size == 0)) {
|
|
// remove index
|
|
if (!(plasmaWordIndexEntity.removePlasmaIndex(databaseRoot, wordHash)))
|
|
System.out.println("DEBUG: cannot remove index file for word hash " + wordHash);
|
|
// remove cache
|
|
indexCache.remove(wordHash.getBytes());
|
|
}
|
|
return count;
|
|
}
|
|
|
|
private synchronized void flushCache(byte[][] row, Vector indexEntries) throws IOException {
|
|
String wordHash = new String(row[0]);
|
|
int entries = (int) row[1][0];
|
|
if ((entries == 0) && ((indexEntries == null) || (indexEntries.size() == 0))) return;
|
|
|
|
// open file
|
|
plasmaWordIndexEntity pi = new plasmaWordIndexEntity(databaseRoot, wordHash, false);
|
|
|
|
// write from array
|
|
plasmaWordIndexEntry entry;
|
|
for (int i = 0; i < entries; i++) {
|
|
entry = new plasmaWordIndexEntry(new String(row[i + 2], 0, plasmaCrawlLURL.urlHashLength),
|
|
new String(row[i + 2], plasmaCrawlLURL.urlHashLength, plasmaWordIndexEntry.attrSpaceShort));
|
|
pi.addEntry(entry);
|
|
}
|
|
|
|
// write from vector
|
|
if (indexEntries != null) {
|
|
for (int i = 0; i < indexEntries.size(); i++)
|
|
pi.addEntry((plasmaWordIndexEntry) indexEntries.elementAt(i));
|
|
}
|
|
|
|
// close and return
|
|
pi.close();
|
|
pi = null;
|
|
}
|
|
|
|
private int size(String wordHash) throws IOException {
|
|
// return number of entries in specific cache
|
|
byte[][] row = indexCache.get(wordHash.getBytes());
|
|
if (row == null) return 0;
|
|
return (int) row[1][0];
|
|
}
|
|
|
|
protected int size() {
|
|
if (indexCache == null) return 0; else return indexCache.size();
|
|
}
|
|
|
|
/*
|
|
private plasmaIndex getIndexF(String wordHash) throws IOException {
|
|
return new plasmaIndex(databaseRoot, wordHash);
|
|
}
|
|
|
|
private void addEntryToIndexF(String wordHash, plasmaIndexEntry entry) throws IOException {
|
|
plasmaIndex pi = new plasmaIndex(databaseRoot, wordHash);
|
|
pi.addEntry(entry);
|
|
pi.close();
|
|
}
|
|
*/
|
|
|
|
}
|