boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for http://www.yacy-forum.de/viewtopic.php?p=28544#28544

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3056 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent bf0d820659
commit 93a5ace330
3 changed files with 21 additions and 27 deletions
Show Stats Download Patch File Download Diff File

11
source/de/anomic/plasma/dbImport/AssortmentImporter.java
Unescape View File

@ -1,7 +1,6 @@
package de.anomic.plasma.dbImport;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import de.anomic.index.indexContainer;
@ -59,12 +58,7 @@ public class AssortmentImporter extends AbstractImporter implements dbImporter{
// initializing the import assortment db
this.log.logInfo("Initializing source assortment file " + theImportAssortmentFile);
try {
this.assortmentFile = new plasmaWordIndexAssortment(importAssortmentPath, assortmentNr, this.cacheSize/1024, preloadTime, this.log);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
this.assortmentFile = new plasmaWordIndexAssortment(importAssortmentPath, assortmentNr, this.cacheSize/1024, preloadTime, this.log);
this.importStartSize = this.assortmentFile.size();
}
@ -92,7 +86,8 @@ public class AssortmentImporter extends AbstractImporter implements dbImporter{
public void run() {
try {
// getting a content interator
Iterator contentIterator = this.assortmentFile.wordContainers(null, true, false);
Iterator contentIterator = this.assortmentFile.wordContainers();
this.log.logFine("Started import of file " + this.assortmentFile.getName());
while (contentIterator.hasNext()) {
this.wordEntityCount++;

33
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -60,7 +60,6 @@ import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntryNew;
import de.anomic.index.indexRWIEntryOld;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroColumn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroRow;
@ -76,7 +75,7 @@ public final class plasmaWordIndexAssortment {
// class variables
private File assortmentFile;
private serverLog log;
private kelondroCache assortments;
private kelondroTree assortment;
private long bufferSize;
private static String intx(int x) {
@ -99,7 +98,7 @@ public final class plasmaWordIndexAssortment {
return (rowsize - yacySeedDB.commonHashLength - 12) / indexRWIEntryOld.urlEntryRow.objectsize();
}
public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) throws IOException {
public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) {
if (!(storagePath.exists())) storagePath.mkdirs();
this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db");
//this.bufferStructureLength = 3 + 2 * assortmentLength;
@ -107,17 +106,21 @@ public final class plasmaWordIndexAssortment {
this.log = log;
// open assortment tree file
long start = System.currentTimeMillis();
assortments = new kelondroCache(kelondroTree.open(assortmentFile, bufferSize / 2, preloadTime, bufferStructure(assortmentLength)), bufferSize / 2, true, false);
assortment = kelondroTree.open(assortmentFile, bufferSize / 2, preloadTime, bufferStructure(assortmentLength));
long stop = System.currentTimeMillis();
if (log != null) log.logConfig("Opened Assortment, " +
assortments.size() + " entries, width " +
assortment.size() + " entries, width " +
assortmentLength + ", " + bufferkb + "kb buffer, " +
preloadTime + " ms preloadTime, " +
(stop - start) + " ms effective, " +
assortments.cacheNodeStatus()[1] + " preloaded");
assortment.cacheNodeStatus()[1] + " preloaded");
}
public String getName() {
return this.assortmentFile.toString();
}
public final indexContainer row2container(kelondroRow.Entry row) {
if (row == null) return null;
String wordHash = row.getColString(0, null);
@ -131,12 +134,12 @@ public final class plasmaWordIndexAssortment {
return container;
}
public Iterator wordContainers(String startWordHash, boolean up, boolean rot) throws IOException {
public Iterator wordContainers() {
// returns an iteration of indexContainer elements
try {
return new containerIterator(startWordHash, up, rot);
return new containerIterator();
} catch (kelondroException e) {
log.logSevere("iterateAssortment/kelondro-error: " + e.getMessage() + " - reset assortment-DB " + assortmentFile, e);
log.logSevere("iterateAssortment/kelondro-error: " + e.getMessage(), e);
return null;
}
}
@ -145,8 +148,8 @@ public final class plasmaWordIndexAssortment {
private Iterator rowIterator;
public containerIterator(String startWordHash, boolean up, boolean rot) throws IOException {
rowIterator = assortments.rows(up, rot, (startWordHash == null) ? null : startWordHash.getBytes());
public containerIterator() {
rowIterator = assortment.contentRows(-1);
}
public boolean hasNext() {
@ -165,16 +168,12 @@ public final class plasmaWordIndexAssortment {
}
public int size() {
try {
return assortments.size();
} catch (IOException e) {
return 0;
}
return assortment.size();
}
public void close() {
try {
assortments.close();
assortment.close();
} catch (IOException e){
log.logSevere("unable to close assortment database: " + e.getMessage(), e);
}

4
source/yacy.java
Unescape View File

@ -1273,7 +1273,7 @@ public final class yacy {
serverLog log = new serverLog("HASHLIST");
File homeDBroot = new File(new File(homePath), "DATA/PLASMADB");
File indexRoot = new File(new File(homePath), "DATA/INDEX");
String wordChunkStartHash = "------------";
String wordChunkStartHash = "AAAAAAAAAAAA";
try {serverLog.configureLogging(new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
log.logInfo("STARTING CREATION OF RWI-HASHLIST");
File root = new File(homePath);
@ -1285,7 +1285,7 @@ public final class yacy {
} else if (resource.startsWith("assortment")) {
int a = Integer.parseInt(resource.substring(10));
plasmaWordIndexAssortment assortment = new plasmaWordIndexAssortment(new File(homeDBroot, "ACLUSTER"), a, 8*1024*1024, 3000, null);
indexContainerIterator = assortment.wordContainers(wordChunkStartHash, true, false);
indexContainerIterator = assortment.wordContainers();
} else if (resource.equals("words")) {
plasmaWordIndexFileCluster fileDB = new plasmaWordIndexFileCluster(homeDBroot);
indexContainerIterator = fileDB.wordContainers(wordChunkStartHash, false);

Write Preview
Loading…
Cancel
Save