boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

finals; cleaned;

Browse Source 
Properties;

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@756 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 19 years ago
parent e748ba3f6e
commit 6d1de8abfd
2 changed files with 157 additions and 152 deletions
Show Stats Download Patch File Download Diff File

99
source/de/anomic/kelondro/kelondroMap.java
Unescape View File

@ -4,7 +4,10 @@
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004
// last major change: 26.10.2004
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -39,7 +42,6 @@
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.kelondro;
import java.io.IOException;
@ -79,14 +81,18 @@ public class kelondroMap {
if (sortfields == null) sortClusterMap = null; else {
sortClusterMap = new HashMap();
cluster = new kelondroMScoreCluster[sortfields.length];
for (int i = 0; i < sortfields.length; i++) cluster[i] = new kelondroMScoreCluster();
for (int i = 0; i < sortfields.length; i++) {
cluster[i] = new kelondroMScoreCluster();
}
}
Long[] accumulator = null;
if (accfields == null) accMap = null; else {
accMap = new HashMap();
accumulator = new Long[accfields.length];
for (int i = 0; i < accfields.length; i++) accumulator[i] = new Long(0);
for (int i = 0; i < accfields.length; i++) {
accumulator[i] = new Long(0);
}
}
// fill cluster and accumulator with values
@ -97,7 +103,7 @@ public class kelondroMap {
Map map;
while (it.hasNext()) {
key = (String) it.next();
//System.out.println("kelondroMap: enumerating key " + key);
// System.out.println("kelondroMap: enumerating key " + key);
map = get(key);
if (sortfields != null) for (int i = 0; i < sortfields.length; i++) {
@ -138,7 +144,7 @@ public class kelondroMap {
public synchronized void set(String key, Map newMap) throws IOException {
// update elementCount
if ((sortfields != null) || (accfields != null)) {
Map oldMap = get(key, false);
final Map oldMap = get(key, false);
if (oldMap == null) {
// new element
elementCount++;
@ -147,7 +153,6 @@ public class kelondroMap {
if (accfields != null) updateAcc(oldMap, false);
}
}
// write entry
writeKra(key, newMap, "");
@ -165,11 +170,11 @@ public class kelondroMap {
if (accfields != null) updateAcc(newMap, true);
}
private synchronized void writeKra(String key, Map newMap, String comment) throws IOException {
private synchronized void writeKra(final String key, final Map newMap, String comment) throws IOException {
// write map to kra
kelondroRA kra = dyn.getRA(key);
kra.writeMap(newMap, comment);
kra.close();
final kelondroRA kra = dyn.getRA(key);
kra.writeMap(newMap, comment);
kra.close();
}
private void updateAcc(Map map, boolean add) {
@ -178,18 +183,21 @@ public class kelondroMap {
Long accumulator;
for (int i = 0; i < accfields.length; i++) {
value = (String) map.get(accfields[i]);
if (value != null) try {
valuel = Long.parseLong(value);
accumulator = (Long) accMap.get(accfields[i]);
if (add)
accMap.put(accfields[i], new Long(accumulator.longValue() + ((long) valuel)));
else
accMap.put(accfields[i], new Long(accumulator.longValue() - ((long) valuel)));
} catch (NumberFormatException e) {}
if (value != null) {
try {
valuel = Long.parseLong(value);
accumulator = (Long) accMap.get(accfields[i]);
if (add) {
accMap.put(accfields[i], new Long(accumulator.longValue() + ((long) valuel)));
} else {
accMap.put(accfields[i], new Long(accumulator.longValue() - ((long) valuel)));
}
} catch (NumberFormatException e) {}
}
}
}
private void updateSortCluster(String key, Map map) {
private void updateSortCluster(final String key, final Map map) {
String value;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
@ -206,7 +214,7 @@ public class kelondroMap {
// update elementCount
if (key == null) return;
if ((sortfields != null) || (accfields != null)) {
Map map = get(key);
final Map map = get(key);
if (map != null) {
// update count
elementCount--;
@ -218,7 +226,6 @@ public class kelondroMap {
if (sortfields != null) deleteSortCluster(key);
}
}
// remove from cache
cacheScore.deleteScore(key);
cache.remove(key);
@ -227,7 +234,7 @@ public class kelondroMap {
dyn.remove(key);
}
private void deleteSortCluster(String key) {
private void deleteSortCluster(final String key) {
if (key == null) return;
kelondroMScoreCluster cluster;
for (int i = 0; i < sortfields.length; i++) {
@ -237,21 +244,21 @@ public class kelondroMap {
}
}
public synchronized Map get(String key) throws IOException {
public synchronized Map get(final String key) throws IOException {
if (key == null) return null;
return get(key, true);
}
private synchronized Map get(String key, boolean storeCache) throws IOException {
// load map from cache
private synchronized Map get(final String key, final boolean storeCache) throws IOException {
// load map from cache
Map map = (Map) cache.get(key);
if (map != null) return map;
// load map from kra
// load map from kra
if (!(dyn.existsDyn(key))) return null;
kelondroRA kra = dyn.getRA(key);
map = kra.readMap();
kra.close();
final kelondroRA kra = dyn.getRA(key);
map = kra.readMap();
kra.close();
if (storeCache) {
// cache it also
@ -262,53 +269,52 @@ public class kelondroMap {
}
// return value
return map;
return map;
}
private synchronized void checkCacheSpace() {
// check for space in cache
if (cache.size() >= cachesize) {
// delete one entry
String delkey = (String) cacheScore.getMinObject();
final String delkey = (String) cacheScore.getMinObject();
cacheScore.deleteScore(delkey);
cache.remove(delkey);
}
}
public synchronized kelondroDyn.dynKeyIterator keys(boolean up, boolean rotating) throws IOException {
public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating) throws IOException {
// simple enumeration of key names without special ordering
return dyn.dynKeys(up, rotating);
}
public synchronized kelondroDyn.dynKeyIterator keys(boolean up, boolean rotating, byte[] firstKey) throws IOException {
public synchronized kelondroDyn.dynKeyIterator keys(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
// simple enumeration of key names without special ordering
return dyn.dynKeys(up, rotating, firstKey);
}
public synchronized Iterator keys(boolean up, /* sorted by */ String field) {
public synchronized Iterator keys(final boolean up, /* sorted by */ String field) {
// sorted iteration using the sortClusters
if (sortClusterMap == null) return null;
kelondroMScoreCluster cluster = (kelondroMScoreCluster) sortClusterMap.get(field);
final kelondroMScoreCluster cluster = (kelondroMScoreCluster) sortClusterMap.get(field);
if (cluster == null) return null; // sort field does not exist
//System.out.println("DEBUG: cluster for field " + field + ": " + cluster.toString());
return cluster.scores(up);
}
public synchronized mapIterator maps(boolean up, boolean rotating) throws IOException {
public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException {
return new mapIterator(keys(up, rotating));
}
public synchronized mapIterator maps(boolean up, boolean rotating, byte[] firstKey) throws IOException {
public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey) throws IOException {
return new mapIterator(keys(up, rotating, firstKey));
}
public synchronized mapIterator maps(boolean up, String field) {
public synchronized mapIterator maps(final boolean up, final String field) {
return new mapIterator(keys(up, field));
}
public synchronized long getAcc(String field) {
Long accumulator = (Long) accMap.get(field);
public synchronized long getAcc(final String field) {
final Long accumulator = (Long) accMap.get(field);
if (accumulator == null) return -1; else return accumulator.longValue();
}
@ -332,7 +338,6 @@ public class kelondroMap {
dyn.close();
}
public class mapIterator implements Iterator {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
@ -350,13 +355,13 @@ public class kelondroMap {
}
public Object next() {
String nextKey = (String) keyIterator.next();
final String nextKey = (String) keyIterator.next();
if (nextKey == null) {
finish = true;
return null;
}
try {
Map map = get(nextKey);
final Map map = get(nextKey);
if (map == null) throw new kelondroException(dyn.filename, "no more elements available");
map.put("key", nextKey);
return map;
@ -369,6 +374,6 @@ public class kelondroMap {
public void remove() {
throw new UnsupportedOperationException();
}
} // class mapIterator
}
}
} // class kelondroMap

62
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -4,7 +4,10 @@
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
// last major change: 02.02.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@ -42,7 +45,6 @@
// compile with
// javac -classpath classes -sourcepath source -d classes -g source/de/anomic/plasma/*.java
package de.anomic.plasma;
import java.io.File;
@ -112,7 +114,7 @@ public final class plasmaWordIndex {
}
public void intermission(long pause) {
this.ramCache.intermission(pause);
this.ramCache.intermission(pause);
}
public void close(int waitingBoundSeconds) {
@ -129,13 +131,12 @@ public final class plasmaWordIndex {
}
private class correctedWordIterator implements Iterator {
Iterator ii;
Iterator iter;
String nextWord;
public correctedWordIterator(boolean up, boolean rotating, String firstWord) {
ii = ramCache.wordHashes(firstWord, up);
nextWord = (ii.hasNext()) ? (String) ii.next() : null;
iter = ramCache.wordHashes(firstWord, up);
nextWord = (iter.hasNext()) ? (String) iter.next() : null;
boolean corrected = true;
int cc = 0; // to avoid rotation loops
while ((nextWord != null) && (corrected) && (cc < 50)) {
@ -144,12 +145,12 @@ public final class plasmaWordIndex {
if ((c > 0) && (up)) {
// firstKey > nextNode.getKey()
//System.out.println("CORRECTING WORD ITERATOR: firstWord=" + firstWord + ", nextWord=" + nextWord);
nextWord = (ii.hasNext()) ? (String) ii.next() : null;
nextWord = (iter.hasNext()) ? (String) iter.next() : null;
corrected = true;
cc++;
}
if ((c < 0) && (!(up))) {
nextWord = (ii.hasNext()) ? (String) ii.next() : null;
nextWord = (iter.hasNext()) ? (String) iter.next() : null;
corrected = true;
cc++;
}
@ -157,24 +158,24 @@ public final class plasmaWordIndex {
}
public void finalize() {
ii = null;
iter = null;
nextWord = null;
}
public boolean hasNext() {
public boolean hasNext() {
return nextWord != null;
}
}
public Object next() {
String r = nextWord;
nextWord = (ii.hasNext()) ? (String) ii.next() : null;
nextWord = (iter.hasNext()) ? (String) iter.next() : null;
return r;
}
public void remove() {
throw new java.lang.UnsupportedOperationException("kelondroTree: remove in kelondro Tables not yet supported");
}
}
}
} // correctedWordIterator
public Iterator fileIterator(String startHash, boolean up, boolean deleteEmpty) {
return new iterateFiles(startHash, up, deleteEmpty);
@ -188,7 +189,6 @@ public final class plasmaWordIndex {
private String buffer; // the prefetch-buffer
private boolean delete;
public iterateFiles(String startHash, boolean up, boolean deleteEmpty) {
this.hierarchy = new ArrayList();
this.comp = kelondroMSetTools.fastStringComparator(up);
@ -228,12 +228,12 @@ public final class plasmaWordIndex {
}
private TreeSet list(File path) {
//System.out.println("PATH: " + path);
// System.out.println("PATH: " + path);
TreeSet t = new TreeSet(comp);
String[] l = path.list();
if (l != null) for (int i = 0; i < l.length; i++) t.add(path + "/" + l[i]);
//else System.out.println("DEBUG: wrong path " + path);
//System.out.println(t);
// else System.out.println("DEBUG: wrong path " + path);
// System.out.println(t);
return t;
}
@ -267,11 +267,11 @@ public final class plasmaWordIndex {
}
} while (f == null);
// thats it
if ((f == null) || ((n = f.getName()) == null) || (n.length() < yacySeedDB.commonHashLength)) {
return null;
} else {
return n.substring(0, yacySeedDB.commonHashLength);
}
if ((f == null) || ((n = f.getName()) == null) || (n.length() < yacySeedDB.commonHashLength)) {
return null;
} else {
return n.substring(0, yacySeedDB.commonHashLength);
}
}
public boolean hasNext() {
@ -285,18 +285,18 @@ public final class plasmaWordIndex {
}
public void remove() {
}
}
public static void main(String[] args) {
//System.out.println(kelondroMSetTools.fastStringComparator(true).compare("RwGeoUdyDQ0Y", "rwGeoUdyDQ0Y"));
// System.out.println(kelondroMSetTools.fastStringComparator(true).compare("RwGeoUdyDQ0Y", "rwGeoUdyDQ0Y"));
try {
plasmaWordIndex index = new plasmaWordIndex(new File("D:\\dev\\proxy\\DATA\\PLASMADB"), 555, new serverLog("TESTAPP"));
Iterator i = index.wordHashes("5A8yhZMh_Kmv", true, true);
while (i.hasNext()) {
System.out.println("File: " + (String) i.next());
}
plasmaWordIndex index = new plasmaWordIndex(new File("D:\\dev\\proxy\\DATA\\PLASMADB"), 555, new serverLog("TESTAPP"));
Iterator iter = index.wordHashes("5A8yhZMh_Kmv", true, true);
while (iter.hasNext()) {
System.out.println("File: " + (String) iter.next());
}
} catch (IOException e) {}
}
}

Write Preview
Loading…
Cancel
Save