boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

small redesign of blob column index and usage

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent d5c1f2746e
commit b4409cc803
3 changed files with 37 additions and 59 deletions
Show Stats Download Patch File Download Diff File

26
source/net/yacy/kelondro/blob/MapColumnIndex.java
Unescape View File

@ -38,31 +38,37 @@ import net.yacy.kelondro.order.NaturalOrder;
/**
* a mapping from a column name to maps with the value of the columns to the primary keys where the entry exist in the table
*/
public class MapColumnIndex extends HashMap<String, Map<String, Collection<byte[]>>> implements Map<String, Map<String, Collection<byte[]>>> {
public class MapColumnIndex {
private static final long serialVersionUID=-424741536889467566L;
private final Map<String, Map<String, Collection<byte[]>>> index;
public MapColumnIndex() {
super();
this.index = new HashMap<String, Map<String, Collection<byte[]>>>();
}
public Collection<byte[]> getIndex(final String whereKey, final String isValue) throws UnsupportedOperationException {
Map<String, Collection<byte[]>> references = this.get(whereKey);
public synchronized Collection<byte[]> getIndex(final String whereKey, final String isValue) throws UnsupportedOperationException {
Map<String, Collection<byte[]>> references = this.index.get(whereKey);
if (references == null) throw new UnsupportedOperationException();
Collection<byte[]> indexes = references.get(isValue);
if (indexes == null) return new ArrayList<byte[]>(0); // empty collection
return indexes;
}
public synchronized void clear() {
this.index.clear();
}
/**
* create a full index for the whereKey
* @param whereKey
* @param isValue
* @param table
*/
public void init(final String whereKey, final String isValue, final Iterator<Map.Entry<byte[], Map<String, String>>> table) {
public synchronized void init(final String whereKey, final String isValue, final Iterator<Map.Entry<byte[], Map<String, String>>> table) {
Map<String, Collection<byte[]>> valueIdxMap = new HashMap<String, Collection<byte[]>>();
this.put(whereKey, valueIdxMap);
this.index.put(whereKey, valueIdxMap);
Map.Entry<byte[], Map<String, String>> line;
while (table.hasNext()) {
line = table.next();
@ -77,8 +83,8 @@ public class MapColumnIndex extends HashMap<String, Map<String, Collection<byte[
* @param primarykey the primary key for the row that is updated
* @param row the row that was updated (a mapping from column names to values)
*/
public void update(final byte[] primarykey, final Map<String, String> row) {
for (Map.Entry<String, Map<String, Collection<byte[]>>> entry: this.entrySet()) {
public synchronized void update(final byte[] primarykey, final Map<String, String> row) {
for (Map.Entry<String, Map<String, Collection<byte[]>>> entry: this.index.entrySet()) {
// create an index for all columns that we track
String value = row.get(entry.getKey());
if (value == null) continue; // we don't need to remember that
@ -106,8 +112,8 @@ public class MapColumnIndex extends HashMap<String, Map<String, Collection<byte[
* delete all references to the primary key
* @param primarykey
*/
public void delete(final byte[] primarykey) {
for (Map.Entry<String, Map<String, Collection<byte[]>>> entry: this.entrySet()) {
public synchronized void delete(final byte[] primarykey) {
for (Map.Entry<String, Map<String, Collection<byte[]>>> entry: this.index.entrySet()) {
// we must check all index reference maps: iterate over entries
indexdelete(primarykey, entry.getValue());
}

15
source/net/yacy/kelondro/blob/MapDataMining.java
Unescape View File

@ -33,7 +33,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.UTF8;
@ -357,7 +356,7 @@ public class MapDataMining extends MapHeap {
}
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final String whereKey, final String isValue) throws IOException {
public synchronized Collection<byte[]> select(final String whereKey, final String isValue) throws IOException {
Collection<byte[]> idx = null;
try {
idx = this.columnIndex.getIndex(whereKey, isValue);
@ -369,17 +368,7 @@ public class MapDataMining extends MapHeap {
throw ee;
}
}
Map<byte[], Map<String, String>> resultMap = new TreeMap<byte[], Map<String, String>>(this.ordering());
for (byte[] pk: idx) {
try {
resultMap.put(pk, this.get(pk));
} catch (final IOException e) {
Log.logException(e);
} catch (final RowSpaceExceededException e) {
Log.logException(e);
}
}
return resultMap.entrySet().iterator();
return idx;
}
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final String field) {

55
source/net/yacy/peers/SeedDB.java
Unescape View File

@ -31,6 +31,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
@ -623,13 +624,10 @@ public final class SeedDB implements AlternativeDomainNames {
// enumerate the cache
String name = Seed.checkPeerName(peerName);
Map.Entry<byte[], Map<String, String>> entry;
synchronized (this) { try {
Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedActiveDB.entries(Seed.NAME, name);
while (mmap.hasNext()) {
entry = mmap.next();
if (entry == null) break;
seed = this.getConnected(ASCII.String(entry.getKey()));
Collection<byte[]> idx = this.seedActiveDB.select(Seed.NAME, name);
for (byte[] pk: idx) {
seed = this.getConnected(ASCII.String(pk));
if (seed == null) continue;
//System.out.println("*** found lookupByName in seedActiveDB: " + peerName);
return seed;
@ -637,11 +635,9 @@ public final class SeedDB implements AlternativeDomainNames {
} catch ( IOException e ) {
}}
synchronized (this) { try {
Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedPassiveDB.entries(Seed.NAME, name);
while (mmap.hasNext()) {
entry = mmap.next();
if (entry == null) break;
seed = this.getConnected(ASCII.String(entry.getKey()));
Collection<byte[]> idx = this.seedPassiveDB.select(Seed.NAME, name);
for (byte[] pk: idx) {
seed = this.getDisconnected(ASCII.String(pk));
if (seed == null) continue;
//System.out.println("*** found lookupByName in seedPassiveDB: " + peerName);
return seed;
@ -677,18 +673,13 @@ public final class SeedDB implements AlternativeDomainNames {
Seed seed = null;
String ipString = peerIP.getHostAddress();
Map.Entry<byte[], Map<String, String>> entry;
if (lookupConnected) synchronized (this) {
try {
Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedActiveDB.entries(Seed.IP, ipString);
while (mmap.hasNext()) {
entry = mmap.next();
if (entry == null) break;
String p = entry.getValue().get(Seed.PORT);
if (p == null) continue;
if (port > 0 && Integer.parseInt(p) != port) continue;
seed = this.getConnected(ASCII.String(entry.getKey()));
Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);
for (byte[] pk: idx) {
seed = this.getConnected(ASCII.String(pk));
if (seed == null) continue;
if (seed.getPort() != port) continue;
//System.out.println("*** found lookupByIP in connected: " + peerIP.toString() + " -> " + seed.getName());
return seed;
}
@ -698,15 +689,11 @@ public final class SeedDB implements AlternativeDomainNames {
if (lookupDisconnected) synchronized (this) {
try {
Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedPassiveDB.entries(Seed.IP, ipString);
while (mmap.hasNext()) {
entry = mmap.next();
if (entry == null) break;
String p = entry.getValue().get(Seed.PORT);
if (p == null) continue;
if (port > 0 && Integer.parseInt(p) != port) continue;
seed = this.getDisconnected(ASCII.String(entry.getKey()));
Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);
for (byte[] pk: idx) {
seed = this.getDisconnected(ASCII.String(pk));
if (seed == null) continue;
if (seed.getPort() != port) continue;
//System.out.println("*** found lookupByIP in disconnected: " + peerIP.toString() + " -> " + seed.getName());
return seed;
}
@ -716,15 +703,11 @@ public final class SeedDB implements AlternativeDomainNames {
if (lookupPotential) synchronized (this) {
try {
Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedPotentialDB.entries(Seed.IP, ipString);
while (mmap.hasNext()) {
entry = mmap.next();
if (entry == null) break;
String p = entry.getValue().get(Seed.PORT);
if (p == null) continue;
if (port > 0 && Integer.parseInt(p) != port) continue;
seed = this.getPotential(ASCII.String(entry.getKey()));
Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);
for (byte[] pk: idx) {
seed = this.getPotential(ASCII.String(pk));
if (seed == null) continue;
if (seed.getPort() != port) continue;
//System.out.println("*** found lookupByIP in potential: " + peerIP.toString() + " -> " + seed.getName());
return seed;
}

Write Preview
Loading…
Cancel
Save