boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed synchronization in table creation

Browse Source 
to avoid possible deadlocks when handling OnDemandOpenFileIndex
which happens quite often during wide crawling
pull/436/head
Michael Peter Christen 4 years ago
parent 8084960392
commit 4cadd557dc
1 changed files with 168 additions and 167 deletions
Show Stats Download Patch File Download Diff File

335
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListMap;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.CloneableIterator;
@ -71,9 +72,9 @@ public class Table implements Index, Iterable<Row.Entry> {
// static tracker objects // static tracker objects
private final static ConcurrentLog log = new ConcurrentLog("TABLE"); private final static ConcurrentLog log = new ConcurrentLog("TABLE");
/** Map all active table instances by file name */ /** Map all active table instances by file name */
private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>(); private final static Map<String, Table> tableTracker = new ConcurrentSkipListMap<String, Table>();
private final static long maxarraylength = 134217727L; // (2^27-1) that may be the maximum size of array length in some JVMs private final static long maxarraylength = 134217727L; // (2^27-1) that may be the maximum size of array length in some JVMs
private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned
@ -85,13 +86,13 @@ public class Table implements Index, Iterable<Row.Entry> {
private RowSet table; private RowSet table;
public Table( public Table(
final File tablefile, final File tablefile,
final Row rowdef, final Row rowdef,
final int buffersize, final int buffersize,
final int initialSpace, final int initialSpace,
boolean useTailCache, boolean useTailCache,
final boolean exceed134217727, final boolean exceed134217727,
final boolean warmUp) throws SpaceExceededException, kelondroException { final boolean warmUp) throws SpaceExceededException, kelondroException {
this.rowdef = rowdef; this.rowdef = rowdef;
this.buffersize = buffersize; this.buffersize = buffersize;
@ -111,7 +112,7 @@ public class Table implements Index, Iterable<Row.Entry> {
freshFile = true; freshFile = true;
FileOutputStream fos = null; FileOutputStream fos = null;
if(tablefile.getParentFile() != null) { if(tablefile.getParentFile() != null) {
tablefile.getParentFile().mkdirs(); tablefile.getParentFile().mkdirs();
} }
try { try {
fos = new FileOutputStream(tablefile); fos = new FileOutputStream(tablefile);
@ -131,17 +132,17 @@ public class Table implements Index, Iterable<Row.Entry> {
final int records = Math.max(fileSize, initialSpace); final int records = Math.max(fileSize, initialSpace);
final long neededRAM4table = 200L * 1024L * 1024L + records * (this.taildef.objectsize + rowdef.primaryKeyLength + 4L) * 3L / 2L; final long neededRAM4table = 200L * 1024L * 1024L + records * (this.taildef.objectsize + rowdef.primaryKeyLength + 4L) * 3L / 2L;
this.table = null; this.table = null;
try { try {
this.table = ((exceed134217727 || neededRAM4table < maxarraylength) && this.table = ((exceed134217727 || neededRAM4table < maxarraylength) &&
useTailCache && MemoryControl.available() > 600L * 1024L * 1024L && useTailCache && MemoryControl.available() > 600L * 1024L * 1024L &&
MemoryControl.request(neededRAM4table, true)) ? new RowSet(this.taildef, records) : null; MemoryControl.request(neededRAM4table, true)) ? new RowSet(this.taildef, records) : null;
} catch (final SpaceExceededException e) { } catch (final SpaceExceededException e) {
this.table = null; this.table = null;
} catch (final Throwable e) { } catch (final Throwable e) {
this.table = null; this.table = null;
} }
if (log.isFine()) log.fine("initialization of " + tablefile.getName() + ". table copy: " + ((this.table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024L / 1024L) + "MB, needed: " + (neededRAM4table / 1024L / 1024L) + "MB, allocating space for " + records + " entries"); if (log.isFine()) log.fine("initialization of " + tablefile.getName() + ". table copy: " + ((this.table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024L / 1024L) + "MB, needed: " + (neededRAM4table / 1024L / 1024L) + "MB, allocating space for " + records + " entries");
final long neededRAM4index = 100L * 1024L * 1024L + records * (rowdef.primaryKeyLength + 4L) * 3L / 2L; final long neededRAM4index = 100L * 1024L * 1024L + records * (rowdef.primaryKeyLength + 4L) * 3L / 2L;
if (records > 0 && !MemoryControl.request(neededRAM4index, true)) { if (records > 0 && !MemoryControl.request(neededRAM4index, true)) {
@ -163,75 +164,75 @@ public class Table implements Index, Iterable<Row.Entry> {
if (this.table == null) { if (this.table == null) {
final ChunkIterator ki = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.primaryKeyLength); final ChunkIterator ki = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.primaryKeyLength);
try { try {
while (ki.hasNext()) { while (ki.hasNext()) {
key = ki.next(); key = ki.next();
// write the key into the index table // write the key into the index table
assert key != null; assert key != null;
if (key == null) {i++; continue;} if (key == null) {i++; continue;}
if (rowdef.objectOrder.wellformed(key)) { if (rowdef.objectOrder.wellformed(key)) {
this.index.putUnique(key, i++); this.index.putUnique(key, i++);
} else { } else {
errors.putUnique(key, i++); errors.putUnique(key, i++);
} }
} }
} finally { } finally {
/* If any error occurred while looping over the iterator, we /* If any error occurred while looping over the iterator, we
* must ensure the underlying stream is closed before * must ensure the underlying stream is closed before
* transmitting the exception to the upper layer * transmitting the exception to the upper layer
*/ */
if(ki.hasNext()) { if(ki.hasNext()) {
try { try {
ki.close(); ki.close();
} catch(IOException ioe) { } catch(IOException ioe) {
/* Do not block if closing is not possible but anyway keep a trace in log */ /* Do not block if closing is not possible but anyway keep a trace in log */
log.warn("Could not close input stream on the file " + tablefile); log.warn("Could not close input stream on the file " + tablefile);
} }
} }
} }
} else { } else {
byte[] record; byte[] record;
key = new byte[rowdef.primaryKeyLength]; key = new byte[rowdef.primaryKeyLength];
final ChunkIterator ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize); final ChunkIterator ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize);
try { try {
while (ri.hasNext()) { while (ri.hasNext()) {
record = ri.next(); record = ri.next();
assert record != null; assert record != null;
if (record == null) {i++; continue;} if (record == null) {i++; continue;}
System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength); System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
// write the key into the index table // write the key into the index table
if (rowdef.objectOrder.wellformed(key)) { if (rowdef.objectOrder.wellformed(key)) {
this.index.putUnique(key, i++); this.index.putUnique(key, i++);
// write the tail into the table // write the tail into the table
try { try {
this.table.addUnique(this.taildef.newEntry(record, rowdef.primaryKeyLength, true)); this.table.addUnique(this.taildef.newEntry(record, rowdef.primaryKeyLength, true));
} catch (final SpaceExceededException e) { } catch (final SpaceExceededException e) {
this.table = null; this.table = null;
try { try {
ri.close(); // close inputstream of chunkiterator ri.close(); // close inputstream of chunkiterator
} finally { } finally {
/* Do not block if closing is not possible but anyway keep a trace in log */ /* Do not block if closing is not possible but anyway keep a trace in log */
log.warn("Could not close input stream on the file " + tablefile); log.warn("Could not close input stream on the file " + tablefile);
} }
break; break;
} }
} else { } else {
errors.putUnique(key, i++); errors.putUnique(key, i++);
} }
} }
} finally { } finally {
/* If any error occurred while looping over the iterator, we /* If any error occurred while looping over the iterator, we
* must ensure the underlying stream is closed before * must ensure the underlying stream is closed before
* transmitting the exception to the upper layer * transmitting the exception to the upper layer
*/ */
if(ri.hasNext()) { if(ri.hasNext()) {
try { try {
ri.close(); ri.close();
} catch(IOException ioe) { } catch(IOException ioe) {
/* Do not block if closing is not possible but anyway keep a trace in log */ /* Do not block if closing is not possible but anyway keep a trace in log */
log.warn("Could not close input stream on the file " + tablefile); log.warn("Could not close input stream on the file " + tablefile);
} }
} }
} }
Runtime.getRuntime().gc(); Runtime.getRuntime().gc();
if (abandonTable()) { if (abandonTable()) {
@ -270,14 +271,14 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
// track this table // track this table
synchronized (tableTracker) {tableTracker.put(tablefile.toString(), this);} tableTracker.put(tablefile.toString(), this);
} }
public synchronized void warmUp() { public synchronized void warmUp() {
warmUp0(); warmUp0();
} }
private void warmUp0() { private final void warmUp0() {
// remove doubles // remove doubles
try { try {
final ArrayList<long[]> doubles = this.index.removeDoubles(); final ArrayList<long[]> doubles = this.index.removeDoubles();
@ -378,62 +379,62 @@ public class Table implements Index, Iterable<Row.Entry> {
* Table memory usage statistics * Table memory usage statistics
*/ */
public class TableStatistics { public class TableStatistics {
/** The table entries number */ /** The table entries number */
private int tableSize = 0; private int tableSize = 0;
/** Size of a key chunk in bytes */ /** Size of a key chunk in bytes */
private int keyChunkSize = 0; private int keyChunkSize = 0;
/** Total size of keys in bytes */ /** Total size of keys in bytes */
private long keyMem = 0; private long keyMem = 0;
/** Size of a value chunk in bytes */ /** Size of a value chunk in bytes */
private int valueChunkSize = 0; private int valueChunkSize = 0;
/** Total size of values in bytes */ /** Total size of values in bytes */
private long valueMem = 0; private long valueMem = 0;
/** /**
* @return the size of a key chunk in bytes * @return the size of a key chunk in bytes
*/ */
public int getKeyChunkSize() { public int getKeyChunkSize() {
return this.keyChunkSize; return this.keyChunkSize;
} }
/** /**
* @return the total size of the table in-memory keys in bytes * @return the total size of the table in-memory keys in bytes
*/ */
public long getKeyMem() { public long getKeyMem() {
return this.keyMem; return this.keyMem;
} }
/** /**
* @return the table entries number * @return the table entries number
*/ */
public int getTableSize() { public int getTableSize() {
return this.tableSize; return this.tableSize;
} }
/** /**
* @return the size of a value chunk in bytes * @return the size of a value chunk in bytes
*/ */
public int getValueChunkSize() { public int getValueChunkSize() {
return this.valueChunkSize; return this.valueChunkSize;
} }
/** /**
* @return the total size of the table in-memory values in bytes * @return the total size of the table in-memory values in bytes
*/ */
public long getValueMem() { public long getValueMem() {
return this.valueMem; return this.valueMem;
} }
/** /**
* @return the total memory used by the table in bytes * @return the total memory used by the table in bytes
*/ */
public long getTotalMem() { public long getTotalMem() {
return this.keyMem + this.valueMem; return this.keyMem + this.valueMem;
} }
} }
/** /**
@ -448,12 +449,12 @@ public class Table implements Index, Iterable<Row.Entry> {
if (this.index == null) return stats; // possibly closed or being closed if (this.index == null) return stats; // possibly closed or being closed
stats.tableSize = this.index.size(); stats.tableSize = this.index.size();
if(this.index instanceof RowHandleMap) { if(this.index instanceof RowHandleMap) {
stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize); stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize);
stats.keyMem = (long)((RowHandleMap) this.index).row().objectsize * (long)this.index.size(); stats.keyMem = (long)((RowHandleMap) this.index).row().objectsize * (long)this.index.size();
} }
if(table != null) { if(this.table != null) {
stats.valueChunkSize = this.table.row().objectsize; stats.valueChunkSize = this.table.row().objectsize;
stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size(); stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size();
} }
return stats; return stats;
@ -579,17 +580,17 @@ public class Table implements Index, Iterable<Row.Entry> {
@Override @Override
public void close() { public void close() {
String tablefile = null; String tablefile = null;
if (this.file != null) { if (this.file != null) {
tablefile = this.file.filename().toString(); tablefile = this.file.filename().toString();
this.file.close(); this.file.close();
} }
this.file = null; this.file = null;
if (this.table != null) this.table.close(); if (this.table != null) this.table.close();
this.table = null; this.table = null;
if (this.index != null) this.index.close(); if (this.index != null) this.index.close();
this.index = null; this.index = null;
if (tablefile != null) tableTracker.remove(tablefile); if (tablefile != null) tableTracker.remove(tablefile);
} }
@Override @Override
@ -616,8 +617,8 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
} }
private Entry get0(final byte[] key) throws IOException { private final Entry get0(final byte[] key) throws IOException {
if (this.file == null || this.index == null) return null; if (this.file == null || this.index == null) return null;
final int i = (int) this.index.get(key); final int i = (int) this.index.get(key);
if (i == -1) return null; if (i == -1) return null;
final byte[] b = new byte[this.rowdef.objectsize]; final byte[] b = new byte[this.rowdef.objectsize];
@ -643,7 +644,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException { public final Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException {
final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder); final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder);
Row.Entry entry; Row.Entry entry;
for (final byte[] key: keys) { for (final byte[] key: keys) {
@ -654,13 +655,13 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public boolean has(final byte[] key) { public final boolean has(final byte[] key) {
if (this.index == null) return false; if (this.index == null) return false;
return this.index.has(key); return this.index.has(key);
} }
@Override @Override
public synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException { public final synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
return this.index.keys(up, firstKey); return this.index.keys(up, firstKey);
} }
@ -723,7 +724,7 @@ public class Table implements Index, Iterable<Row.Entry> {
* @throws SpaceExceededException * @throws SpaceExceededException
*/ */
@Override @Override
public boolean put(final Entry row) throws IOException, SpaceExceededException { public final boolean put(final Entry row) throws IOException, SpaceExceededException {
assert row != null; assert row != null;
if (this.file == null || row == null) return true; if (this.file == null || row == null) return true;
final byte[] rowb = row.bytes(); final byte[] rowb = row.bytes();
@ -769,7 +770,7 @@ public class Table implements Index, Iterable<Row.Entry> {
* @throws IOException * @throws IOException
* @throws SpaceExceededException * @throws SpaceExceededException
*/ */
private void removeInFile(final int i) throws IOException, SpaceExceededException { private final void removeInFile(final int i) throws IOException, SpaceExceededException {
assert i >= 0; assert i >= 0;
final byte[] p = new byte[this.rowdef.objectsize]; final byte[] p = new byte[this.rowdef.objectsize];
@ -828,12 +829,12 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public boolean delete(final byte[] key) throws IOException { public final boolean delete(final byte[] key) throws IOException {
return remove(key) != null; return remove(key) != null;
} }
@Override @Override
public synchronized Entry remove(final byte[] key) throws IOException { public final synchronized Entry remove(final byte[] key) throws IOException {
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
assert key.length == this.rowdef.primaryKeyLength; assert key.length == this.rowdef.primaryKeyLength;
@ -928,7 +929,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public synchronized Entry removeOne() throws IOException { public final synchronized Entry removeOne() throws IOException {
//assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); //assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
final byte[] le = new byte[this.rowdef.objectsize]; final byte[] le = new byte[this.rowdef.objectsize];
@ -955,7 +956,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public List<Row.Entry> top(int count) throws IOException { public final List<Row.Entry> top(int count) throws IOException {
if (count > this.size()) count = this.size(); if (count > this.size()) count = this.size();
final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>(); final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list; if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list;
@ -971,7 +972,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public List<Row.Entry> random(int count) throws IOException { public final List<Row.Entry> random(int count) throws IOException {
if (count > this.size()) count = this.size(); if (count > this.size()) count = this.size();
final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>(); final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list; if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list;
@ -996,23 +997,23 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public Row row() { public final Row row() {
return this.rowdef; return this.rowdef;
} }
@Override @Override
public int size() { public final int size() {
if (this.index == null) return 0; if (this.index == null) return 0;
return this.index.size(); return this.index.size();
} }
@Override @Override
public boolean isEmpty() { public final boolean isEmpty() {
return this.index == null || this.index.isEmpty(); return this.index == null || this.index.isEmpty();
} }
@Override @Override
public Iterator<Entry> iterator() { public final Iterator<Entry> iterator() {
try { try {
return rows(); return rows();
} catch (final IOException e) { } catch (final IOException e) {
@ -1026,7 +1027,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return new rowIteratorNoOrder(); return new rowIteratorNoOrder();
} }
private class rowIteratorNoOrder implements CloneableIterator<Entry> { private final class rowIteratorNoOrder implements CloneableIterator<Entry> {
Iterator<Map.Entry<byte[], Long>> i; Iterator<Map.Entry<byte[], Long>> i;
long idx; long idx;
byte[] key; byte[] key;
@ -1038,17 +1039,17 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public CloneableIterator<Entry> clone(final Object modifier) { public final CloneableIterator<Entry> clone(final Object modifier) {
return new rowIteratorNoOrder(); return new rowIteratorNoOrder();
} }
@Override @Override
public boolean hasNext() { public final boolean hasNext() {
return this.i != null && this.i.hasNext(); return this.i != null && this.i.hasNext();
} }
@Override @Override
public Entry next() { public final Entry next() {
final Map.Entry<byte[], Long> entry = this.i.next(); final Map.Entry<byte[], Long> entry = this.i.next();
if (entry == null) return null; if (entry == null) return null;
this.key = entry.getKey(); this.key = entry.getKey();
@ -1062,7 +1063,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public void remove() { public final void remove() {
if (this.key != null) { if (this.key != null) {
try { try {
removeInFile((int) this.idx); removeInFile((int) this.idx);
@ -1074,7 +1075,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public void close() { public final void close() {
if (this.i instanceof CloneableIterator) { if (this.i instanceof CloneableIterator) {
((CloneableIterator<Map.Entry<byte[], Long>>) this.i).close(); ((CloneableIterator<Map.Entry<byte[], Long>>) this.i).close();
} }
@ -1087,7 +1088,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return new rowIterator(up, firstKey); return new rowIterator(up, firstKey);
} }
private class rowIterator implements CloneableIterator<Entry> { private final class rowIterator implements CloneableIterator<Entry> {
private final CloneableIterator<byte[]> i; private final CloneableIterator<byte[]> i;
private final boolean up; private final boolean up;
private final byte[] fk; private final byte[] fk;
@ -1167,7 +1168,7 @@ public class Table implements Index, Iterable<Row.Entry> {
for (int pos = 1; pos < source.length() - 1; pos++) { for (int pos = 1; pos < source.length() - 1; pos++) {
result[perm * source.length() + pos] = recres[perm].substring(0, pos) + c + recres[perm].substring(pos); result[perm * source.length() + pos] = recres[perm].substring(0, pos) + c + recres[perm].substring(pos);
} }
result[perm * source.length() + source.length() - 1] = recres[perm] + c; result[perm * source.length() + source.length() - 1] = recres[perm] + c;
} }
return result; return result;
} }

Write Preview
Loading…
Cancel
Save