boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed synchronization in table creation

Browse Source 
to avoid possible deadlocks when handling OnDemandOpenFileIndex
which happens quite often during wide crawling
pull/436/head
Michael Peter Christen 4 years ago
parent 8084960392
commit 4cadd557dc
1 changed files with 168 additions and 167 deletions
Show Stats Download Patch File Download Diff File

53
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.ConcurrentSkipListMap;
import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.CloneableIterator;
@ -73,7 +74,7 @@ public class Table implements Index, Iterable<Row.Entry> {
private final static ConcurrentLog log = new ConcurrentLog("TABLE"); private final static ConcurrentLog log = new ConcurrentLog("TABLE");
/** Map all active table instances by file name */ /** Map all active table instances by file name */
private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>(); private final static Map<String, Table> tableTracker = new ConcurrentSkipListMap<String, Table>();
private final static long maxarraylength = 134217727L; // (2^27-1) that may be the maximum size of array length in some JVMs private final static long maxarraylength = 134217727L; // (2^27-1) that may be the maximum size of array length in some JVMs
private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned private final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned
@ -270,14 +271,14 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
// track this table // track this table
synchronized (tableTracker) {tableTracker.put(tablefile.toString(), this);} tableTracker.put(tablefile.toString(), this);
} }
public synchronized void warmUp() { public synchronized void warmUp() {
warmUp0(); warmUp0();
} }
private void warmUp0() { private final void warmUp0() {
// remove doubles // remove doubles
try { try {
final ArrayList<long[]> doubles = this.index.removeDoubles(); final ArrayList<long[]> doubles = this.index.removeDoubles();
@ -451,7 +452,7 @@ public class Table implements Index, Iterable<Row.Entry> {
stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize); stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize);
stats.keyMem = (long)((RowHandleMap) this.index).row().objectsize * (long)this.index.size(); stats.keyMem = (long)((RowHandleMap) this.index).row().objectsize * (long)this.index.size();
} }
if(table != null) { if(this.table != null) {
stats.valueChunkSize = this.table.row().objectsize; stats.valueChunkSize = this.table.row().objectsize;
stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size(); stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size();
} }
@ -616,7 +617,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
} }
private Entry get0(final byte[] key) throws IOException { private final Entry get0(final byte[] key) throws IOException {
if (this.file == null || this.index == null) return null; if (this.file == null || this.index == null) return null;
final int i = (int) this.index.get(key); final int i = (int) this.index.get(key);
if (i == -1) return null; if (i == -1) return null;
@ -643,7 +644,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException { public final Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException {
final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder); final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder);
Row.Entry entry; Row.Entry entry;
for (final byte[] key: keys) { for (final byte[] key: keys) {
@ -654,13 +655,13 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public boolean has(final byte[] key) { public final boolean has(final byte[] key) {
if (this.index == null) return false; if (this.index == null) return false;
return this.index.has(key); return this.index.has(key);
} }
@Override @Override
public synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException { public final synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
return this.index.keys(up, firstKey); return this.index.keys(up, firstKey);
} }
@ -723,7 +724,7 @@ public class Table implements Index, Iterable<Row.Entry> {
* @throws SpaceExceededException * @throws SpaceExceededException
*/ */
@Override @Override
public boolean put(final Entry row) throws IOException, SpaceExceededException { public final boolean put(final Entry row) throws IOException, SpaceExceededException {
assert row != null; assert row != null;
if (this.file == null || row == null) return true; if (this.file == null || row == null) return true;
final byte[] rowb = row.bytes(); final byte[] rowb = row.bytes();
@ -769,7 +770,7 @@ public class Table implements Index, Iterable<Row.Entry> {
* @throws IOException * @throws IOException
* @throws SpaceExceededException * @throws SpaceExceededException
*/ */
private void removeInFile(final int i) throws IOException, SpaceExceededException { private final void removeInFile(final int i) throws IOException, SpaceExceededException {
assert i >= 0; assert i >= 0;
final byte[] p = new byte[this.rowdef.objectsize]; final byte[] p = new byte[this.rowdef.objectsize];
@ -828,12 +829,12 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public boolean delete(final byte[] key) throws IOException { public final boolean delete(final byte[] key) throws IOException {
return remove(key) != null; return remove(key) != null;
} }
@Override @Override
public synchronized Entry remove(final byte[] key) throws IOException { public final synchronized Entry remove(final byte[] key) throws IOException {
assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
assert key.length == this.rowdef.primaryKeyLength; assert key.length == this.rowdef.primaryKeyLength;
@ -928,7 +929,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public synchronized Entry removeOne() throws IOException { public final synchronized Entry removeOne() throws IOException {
//assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size(); //assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();
assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size(); assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();
final byte[] le = new byte[this.rowdef.objectsize]; final byte[] le = new byte[this.rowdef.objectsize];
@ -955,7 +956,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public List<Row.Entry> top(int count) throws IOException { public final List<Row.Entry> top(int count) throws IOException {
if (count > this.size()) count = this.size(); if (count > this.size()) count = this.size();
final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>(); final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list; if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list;
@ -971,7 +972,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public List<Row.Entry> random(int count) throws IOException { public final List<Row.Entry> random(int count) throws IOException {
if (count > this.size()) count = this.size(); if (count > this.size()) count = this.size();
final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>(); final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list; if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list;
@ -996,23 +997,23 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public Row row() { public final Row row() {
return this.rowdef; return this.rowdef;
} }
@Override @Override
public int size() { public final int size() {
if (this.index == null) return 0; if (this.index == null) return 0;
return this.index.size(); return this.index.size();
} }
@Override @Override
public boolean isEmpty() { public final boolean isEmpty() {
return this.index == null || this.index.isEmpty(); return this.index == null || this.index.isEmpty();
} }
@Override @Override
public Iterator<Entry> iterator() { public final Iterator<Entry> iterator() {
try { try {
return rows(); return rows();
} catch (final IOException e) { } catch (final IOException e) {
@ -1026,7 +1027,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return new rowIteratorNoOrder(); return new rowIteratorNoOrder();
} }
private class rowIteratorNoOrder implements CloneableIterator<Entry> { private final class rowIteratorNoOrder implements CloneableIterator<Entry> {
Iterator<Map.Entry<byte[], Long>> i; Iterator<Map.Entry<byte[], Long>> i;
long idx; long idx;
byte[] key; byte[] key;
@ -1038,17 +1039,17 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public CloneableIterator<Entry> clone(final Object modifier) { public final CloneableIterator<Entry> clone(final Object modifier) {
return new rowIteratorNoOrder(); return new rowIteratorNoOrder();
} }
@Override @Override
public boolean hasNext() { public final boolean hasNext() {
return this.i != null && this.i.hasNext(); return this.i != null && this.i.hasNext();
} }
@Override @Override
public Entry next() { public final Entry next() {
final Map.Entry<byte[], Long> entry = this.i.next(); final Map.Entry<byte[], Long> entry = this.i.next();
if (entry == null) return null; if (entry == null) return null;
this.key = entry.getKey(); this.key = entry.getKey();
@ -1062,7 +1063,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public void remove() { public final void remove() {
if (this.key != null) { if (this.key != null) {
try { try {
removeInFile((int) this.idx); removeInFile((int) this.idx);
@ -1074,7 +1075,7 @@ public class Table implements Index, Iterable<Row.Entry> {
} }
@Override @Override
public void close() { public final void close() {
if (this.i instanceof CloneableIterator) { if (this.i instanceof CloneableIterator) {
((CloneableIterator<Map.Entry<byte[], Long>>) this.i).close(); ((CloneableIterator<Map.Entry<byte[], Long>>) this.i).close();
} }
@ -1087,7 +1088,7 @@ public class Table implements Index, Iterable<Row.Entry> {
return new rowIterator(up, firstKey); return new rowIterator(up, firstKey);
} }
private class rowIterator implements CloneableIterator<Entry> { private final class rowIterator implements CloneableIterator<Entry> {
private final CloneableIterator<byte[]> i; private final CloneableIterator<byte[]> i;
private final boolean up; private final boolean up;
private final byte[] fk; private final byte[] fk;

Write Preview
Loading…
Cancel
Save