yacy_search_server/source/net/yacy/kelondro/index/RAMIndex.java

/**
 *  RAMIndex
 *  Copyright 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
 *  First released 07.01.2008 at http://yacy.net
 *
 *  $LastChangedDate$
 *  $LastChangedRevision$
 *  $LastChangedBy$
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program in the file lgpl21.txt
 *  If not, see <http://www.gnu.org/licenses/>.
 */

package net.yacy.kelondro.index;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import net.yacy.cora.order.CloneableIterator;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.util.MergeIterator;
import net.yacy.kelondro.util.StackIterator;


public final class RAMIndex implements Index, Iterable<Row.Entry> {

    private static final Map<String, RAMIndex> objectTracker = Collections.synchronizedSortedMap(new TreeMap<String, RAMIndex>());

    private final String name;
    private final Row rowdef;
    private RowSet index0;
    private RowSet index1;
    private final Row.EntryComparator entryComparator;
    //private final int spread;

    public RAMIndex(final String name, final Row rowdef) {
        this.name = name;
        this.rowdef = rowdef;
        this.entryComparator = new Row.EntryComparator(rowdef.objectOrder);
        reset();
        objectTracker.put(name, this);
    }

    private RAMIndex(final String name, final Row rowdef, final RowSet index0, final RowSet index1, final Row.EntryComparator entryComparator) {
        this.name = name;
        this.rowdef = rowdef;
        this.index0 = index0;
        this.index1 = index1;
        this.entryComparator = entryComparator;
        objectTracker.put(name, this);
    }

    public static final Iterator<Map.Entry<String, RAMIndex>> objects() {
        return objectTracker.entrySet().iterator();
    }

    @Override
    public RAMIndex clone() {
        return new RAMIndex(this.name + ".clone", this.rowdef, this.index0.clone(), this.index1.clone(), this.entryComparator);
    }

    @Override
    public void clear() {
		reset();
	}

    @Override
    public void optimize() {
        if (this.index0 != null) this.index0.optimize();
        if (this.index1 != null) this.index1.optimize();
    }

    public final synchronized void reset() {
        this.index0 = null; // first flush RAM to make room
        this.index0 = new RowSet(this.rowdef);
        this.index1 = null; // to show that this is the initialization phase
    }

    public final synchronized void reset(final int initialspace) throws SpaceExceededException {
        this.index0 = null; // first flush RAM to make room
        this.index0 = new RowSet(this.rowdef, initialspace);
        this.index1 = null; // to show that this is the initialization phase
    }

    @Override
    public final Row row() {
        return this.index0.row();
    }

    protected final void finishInitialization() {
    	if (this.index1 == null) {
            // finish initialization phase
            this.index0.sort();
            this.index0.uniq();
            this.index0.trim();
            this.index1 = new RowSet(this.rowdef); //new RowSetArray(rowdef, spread);
        }
    }

    @Override
    public final synchronized byte[] smallestKey() {
        final byte[] b0 = this.index0.smallestKey();
        if (b0 == null) return null;
        if (this.index1 == null) return b0;
        final byte[] b1 = this.index0.smallestKey();
        if (b1 == null || this.rowdef.objectOrder.compare(b1, b0) > 0) return b0;
        return b1;
    }

    @Override
    public final synchronized byte[] largestKey() {
        final byte[] b0 = this.index0.largestKey();
        if (b0 == null) return null;
        if (this.index1 == null) return b0;
        final byte[] b1 = this.index0.largestKey();
        if (b1 == null || this.rowdef.objectOrder.compare(b0, b1) > 0) return b0;
        return b1;
    }

    @Override
    public final synchronized Row.Entry get(final byte[] key, final boolean forceclone) {
        assert (key != null);
        finishInitialization();
        assert this.index0.isSorted();
        final Row.Entry indexentry = this.index0.get(key, forceclone);
        if (indexentry != null) return indexentry;
        return this.index1.get(key, forceclone);
    }

    @Override
    public Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException {
        final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder);
        Row.Entry entry;
        for (final byte[] key: keys) {
            entry = get(key, forcecopy);
            if (entry != null) map.put(key, entry);
        }
        return map;
    }

    @Override
    public final synchronized boolean has(final byte[] key) {
		assert (key != null);
        finishInitialization();
        assert this.index0.isSorted();
        if (this.index0.has(key)) return true;
        return this.index1.has(key);
	}

	@Override
    public final synchronized Row.Entry replace(final Row.Entry entry) throws SpaceExceededException {
        assert (entry != null);
        finishInitialization();
        // if the new entry is within the initialization part, just overwrite it
        assert this.index0.isSorted();
        final byte[] key = entry.getPrimaryKeyBytes();
        if (this.index0.has(key)) {
            // replace the entry
            return this.index0.replace(entry);
        }
        // else place it in the index1
        return this.index1.replace(entry);
    }

	/**
     * Adds the row to the index. The row is identified by the primary key of the row.
     * @param row a index row
     * @return true if this set did _not_ already contain the given row.
     * @throws IOException
     * @throws SpaceExceededException
     */
	@Override
    public final boolean put(final Row.Entry entry) throws SpaceExceededException {
        assert (entry != null);
        if (entry == null) return true;
        synchronized (this) {
            finishInitialization();
            // if the new entry is within the initialization part, just overwrite it
            assert this.index0.isSorted();
            final byte[] key = entry.getPrimaryKeyBytes();
            if (this.index0.has(key)) {
                // replace the entry
                this.index0.put(entry);
                return false;
            }
            // else place it in the index1
            return this.index1.put(entry);
        }
    }

    @Override
    public final void addUnique(final Row.Entry entry) throws SpaceExceededException {
    	assert (entry != null);
    	if (entry == null) return;
    	synchronized (this) {
            if (this.index1 == null) {
                // we are in the initialization phase
            	this.index0.addUnique(entry);
            	return;
            }
            // initialization is over, add to secondary index
            this.index1.addUnique(entry);
    	}
    }

	public final void addUnique(final List<Entry> rows) throws SpaceExceededException {
		final Iterator<Entry> i = rows.iterator();
		while (i.hasNext()) addUnique(i.next());
	}

	public final synchronized long inc(final byte[] key, final int col, final long add, final Row.Entry initrow) throws SpaceExceededException {
        assert (key != null);
        finishInitialization();
        assert this.index0.isSorted();
        final long l = this.index0.inc(key, col, add, null);
        if (l != Long.MIN_VALUE) return l;
        return this.index1.inc(key, col, add, initrow);
    }

    @Override
    public final synchronized ArrayList<RowCollection> removeDoubles() throws SpaceExceededException {
	    // finish initialization phase explicitely
        this.index0.sort();
	    if (this.index1 == null) {
	        return this.index0.removeDoubles();
	    }
	    final ArrayList<RowCollection> d0 = this.index0.removeDoubles();
	    final ArrayList<RowCollection> d1 = this.index1.removeDoubles();
        d0.addAll(d1);
        return d0;
	}

    @Override
    public final synchronized boolean delete(final byte[] key) {
        finishInitialization();
        // if the new entry is within the initialization part, just delete it
        boolean b = this.index0.delete(key);
        if (b) {
            assert !this.index0.has(key); // check if remove worked
            return true;
        }
        // else remove it from the index1
        b = this.index1.delete(key);
        assert this.index1.has(key) : "removed " + ((b) ? " true" : " false") + ", and index entry still exists"; // check if remove worked
        return b;
    }

    @Override
    public final synchronized Row.Entry remove(final byte[] key) {
        finishInitialization();
        // if the new entry is within the initialization part, just delete it
        int s = this.index0.size();
        final Row.Entry indexentry = this.index0.remove(key);
        if (indexentry != null) {
            assert this.index0.size() < s: "s = " + s + ", index0.size() = " + this.index0.size();
            assert !this.index0.has(key); // check if remove worked
            return indexentry;
        }
        // else remove it from the index1
        s = this.index1.size();
        final Row.Entry removed = this.index1.remove(key);
        assert removed == null || this.index1.size() < s: "s = " + s + ", index1.size() = " + this.index1.size();
        assert !this.index1.has(key) : "removed " + ((removed == null) ? " is null" : " is not null") + ", and index entry still exists"; // check if remove worked
        return removed;
    }

    @Override
    public final synchronized Row.Entry removeOne() {
        if (this.index1 != null && !this.index1.isEmpty()) {
            return this.index1.removeOne();
        }
        if (this.index0 != null && !this.index0.isEmpty()) {
        	return this.index0.removeOne();
        }
        return null;
    }

    @Override
    public synchronized List<Row.Entry> top(final int count) throws IOException {
        final List<Row.Entry> list = new ArrayList<Row.Entry>();
        List<Row.Entry> list0 = this.index1.top(count);
        list.addAll(list0);
        list0 = this.index0.top(count - list.size());
        list.addAll(list0);
        return list;
    }
    
    @Override
    public synchronized List<Row.Entry> random(final int count) throws IOException {
        final List<Row.Entry> list = new ArrayList<Row.Entry>();
        List<Row.Entry> list0 = this.index1.random(count);
        list.addAll(list0);
        list0 = this.index0.random(count - list.size());
        list.addAll(list0);
        return list;
    }

    @Override
    public long mem() {
        if (this.index0 != null && this.index1 == null) {
            return this.index0.mem();
        }
        if (this.index0 == null && this.index1 != null) {
            return this.index1.mem();
        }
        assert (this.index0 != null && this.index1 != null);
        return this.index0.mem() + this.index1.mem();
    }

    @Override
    public final int size() {
        if (this.index0 != null && this.index1 == null) {
            return this.index0.size();
        }
        if (this.index0 == null && this.index1 != null) {
            return this.index1.size();
        }
        assert (this.index0 != null && this.index1 != null);
        return this.index0.size() + this.index1.size();
    }

    @Override
    public final boolean isEmpty() {
        if (this.index0 != null && this.index1 == null) {
            return this.index0.isEmpty();
        }
        if (this.index0 == null && this.index1 != null) {
            return this.index1.isEmpty();
        }
        assert (this.index0 != null && this.index1 != null);
        if (!this.index0.isEmpty()) return false;
        if (!this.index1.isEmpty()) return false;
        return true;
    }


    @Override
    public final synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) {
        // returns the key-iterator of the underlying kelondroIndex
        if (this.index1 == null) {
            // finish initialization phase
            this.index0.sort();
            this.index0.uniq();
            this.index1 = new RowSet(this.rowdef); //new RowSetArray(rowdef, spread);
            return this.index0.keys(up, firstKey);
        }
        assert (this.index1 != null);
        if (this.index0 == null) {
            //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
            return this.index1.keys(up, firstKey);
        }
        // index0 should be sorted
        // sort index1 to enable working of the merge iterator
        //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
        final CloneableIterator<byte[]> k0 = this.index0.keys(up, firstKey);
        final CloneableIterator<byte[]> k1 = this.index1.keys(up, firstKey);
        if (k0 == null) return k1;
        if (k1 == null) return k0;
        return new MergeIterator<byte[]>(
                k0,
                k1,
                this.rowdef.objectOrder,
                MergeIterator.simpleMerge,
                up);
    }

    @Override
    public final synchronized CloneableIterator<Row.Entry> rows(final boolean up, final byte[] firstKey) {
        // returns the row-iterator of the underlying kelondroIndex
        if (this.index1 == null) {
            // finish initialization phase
            this.index0.sort();
            this.index0.uniq();
            this.index1 = new RowSet(this.rowdef); //new RowSetArray(rowdef, spread);
            return this.index0.rows(up, firstKey);
        }
        assert (this.index1 != null);
        if (this.index0 == null) {
            //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
            return this.index1.rows(up, firstKey);
        }
        // index0 should be sorted
        // sort index1 to enable working of the merge iterator
        //index1.sort();
        //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
        final CloneableIterator<Row.Entry> k0 = this.index0.rows(up, firstKey);
        final CloneableIterator<Row.Entry> k1 = this.index1.rows(up, firstKey);
        if (k0 == null) return k1;
        if (k1 == null) return k0;
        return new MergeIterator<Row.Entry>(
                k0,
                k1,
                this.entryComparator,
                MergeIterator.simpleMerge,
                up);
    }

    @Override
    public final Iterator<Entry> iterator() {
        return rows();
    }

    @Override
    public final synchronized CloneableIterator<Row.Entry> rows() {
        // returns the row-iterator of the underlying kelondroIndex
        if (this.index1 == null) {
            // finish initialization phase
            this.index0.sort();
            this.index0.uniq();
            this.index1 = new RowSet(this.rowdef); //new RowSetArray(rowdef, spread);
            return this.index0.rows();
        }
        assert (this.index1 != null);
        if (this.index0 == null) {
            //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
            return this.index1.rows();
        }
        // index0 should be sorted
        // sort index1 to enable working of the merge iterator
        //index1.sort();
        //assert consistencyAnalysis0() : "consistency problem: " + consistencyAnalysis();
        return new StackIterator<Row.Entry>(this.index0.rows(), this.index1.rows(), null, true);
    }

    @Override
    public final synchronized void close() {
        if (this.index0 != null) this.index0.close();
        if (this.index1 != null) this.index1.close();
        objectTracker.remove(this.name);
    }

	@Override
    public final String filename() {
		return null; // this does not have a file name
	}

	@Override
    public final void deleteOnExit() {
        // do nothing, there is no file
    }

}