yacy_search_server/source/net/yacy/peers/SeedDB.java

// yacySeedDB.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

package net.yacy.peers;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.federate.yacy.Distribution;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.blob.MapDataMining;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.kelondroException;
import net.yacy.peers.operation.yacySeedUploader;
import net.yacy.search.Switchboard;
import net.yacy.server.serverCore;
import net.yacy.server.serverSwitch;
import net.yacy.server.http.AlternativeDomainNames;

public final class SeedDB implements AlternativeDomainNames {

    // global statics

    private static final int dhtActivityMagic = 32;

    /**
     * <p><code>public static final String <strong>DBFILE_OWN_SEED</strong> = "mySeed.txt"</code></p>
     * <p>Name of the file containing the database holding this peer's seed</p>
     */
    public static final String DBFILE_OWN_SEED = "mySeed.txt";

    public static final String[]      sortFields = new String[] {Seed.LCOUNT, Seed.RCOUNT, Seed.ICOUNT, Seed.UPTIME, Seed.VERSION, Seed.LASTSEEN};
    public static final String[]   longaccFields = new String[] {Seed.LCOUNT, Seed.ICOUNT, Seed.ISPEED};
    public static final String[] doubleaccFields = new String[] {Seed.RSPEED};

    // class objects
    private File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
    private File myOwnSeedFile;
    private MapDataMining seedActiveDB, seedPassiveDB, seedPotentialDB;

    protected int lastSeedUpload_seedDBSize = 0;
    public long lastSeedUpload_timeStamp = System.currentTimeMillis();
    protected String lastSeedUpload_myIP = "";

    public  PeerActions peerActions;
    public  NewsPool newsPool;

    private int netRedundancy;
    public  Distribution scheme;

    private Seed mySeed; // my own seed

    public SeedDB(
            final File networkRoot,
            final String seedActiveDBFileName,
            final String seedPassiveDBFileName,
            final String seedPotentialDBFileName,
            final File myOwnSeedFile,
            final int redundancy,
            final int partitionExponent,
            final boolean useTailCache,
            final boolean exceed134217727) {
        this.seedActiveDBFile = new File(networkRoot, seedActiveDBFileName);
        this.seedPassiveDBFile = new File(networkRoot, seedPassiveDBFileName);
        this.seedPotentialDBFile = new File(networkRoot, seedPotentialDBFileName);
        this.mySeed = null; // my own seed
        this.myOwnSeedFile = myOwnSeedFile;
        this.netRedundancy = redundancy;
        this.scheme = new Distribution(partitionExponent);

        // set up seed database
        this.seedActiveDB = openSeedTable(this.seedActiveDBFile);
        this.seedPassiveDB = openSeedTable(this.seedPassiveDBFile);
        this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);

        // check if we are in the seedCaches: this can happen if someone else published our seed
        removeMySeed();

        this.lastSeedUpload_seedDBSize = sizeConnected();

        // create or init news database
        this.newsPool = new NewsPool(networkRoot, useTailCache, exceed134217727);

        // deploy peer actions
        this.peerActions = new PeerActions(this, this.newsPool);
    }

    public void relocate(
            final File newNetworkRoot,
            final int redundancy,
            final int partitionExponent,
            final boolean useTailCache,
            final boolean exceed134217727) {

        // close old databases
        this.seedActiveDB.close();
        this.seedPassiveDB.close();
        this.seedPotentialDB.close();
        this.newsPool.close();
        this.peerActions.close();

        // open new according to the newNetworkRoot
        this.seedActiveDBFile = new File(newNetworkRoot, this.seedActiveDBFile.getName());
        this.seedPassiveDBFile = new File(newNetworkRoot, this.seedPassiveDBFile.getName());
        this.seedPotentialDBFile = new File(newNetworkRoot, this.seedPotentialDBFile.getName());

        // replace my (old) seed with new seed definition from other network
        // but keep the seed name
        final String peername = myName();
        this.mySeed = null; // my own seed
        this.myOwnSeedFile = new File(newNetworkRoot, SeedDB.DBFILE_OWN_SEED);

        this.netRedundancy = redundancy;
        this.scheme = new Distribution(partitionExponent);

        // set up seed database
        this.seedActiveDB = openSeedTable(this.seedActiveDBFile);
        this.seedPassiveDB = openSeedTable(this.seedPassiveDBFile);
        this.seedPotentialDB = openSeedTable(this.seedPotentialDBFile);

        initMySeed();
        this.mySeed.setName(peername);

        // check if we are in the seedCaches: this can happen if someone else published our seed
        removeMySeed();

        this.lastSeedUpload_seedDBSize = sizeConnected();

        // create or init news database
        this.newsPool = new NewsPool(newNetworkRoot, useTailCache, exceed134217727);

        // deploy peer actions
        this.peerActions = new PeerActions(this, this.newsPool);
    }

    private synchronized void initMySeed() {
        if (this.mySeed != null) return;

        // create or init own seed
        if (this.myOwnSeedFile.length() > 0) try {
            // load existing identity
            this.mySeed = Seed.load(this.myOwnSeedFile);
            if (this.mySeed == null) throw new IOException("current seed is null");
        } catch (final IOException e) {
            // create new identity
            ConcurrentLog.severe("SEEDDB", "could not load stored mySeed.txt from " + this.myOwnSeedFile.toString() + ": " + e.getMessage() + ". creating new seed.", e);
            this.mySeed = Seed.genLocalSeed(this);
            try {
                this.mySeed.save(this.myOwnSeedFile);
            } catch (final IOException ee) {
                ConcurrentLog.severe("SEEDDB", "error saving mySeed.txt (1) to " + this.myOwnSeedFile.toString() + ": " + ee.getMessage(), ee);
                ConcurrentLog.logException(ee);
                System.exit(-1);
            }
        } else {
            // create new identity
            ConcurrentLog.info("SEEDDB", "could not find stored mySeed.txt at " + this.myOwnSeedFile.toString() + ": " + ". creating new seed.");
            this.mySeed = Seed.genLocalSeed(this);
            try {
                this.mySeed.save(this.myOwnSeedFile);
            } catch (final IOException ee) {
                ConcurrentLog.severe("SEEDDB", "error saving mySeed.txt (2) to " + this.myOwnSeedFile.toString() + ": " + ee.getMessage(), ee);
                ConcurrentLog.logException(ee);
                System.exit(-1);
            }
        }
        this.mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs());
        this.mySeed.put(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); // markup startup condition
    }

    public int redundancy() {
        if (this.mySeed.isVirgin()) return 1;
        return this.netRedundancy;
    }

    public boolean mySeedIsDefined() {
        return this.mySeed != null;
    }

    public Seed mySeed() {
        if (this.mySeed == null) {
            if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
            initMySeed();
        }
        return this.mySeed;
    }

    public void setMyName(final String name) {
        this.mySeed.setName(name);
    }

    @Override
    public String myAlternativeAddress() {
        return mySeed().getName() + ".yacy";
    }

    @Deprecated
    public String myIP() {
        return mySeed().getIP();
    }
    
    @Override
    public Set<String> myIPs() {
        return mySeed().getIPs();
    }

    @Override
    public int myPort() {
        return mySeed().getPort();
    }

    @Override
    public String myName() {
        return this.mySeed.getName();
    }

    @Override
    public String myID() {
        return this.mySeed.hash;
    }

    public synchronized void removeMySeed() {
        if (this.seedActiveDB.isEmpty() && this.seedPassiveDB.isEmpty() && this.seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
        if (this.mySeed == null) initMySeed();
        try {
            final byte[] mySeedHash = ASCII.getBytes(this.mySeed.hash);
            this.seedActiveDB.delete(mySeedHash);
            this.seedPassiveDB.delete(mySeedHash);
            this.seedPotentialDB.delete(mySeedHash);
        } catch (final IOException e) { ConcurrentLog.warn("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
    }

    public void saveMySeed() {
        try {
          mySeed().save(this.myOwnSeedFile);
        } catch (final IOException e) { ConcurrentLog.warn("yacySeedDB", "could not save mySeed '"+ this.myOwnSeedFile +"': "+ e.getMessage()); }
    }

    public boolean noDHTActivity() {
        // for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
        return sizeConnected() <= dhtActivityMagic;
    }

    private synchronized static MapDataMining openSeedTable(final File seedDBFile) {
        final File parentDir = new File(seedDBFile.getParent());
        if (!parentDir.exists()) {
			if(!parentDir.mkdirs())
				ConcurrentLog.warn("yacySeedDB", "could not create directories for "+ seedDBFile.getParent());
		}
        try {
            return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields);
        } catch (final Exception e) {
            // try again
            FileUtils.deletedelete(seedDBFile);
            try {
                return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields);
            } catch (final IOException e1) {
                ConcurrentLog.logException(e1);
                System.exit(-1);
                return null;
            }
        }
    }

    private synchronized MapDataMining resetSeedTable(MapDataMining seedDB, final File seedDBFile) {
        // this is an emergency function that should only be used if any problem with the
        // seed.db is detected
        Network.log.warn("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
        seedDB.close();
        FileUtils.deletedelete(seedDBFile);
        if (seedDBFile.exists())
        	ConcurrentLog.warn("yacySeedDB", "could not delete file "+ seedDBFile);
        // create new seed database
        seedDB = openSeedTable(seedDBFile);
        return seedDB;
    }

    public synchronized void resetActiveTable() { this.seedActiveDB = resetSeedTable(this.seedActiveDB, this.seedActiveDBFile); }
    private synchronized void resetPassiveTable() { this.seedPassiveDB = resetSeedTable(this.seedPassiveDB, this.seedPassiveDBFile); }
    private synchronized void resetPotentialTable() { this.seedPotentialDB = resetSeedTable(this.seedPotentialDB, this.seedPotentialDBFile); }

    public void close() {
        if (this.seedActiveDB != null) this.seedActiveDB.close();
        if (this.seedPassiveDB != null) this.seedPassiveDB.close();
        if (this.seedPotentialDB != null) this.seedPotentialDB.close();
        this.newsPool.close();
        this.peerActions.close();
    }

    public Iterator<Seed> seedsSortedConnected(final boolean up, final String field) {
        // enumerates seed-type objects: all seeds sequentially ordered by field
        return new seedEnum(up, field, this.seedActiveDB);
    }

    public Iterator<Seed> seedsSortedDisconnected(final boolean up, final String field) {
        // enumerates seed-type objects: all seeds sequentially ordered by field
        return new seedEnum(up, field, this.seedPassiveDB);
    }

    public Iterator<Seed> seedsSortedPotential(final boolean up, final String field) {
        // enumerates seed-type objects: all seeds sequentially ordered by field
        return new seedEnum(up, field, this.seedPotentialDB);
    }

    public TreeSet<byte[]> /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) {
    	// collects seeds according to cluster definition string, which consists of
    	// comma-separated .yacy or .yacyh-domains
    	// the domain may be extended by an alternative address specification of the form
    	// <ip> or <ip>:<port>. The port must be identical to the port specified in the peer seed,
    	// therefore it is optional. The address specification is separated by a '='; the complete
    	// address has therefore the form
    	// address    ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
    	// clusterdef ::= {address}{','address}*
    	final String[] addresses = (clusterdefinition.isEmpty()) ? new String[0] : clusterdefinition.split(",");
    	final TreeSet<byte[]> clustermap = new TreeSet<>(Base64Order.enhancedCoder);
    	Seed seed;
    	String hash, yacydom;
    	int p;
    	for (final String addresse : addresses) {
    		p = addresse.indexOf('=');
    		if (p >= 0) {
    			yacydom = addresse.substring(0, p);
    		} else {
    			yacydom = addresse;
    		}
    		if (yacydom.endsWith(".yacyh")) {
    			// find a peer with its hexhash
    			hash = Seed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));
    			seed = get(hash);
    			if (seed == null) {
    				Network.log.warn("cluster peer '" + yacydom + "' was not found.");
    			} else {
    				clustermap.add(ASCII.getBytes(hash));
    			}
    		} else if (yacydom.endsWith(".yacy")) {
    			// find a peer with its name
    			seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));
    			if (seed == null) {
    				Network.log.warn("cluster peer '" + yacydom + "' was not found.");
    			} else {
    				clustermap.add(ASCII.getBytes(seed.hash));
    			}
    		} else {
    			Network.log.warn("cluster peer '" + addresse + "' has wrong syntax. the name must end with .yacy or .yacyh");
    		}
    	}
    	return clustermap;
    }

    public Iterator<Seed> seedsConnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) {
        // enumerates seed-type objects: all seeds sequentially without order
        return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedActiveDB, minVersion);
    }

    public Iterator<Seed> seedsDisconnected(final boolean up, final boolean rot, final byte[] firstHash, final double minVersion) {
        // enumerates seed-type objects: all seeds sequentially without order
        return new seedEnum(up, rot, (firstHash == null) ? null : firstHash, null, this.seedPassiveDB, minVersion);
    }

    public Seed anySeedVersion(final double minVersion) {
        // return just any seed that has a specific minimum version number
        final Iterator<Seed> e = seedsConnected(true, true, Seed.randomHash(), minVersion);
        return e.next();
    }

    /**
     * count the number of peers that had been seen within the given time limit
     * @param limitMinutes the time limit in minutes. 1440 minutes is a day
     * @return the number of peers seen in the given time
     */
    public int sizeActiveSince(final long limitMinutes) {
        int c = this.seedActiveDB.size();
        Seed seed;
        Iterator<Seed> i = seedsSortedDisconnected(false, Seed.LASTSEEN);
        while (i.hasNext()) {
            seed = i.next();
            if (seed != null) {
                if (Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60) > limitMinutes) break;
                c++;
            }
        }
        i = seedsSortedPotential(false, Seed.LASTSEEN);
        while (i.hasNext()) {
            seed = i.next();
            if (seed != null) {
                if (Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60) > limitMinutes) break;
                c++;
            }
        }
        return c;
    }

     public int sizeConnected() {
        return this.seedActiveDB.size();
    }

    public int sizeDisconnected() {
        return this.seedPassiveDB.size();
    }

    public int sizePotential() {
        return this.seedPotentialDB.size();
    }

    public long countActiveURL() { return this.seedActiveDB.getLongAcc(Seed.LCOUNT); }
    public long countActiveRWI() { return this.seedActiveDB.getLongAcc(Seed.ICOUNT); }
    public long countActivePPM() { return this.seedActiveDB.getLongAcc(Seed.ISPEED); }
    public float countActiveQPM() { return this.seedActiveDB.getFloatAcc(Seed.RSPEED); }
    public long countPassiveURL() { return this.seedPassiveDB.getLongAcc(Seed.LCOUNT); }
    public long countPassiveRWI() { return this.seedPassiveDB.getLongAcc(Seed.ICOUNT); }
    public long countPotentialURL() { return this.seedPotentialDB.getLongAcc(Seed.LCOUNT); }
    public long countPotentialRWI() { return this.seedPotentialDB.getLongAcc(Seed.ICOUNT); }
    
    public void addConnected(final Seed seed) {
        if (seed.isProper(false) != null) return;
        //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
        final ConcurrentMap<String, String> seedPropMap = seed.getMap();
        synchronized (this) {
            try {
                this.seedActiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
                this.seedPassiveDB.delete(ASCII.getBytes(seed.hash));
                this.seedPotentialDB.delete(ASCII.getBytes(seed.hash));
            } catch (final Exception e) {
                Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                resetActiveTable();
            }
        }
    }

    protected void addDisconnected(final Seed seed) {
        if (seed.isProper(false) != null) return;
        synchronized (this) {
            try {
                this.seedActiveDB.delete(ASCII.getBytes(seed.hash));
                this.seedPotentialDB.delete(ASCII.getBytes(seed.hash));
            } catch (final Exception e) { ConcurrentLog.warn("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
            //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
            try {
                final ConcurrentMap<String, String> seedPropMap = seed.getMap();
                this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
            } catch (final Exception e) {
                Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                resetPassiveTable();
            }
        }
    }

    protected void addPotential(final Seed seed) {
        if (seed.isProper(false) != null) return;
        final ConcurrentMap<String, String> seedPropMap = seed.getMap();
        synchronized (this) {
            try {
                this.seedActiveDB.delete(ASCII.getBytes(seed.hash));
                this.seedPassiveDB.delete(ASCII.getBytes(seed.hash));
            } catch (final Exception e) { ConcurrentLog.warn("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
            //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
            try {
                this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
            } catch (final Exception e) {
                Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                resetPotentialTable();
            }
        }
    }

    public synchronized void removeDisconnected(final String peerHash) {
    	if (peerHash == null) return;
    	try {
			this.seedPassiveDB.delete(ASCII.getBytes(peerHash));
		} catch (final IOException e) { ConcurrentLog.warn("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
    }

    public synchronized void removePotential(final String peerHash) {
    	if (peerHash == null) return;
    	try {
			this.seedPotentialDB.delete(ASCII.getBytes(peerHash));
		} catch (final IOException e) { ConcurrentLog.warn("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
    }

    public boolean hasConnected(final byte[] hash) {
        return this.seedActiveDB.containsKey(hash);
    }

    public boolean hasDisconnected(final byte[] hash) {
        return this.seedPassiveDB.containsKey(hash);
    }

    public boolean hasPotential(final byte[] hash) {
        return this.seedPotentialDB.containsKey(hash);
    }

    private Seed get(final String hash, final MapDataMining database) {
        if (hash == null || hash.isEmpty()) return null;
        if ((this.mySeed != null) && (hash.equals(this.mySeed.hash))) return this.mySeed;
        final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
        try {
            final Map<String, String> map = database.get(ASCII.getBytes(hash));
            if (map == null) return null;
            entry.putAll(map);
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
            return null;
        } catch (final SpaceExceededException e) {
            ConcurrentLog.logException(e);
            return null;
        }
        return new Seed(hash, entry);
    }
    
    private Seed get(final byte[] hash, final MapDataMining database) {
        if (hash == null || hash.length == 0) return null;
        if ((this.mySeed != null) && (hash.equals(this.mySeed.hash))) return this.mySeed;
        final ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
        try {
            final Map<String, String> map = database.get(hash);
            if (map == null) return null;
            entry.putAll(map);
        } catch (final IOException e) {
            ConcurrentLog.logException(e);
            return null;
        } catch (final SpaceExceededException e) {
            ConcurrentLog.logException(e);
            return null;
        }
        return new Seed(ASCII.String(hash), entry);
    }

    public Seed getConnected(final String hash) {
        return get(hash, this.seedActiveDB);
    }

    public Seed getConnected(final byte[] hash) {
        return get(hash, this.seedActiveDB);
    }

    public Seed getDisconnected(final String hash) {
        return get(hash, this.seedPassiveDB);
    }

    public Seed getDisconnected(final byte[] hash) {
        return get(hash, this.seedPassiveDB);
    }

    public Seed getPotential(final String hash) {
        return get(hash, this.seedPotentialDB);
    }

    public Seed getPotential(final byte[] hash) {
        return get(hash, this.seedPotentialDB);
    }

    public Seed get(final String hash) {
        Seed seed = getConnected(hash);
        if (seed == null) seed = getDisconnected(hash);
        if (seed == null) seed = getPotential(hash);
        return seed;
    }

    public void updateConnected(final Seed seed) {
        if (seed.isProper(false) != null) return;
        final ConcurrentMap<String, String> seedPropMap = seed.getMap();
        synchronized (this) {
            if (this.seedActiveDB.containsKey(ASCII.getBytes(seed.hash))) try {
                this.seedActiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
            } catch (final Exception e) {
                Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                resetActiveTable();
            }
        }
    }
    public void updateDisconnected(final Seed seed) {
        if (seed.isProper(false) != null) return;
        final ConcurrentMap<String, String> seedPropMap = seed.getMap();
        synchronized (this) {
            if (this.seedPassiveDB.containsKey(ASCII.getBytes(seed.hash))) try {
                this.seedPassiveDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
            } catch (final Exception e) {
                Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                resetActiveTable();
            }
        }
    }
    public void updatePotential(final Seed seed) {
        if (seed.isProper(false) != null) return;
        final ConcurrentMap<String, String> seedPropMap = seed.getMap();
        synchronized (this) {
            if (this.seedPotentialDB.containsKey(ASCII.getBytes(seed.hash))) try {
                this.seedPotentialDB.insert(ASCII.getBytes(seed.hash), seedPropMap);
            } catch (final Exception e) {
                Network.log.severe("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                resetActiveTable();
            }
        }
    }

    public Seed lookupByName(String peerName) {
        // reads a seed by searching by name
        if (peerName.endsWith(".yacy")) peerName = peerName.substring(0, peerName.length() - 5);

        // local peer?
        if (peerName.equals("localpeer")) {
            if (this.mySeed == null) initMySeed();
            return this.mySeed;
        }

        peerName = peerName.toLowerCase();
        Seed seed;

        // enumerate the cache
        String name = Seed.checkPeerName(peerName);
        synchronized (this) { try {
            Collection<byte[]> idx = this.seedActiveDB.select(Seed.NAME, name);
            for (byte[] pk: idx) {
                seed = this.getConnected(pk);
                if (seed == null) continue;
                //System.out.println("*** found lookupByName in seedActiveDB: " + peerName);
                return seed;
            }
        } catch (final  IOException e ) {
        }}
        synchronized (this) { try {
            Collection<byte[]> idx = this.seedPassiveDB.select(Seed.NAME, name);
            for (byte[] pk: idx) {
                seed = this.getDisconnected(pk);
                if (seed == null) continue;
                //System.out.println("*** found lookupByName in seedPassiveDB: " + peerName);
                return seed;
            }
        } catch (final  IOException e ) {
        }}

        // check local seed
        if (this.mySeed == null) initMySeed();
        name = this.mySeed.getName().toLowerCase();
        if (name.equals(peerName)) return this.mySeed;
        // nothing found
        return null;
    }

    public Seed lookupByIPs(
            final Set<String> peerIPs,
            final int port,                 /* port may be -1 if not significant */
            final boolean lookupConnected,
            final boolean lookupDisconnected,
            final boolean lookupPotential
    ) {
        for (String i: peerIPs) {
            Seed s = lookupByIP(Domains.dnsResolve(i), port, lookupConnected, lookupDisconnected, lookupPotential);
            if (s != null) return s;
        }
        return null;
    }
    
    public Seed lookupByIP(
            final InetAddress peerIP,
            final int port,                 /* port may be -1 if not significant */
            final boolean lookupConnected,
            final boolean lookupDisconnected,
            final boolean lookupPotential
    ) {

        if (peerIP == null) return null;

        // local peer?
        if (Domains.isThisHostIP(peerIP)) {
            if (this.mySeed == null) initMySeed();
            if (port > 0 && this.mySeed.getPort() == port) return this.mySeed;
        }

        // then try to use the cache
        Seed seed = null;
        String ipString = peerIP.getHostAddress();

        if (lookupConnected) {
            try {
                Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);
                for (byte[] pk: idx) {
                    seed = this.getConnected(pk);
                    if (seed == null) continue;
                    if (port > 0 && seed.getPort() != port) continue;
                    //System.out.println("*** found lookupByIP in connected: " + peerIP.toString() + " -> " + seed.getName());
                    return seed;
                }
            } catch (final IOException e ) {
                ConcurrentLog.logException(e);
            }
        }

        if (lookupDisconnected) {
            try {
                Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);
                for (byte[] pk: idx) {
                    seed = this.getDisconnected(pk);
                    if (seed == null) continue;
                    if (port > 0 && seed.getPort() != port) continue;
                    //System.out.println("*** found lookupByIP in disconnected: " + peerIP.toString() + " -> " + seed.getName());
                    return seed;
                }
            } catch (final IOException e ) {
                ConcurrentLog.logException(e);
            }
        }

        if (lookupPotential) {
            try {
                Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);
                for (byte[] pk: idx) {
                    seed = this.getPotential(pk);
                    if (seed == null) continue;
                    if (port > 0 && seed.getPort() != port) continue;
                    //System.out.println("*** found lookupByIP in potential: " + peerIP.toString() + " -> " + seed.getName());
                    return seed;
                }
            } catch (final IOException e ) {
                ConcurrentLog.logException(e);
            }
        }

        // check local seed
        if (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null;
        int p = this.mySeed.getPort();
        if (port > 0 && p != port) return null;
        //System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
        return this.mySeed;
    }

    private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {
        PrintWriter pw = null;
        final ArrayList<String> v = new ArrayList<String>(this.seedActiveDB.size() + 1);
        try {

            pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
            List<Seed> seedlist = getSeedlist(Integer.MAX_VALUE, addMySeed, false, 0.0f);
            String line;
            for (Seed seed: seedlist) {
                line = seed.genSeedStr(null);
                v.add(line);
                pw.print(line + serverCore.CRLF_STRING);
            }
            pw.flush();
        } finally {
            if (pw != null) try { pw.close(); } catch (final Exception e) {}
        }
        return v;
    }

    public ArrayList<Seed> getSeedlist(int maxcount, boolean addMySeed, boolean nodeonly, float minversion) {
        final ArrayList<Seed> v = new ArrayList<Seed>(this.seedActiveDB.size() + 1000);
 
        // store own peer seed
        if (addMySeed) v.add(this.mySeed);

        // store active peer seeds
        Seed ys;
        Iterator<Seed> se = this.seedsConnected(true, false, null, minversion);
        while (se.hasNext() && v.size() < maxcount) {
            ys = se.next();
            if (ys != null && (!nodeonly || ys.getFlagRootNode())) v.add(ys);
        }

        // store some of the not-so-old passive peer seeds (limit: 1 day)
        se = this.seedsDisconnected(true, false, null, (float) 0.0);
        final long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
        while (se.hasNext() && v.size() < maxcount) {
            ys = se.next();
            if (ys != null && ys.getLastSeenUTC() >= timeout && (!nodeonly || ys.getFlagRootNode())) v.add(ys);
        }

        final StringBuilder encoded = new StringBuilder(1024);
        for (Seed seed: v) {
            encoded.append(seed.genSeedStr(null)).append(serverCore.CRLF_STRING);
        }
        return v;
    }
    
    protected String uploadSeedList(final yacySeedUploader uploader,
            final serverSwitch sb,
            final SeedDB seedDB,
            final DigestURL seedURL) throws Exception {

        // upload a seed file, if possible
        if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given");

        String log = null;
        File seedFile = null;
        try {
            // create a seed file which for uploading ...
            seedFile = File.createTempFile("seedFile",".txt", seedDB.myOwnSeedFile.getParentFile());
            seedFile.deleteOnExit();
            if (Network.log.isFine()) Network.log.fine("SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());
            final ArrayList<String> uv = storeSeedList(seedFile, true);

            // uploading the seed file
            if (Network.log.isFine()) Network.log.fine("SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
            log = uploader.uploadSeedFile(sb, seedFile);

            // test download
            if (Network.log.isFine()) Network.log.fine("SaveSeedList: Trying to download seed-file '" + seedURL + "'.");
            final Iterator<String> check = downloadSeedFile(seedURL);

            // Comparing if local copy and uploaded copy are equal
            final String errorMsg = checkCache(uv, check);
            if (errorMsg == null)
                log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.CRLF_STRING;
            else {
                throw new Exception("UPLOAD CHECK - Error: the result vector is different. " + errorMsg + serverCore.CRLF_STRING);
            }
        } finally {
            if (seedFile != null)
				try {
				    FileUtils.deletedelete(seedFile);
				} catch (final Exception e) {
					/* ignore this */
				}
        }

        return log;
    }

    private static Iterator<String> downloadSeedFile(final DigestURL seedURL) throws IOException {
        // Configure http headers
        final RequestHeader reqHeader = new RequestHeader();
        reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
        reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
        reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);

        final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
        client.setHeader(reqHeader.entrySet());
        byte[] content = null;
        try {
            // send request
        	content = client.GETbytes(seedURL, null, null, false);
        } catch (final Exception e) {
        	throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
        }

        // check response code
        if (client.getHttpResponse().getStatusLine().getStatusCode() != 200) {
        	throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine());
        }

        try {
            // uncompress it if it is gzipped
            content = FileUtils.uncompressGZipArray(content);

            // convert it into an array
            return FileUtils.strings(content);
        } catch (final Exception e) {
        	throw new IOException("Unable to uncompress seed file '" + seedURL + "'. " + e.getMessage());
        }
    }

    private static String checkCache(final ArrayList<String> uv, final Iterator<String> check) {
        if ((check == null) || (uv == null)) {
            if (Network.log.isFine()) Network.log.fine("SaveSeedList: Local and uploades seed-list are different");
            return "Entry count is different: uv.size() = " + ((uv == null) ? "null" : Integer.toString(uv.size()));
        }

        if (Network.log.isFine()) Network.log.fine("SaveSeedList: Comparing local and uploades seed-list entries ...");
        int i = 0;
        while (check.hasNext() && i < uv.size()) {
        	if (!((uv.get(i)).equals(check.next()))) return "Element at position " + i + " is different.";
        	i++;
        }

        // no difference found
        return null;
    }

    /**
     * resolve a yacy address
     */
    @Override
    public String resolve(String host) {
        Seed seed;
        int p;
        String subdom = null;
        if (host.endsWith(".yacyh")) {
            // this is not functional at the moment
            // caused by lowecasing of hashes at the browser client
            p = host.indexOf('.');
            if ((p > 0) && (p != (host.length() - 6))) {
                subdom = host.substring(0, p);
                host = host.substring(p + 1);
            }
            // check if we have a b64-hash or a hex-hash
            String hash = host.substring(0, host.length() - 6);
            if (hash.length() > Word.commonHashLength) {
                // this is probably a hex-hash
                hash = Seed.hexHash2b64Hash(hash);
            }
            // check remote seeds
            seed = getConnected(hash); // checks only remote, not local
            // check local seed
            if (seed == null) {
                if (this.mySeed == null) initMySeed();
                if (hash.equals(this.mySeed.hash))
                    seed = this.mySeed;
                else return null;
            }
            return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
        } else if (host.endsWith(".yacy")) {
            // identify subdomain
            p = host.indexOf('.');
            if ((p > 0) && (p != (host.length() - 5))) {
                subdom = host.substring(0, p); // no double-dot attack possible, the subdom cannot have ".." in it
                host = host.substring(p + 1); // if ever, the double-dots are here but do not harm
            }
            // identify domain
            final String domain = host.substring(0, host.length() - 5).toLowerCase();
            seed = lookupByName(domain);
            if (seed == null) return null;
            if (this.mySeed == null) initMySeed();
            if (seed == this.mySeed && !(seed.isOnline())) {
                // take local ip instead of external
                return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getLocalPort("port", 8090) + ((subdom == null) ? "" : ("/" + subdom));
            }
            return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
        } else {
            return null;
        }
    }

    private class seedEnum implements Iterator<Seed> {

        private Iterator<Map.Entry<byte[], Map<String, String>>> it;
        private Seed nextSeed;
        private final MapDataMining database;
        private double minVersion;

        private seedEnum(final boolean up, final boolean rot, final byte[] firstKey, final byte[] secondKey, final MapDataMining database, final double minVersion) {
            this.database = database;
            this.minVersion = minVersion;
            try {
                this.it = (firstKey == null) ? database.entries(up, rot) : database.entries(up, rot, firstKey, secondKey);
                double version;
                while (true) {
                    this.nextSeed = internalNext();
                    if (this.nextSeed == null) break;
                    version = this.nextSeed.getVersion();
                    if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers
                }
            } catch (final IOException e) {
                ConcurrentLog.logException(e);
                Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
                if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
                this.it = null;
            } catch (final kelondroException e) {
                ConcurrentLog.logException(e);
                Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
                if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
                this.it = null;
            }
        }

        private seedEnum(final boolean up, final String field, final MapDataMining database) {
            this.database = database;
            try {
                this.it = database.entries(up, field);
                this.nextSeed = internalNext();
            } catch (final kelondroException e) {
                ConcurrentLog.logException(e);
                Network.log.severe("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                if (database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
                if (database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
                if (database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);
                this.it = null;
            }
        }

        @Override
        public boolean hasNext() {
            return (this.nextSeed != null);
        }

        private Seed internalNext() {
            if (this.it == null || !(this.it.hasNext())) return null;
            try {
                Map.Entry<byte[], Map<String, String>> dna0;
                ConcurrentHashMap<String, String> dna;
                while (this.it.hasNext()) {
                    try {
                        dna0 = this.it.next();
                    } catch (final OutOfMemoryError e) {
                        ConcurrentLog.logException(e);
                        dna0 = null;
                    }
                    assert dna0 != null;
                    if (dna0 == null) continue;
                    if (dna0.getValue() instanceof ConcurrentHashMap) {
                        dna = (ConcurrentHashMap<String, String>) dna0.getValue();
                    } else {
                        dna = new ConcurrentHashMap<String, String>();
                        dna.putAll(dna0.getValue());
                    }
                    if (dna0.getKey() == null) continue; // bad seed
                    final String hash = UTF8.String(dna0.getKey());
                    return new Seed(hash, dna);
                }
                return null;
            } catch (final Exception e) {
                ConcurrentLog.logException(e);
                Network.log.severe("ERROR internalNext: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
                if (this.database == SeedDB.this.seedActiveDB) SeedDB.this.seedActiveDB = resetSeedTable(SeedDB.this.seedActiveDB, SeedDB.this.seedActiveDBFile);
                if (this.database == SeedDB.this.seedPassiveDB) SeedDB.this.seedPassiveDB = resetSeedTable(SeedDB.this.seedPassiveDB, SeedDB.this.seedPassiveDBFile);
                if (this.database == SeedDB.this.seedPotentialDB) SeedDB.this.seedPotentialDB = resetSeedTable(SeedDB.this.seedPotentialDB, SeedDB.this.seedPotentialDBFile);
                return null;
            }
        }

        @Override
        public Seed next() {
            final Seed seed = this.nextSeed;
            double version;
            try {while (true) {
                this.nextSeed = internalNext();
                if (this.nextSeed == null) break;
                version = this.nextSeed.getVersion();
                if (version >= this.minVersion || version == 0.0) break; // include 0.0 to access always developer peers
            }} catch (final kelondroException e) {
                ConcurrentLog.logException(e);
            	// emergency reset
            	Network.log.severe("seed-db emergency reset", e);
            	this.database.clear();
				this.nextSeed = null;
				return null;
            }
            return seed;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }

    }

}