// yacySeedDB.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@anomic.de
// first published on http://www.anomic.de
// Frankfurt, Germany, 2004, 2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
package de.anomic.yacy;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.ref.SoftReference;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.http.httpd;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.nxTools;
public final class yacySeedDB {
// global statics
/**
* this is the lenght(12) of the hash key that is used:<br>
* - for seed hashes (this Object)<br>
* - for word hashes (plasmaIndexEntry.wordHashLength)<br>
* - for L-URL hashes (plasmaLURL.urlHashLength)<br><br>
* these hashes all shall be generated by base64.enhancedCoder
*/
public static final int commonHashLength = 12;
public static final int dhtActivityMagic = 32;
public static final String[] sortFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.UPTIME, yacySeed.VERSION, yacySeed.LASTSEEN};
public static final String[] longaccFields = new String[] {yacySeed.LCOUNT, yacySeed.ICOUNT, yacySeed.ISPEED};
public static final String[] doubleaccFields = new String[] {yacySeed.RSPEED};
// class objects
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
private final plasmaSwitchboard sb;
private yacySeed mySeed; // my own seed
private final Hashtable<String, yacySeed> nameLookupCache;
private final Hashtable<InetAddress, SoftReference<yacySeed>> ipLookupCache;
public yacySeedDB(plasmaSwitchboard sb,
File seedActiveDBFile,
File seedPassiveDBFile,
File seedPotentialDBFile) {
this.seedActiveDBFile = seedActiveDBFile;
this.seedPassiveDBFile = seedPassiveDBFile;
this.seedPotentialDBFile = seedPotentialDBFile;
this.mySeed = null; // my own seed
this.sb = sb;
// set up seed database
seedActiveDB = openSeedTable(seedActiveDBFile);
seedPassiveDB = openSeedTable(seedPassiveDBFile);
seedPotentialDB = openSeedTable(seedPotentialDBFile);
// start our virtual DNS service for yacy peers with empty cache
nameLookupCache = new Hashtable<String, yacySeed>();
// cache for reverse name lookup
ipLookupCache = new Hashtable<InetAddress, SoftReference<yacySeed>>();
// check if we are in the seedCaches: this can happen if someone else published our seed
removeMySeed();
}
private synchronized void initMySeed() {
if (this.mySeed != null) return;
// create or init own seed
File myOwnSeedFile = sb.getOwnSeedFile();
if (myOwnSeedFile.length() > 0) try {
// load existing identity
mySeed = yacySeed.load(myOwnSeedFile);
} catch (IOException e) {
// create new identity
mySeed = yacySeed.genLocalSeed(sb);
try {
mySeed.save(myOwnSeedFile);
} catch (IOException ee) {
ee.printStackTrace();
System.exit(-1);
}
} else {
// create new identity
mySeed = yacySeed.genLocalSeed(sb);
try {
mySeed.save(myOwnSeedFile);
} catch (IOException ee) {
ee.printStackTrace();
System.exit(-1);
}
}
if (sb.getConfig("portForwardingEnabled","false").equalsIgnoreCase("true")) {
mySeed.put(yacySeed.PORT, sb.getConfig("portForwardingPort","8080"));
mySeed.put(yacySeed.IP, sb.getConfig("portForwardingHost","localhost"));
} else {
mySeed.put(yacySeed.IP, ""); // we delete the old information to see what we have now
mySeed.put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(sb.getConfig("port", "8080")))); // set my seed's correct port number
}
mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // markup startup condition
}
public boolean mySeedIsDefined() {
return this.mySeed != null;
}
public yacySeed mySeed() {
if (this.mySeed == null) {
if (this.sizeConnected() == 0) try {Thread.sleep(5000);} catch (InterruptedException e) {} // wait for init
initMySeed();
}
return this.mySeed;
}
public synchronized void removeMySeed() {
if ((seedActiveDB.size() == 0) && (seedPassiveDB.size() == 0) && (seedPotentialDB.size() == 0)) return; // avoid that the own seed is initialized too early
if (this.mySeed == null) initMySeed();
try {
seedActiveDB.remove(mySeed.hash);
seedPassiveDB.remove(mySeed.hash);
seedPotentialDB.remove(mySeed.hash);
} catch (IOException e) {}
}
public boolean noDHTActivity() {
// for small networks, we don't perform DHT transmissions, because it is possible to search over all peers
return this.sizeConnected() <= dhtActivityMagic;
}
@SuppressWarnings("unchecked")
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
final boolean usetree = false;
new File(seedDBFile.getParent()).mkdirs();
Class[] args;
try {
args = new Class[]{"".getClass(), Class.forName("java.util.Map")};
} catch (ClassNotFoundException e2){
e2.printStackTrace();
args = null;
}
Method initializeHandlerMethod;
try {
initializeHandlerMethod = this.getClass().getMethod("initializeHandler", args);
} catch (SecurityException e1) {
e1.printStackTrace();
initializeHandlerMethod = null;
} catch (NoSuchMethodException e1) {
e1.printStackTrace();
initializeHandlerMethod = null;
}
try {
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (Exception e) {
// try again
kelondroDyn.delete(seedDBFile);
return new kelondroMapObjects(new kelondroDyn(seedDBFile, true, true, commonHashLength, 480, '#', kelondroBase64Order.enhancedCoder, usetree, false, true), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
}
}
protected synchronized kelondroMapObjects resetSeedTable(kelondroMapObjects seedDB, File seedDBFile) {
// this is an emergency function that should only be used if any problem with the
// seed.db is detected
yacyCore.log.logFine("seed-db " + seedDBFile.toString() + " reset (on-the-fly)");
seedDB.close();
seedDBFile.delete();
// create new seed database
seedDB = openSeedTable(seedDBFile);
return seedDB;
}
public synchronized void resetActiveTable() { seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); }
public synchronized void resetPassiveTable() { seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile); }
public synchronized void resetPotentialTable() { seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile); }
public void close() {
if (seedActiveDB != null) seedActiveDB.close();
if (seedPassiveDB != null) seedPassiveDB.close();
if (seedPotentialDB != null) seedPotentialDB.close();
}
@SuppressWarnings("unchecked")
public void initializeHandler(String mapname, Map map) {
// this is used to set up a lastSeen lookup table
}
public Iterator<yacySeed> seedsSortedConnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedActiveDB);
}
public Iterator<yacySeed> seedsSortedDisconnected(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPassiveDB);
}
public Iterator<yacySeed> seedsSortedPotential(boolean up, String field) {
// enumerates seed-type objects: all seeds sequentially ordered by field
return new seedEnum(up, field, seedPotentialDB);
}
public TreeMap<String, String> /* peer-b64-hashes/ipport */ clusterHashes(String clusterdefinition) {
// collects seeds according to cluster definition string, which consists of
// comma-separated .yacy or .yacyh-domains
// the domain may be extended by an alternative address specification of the form
// <ip> or <ip>:<port>. The port must be identical to the port specified in the peer seed,
// therefore it is optional. The address specification is separated by a '='; the complete
// address has therefore the form
// address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
// clusterdef ::= {address}{','address}*
String[] addresses = (clusterdefinition.length() == 0) ? new String[0] : clusterdefinition.split(",");
TreeMap<String, String> clustermap = new TreeMap<String, String>(kelondroBase64Order.enhancedComparator);
yacySeed seed;
String hash, yacydom, ipport;
int p;
for (int i = 0; i < addresses.length; i++) {
p = addresses[i].indexOf('=');
if (p >= 0) {
yacydom = addresses[i].substring(0, p);
ipport = addresses[i].substring(p + 1);
} else {
yacydom = addresses[i];
ipport = null;
}
if (yacydom.endsWith(".yacyh")) {
// find a peer with its hexhash
hash = yacySeed.hexHash2b64Hash(yacydom.substring(0, yacydom.length() - 6));
seed = get(hash);
if (seed == null) {
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
} else {
clustermap.put(hash, ipport);
}
} else if (yacydom.endsWith(".yacy")) {
// find a peer with its name
seed = lookupByName(yacydom.substring(0, yacydom.length() - 5));
if (seed == null) {
yacyCore.log.logWarning("cluster peer '" + yacydom + "' was not found.");
} else {
clustermap.put(seed.hash, ipport);
}
} else {
yacyCore.log.logWarning("cluster peer '" + addresses[i] + "' has wrong syntax. the name must end with .yacy or .yacyh");
}
}
return clustermap;
}
public Iterator<yacySeed> seedsConnected(boolean up, boolean rot, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedActiveDB, minVersion);
}
public Iterator<yacySeed> seedsDisconnected(boolean up, boolean rot, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPassiveDB, minVersion);
}
public Iterator<yacySeed> seedsPotential(boolean up, boolean rot, String firstHash, float minVersion) {
// enumerates seed-type objects: all seeds sequentially without order
return new seedEnum(up, rot, (firstHash == null) ? null : firstHash.getBytes(), null, seedPotentialDB, minVersion);
}
public yacySeed anySeedVersion(float minVersion) {
// return just any seed that has a specific minimum version number
Iterator<yacySeed> e = seedsConnected(true, true, yacySeed.randomHash(), minVersion);
return (yacySeed) e.next();
}
public HashMap<String, yacySeed> seedsByAge(boolean up, int count) {
// returns a peerhash/yacySeed relation
// to get most recent peers, set up = true; for oldest peers, set up = false
if (count > sizeConnected()) count = sizeConnected();
// fill a score object
kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>();
yacySeed ys;
long absage;
Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
int searchcount = 1000;
if (searchcount > sizeConnected()) searchcount = sizeConnected();
try {
while ((s.hasNext()) && (searchcount-- > 0)) {
ys = s.next();
if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer
} catch (Exception e) {}
}
// result is now in the score object; create a result vector
HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
Iterator<String> it = seedScore.scores(up);
int c = 0;
while ((c < count) && (it.hasNext())) {
c++;
ys = getConnected((String) it.next());
if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys);
}
return result;
} catch (kelondroException e) {
seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
return null;
}
}
public int sizeConnected() {
return seedActiveDB.size();
/*
Enumeration e = seedsConnected(true, false, null);
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
return c;
*/
}
public int sizeDisconnected() {
return seedPassiveDB.size();
/*
Enumeration e = seedsDisconnected(true, false, null);
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
return c;
*/
}
public int sizePotential() {
return seedPotentialDB.size();
/*
Enumeration e = seedsPotential(true, false, null);
int c = 0; while (e.hasMoreElements()) {c++; e.nextElement();}
return c;
*/
}
public long countActiveURL() { return seedActiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countActiveRWI() { return seedActiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countActivePPM() { return seedActiveDB.getLongAcc(yacySeed.ISPEED); }
public double countActiveQPM() { return seedActiveDB.getDoubleAcc(yacySeed.RSPEED); }
public long countPassiveURL() { return seedPassiveDB.getLongAcc(yacySeed.LCOUNT); }
public long countPassiveRWI() { return seedPassiveDB.getLongAcc(yacySeed.ICOUNT); }
public long countPotentialURL() { return seedPotentialDB.getLongAcc(yacySeed.LCOUNT); }
public long countPotentialRWI() { return seedPotentialDB.getLongAcc(yacySeed.ICOUNT); }
public synchronized void addConnected(yacySeed seed) {
if ((seed == null) || (seed.isProper() != null)) return;
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
nameLookupCache.put(seed.getName(), seed);
HashMap<String, String> seedPropMap = seed.getMap();
synchronized(seedPropMap) {
seedActiveDB.set(seed.hash, seedPropMap);
}
seedPassiveDB.remove(seed.hash);
seedPotentialDB.remove(seed.hash);
} catch (IOException e){
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
} catch (kelondroException e){
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
} catch (IllegalArgumentException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
}
}
public synchronized void addDisconnected(yacySeed seed) {
if (seed == null) return;
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.remove(seed.hash);
seedPotentialDB.remove(seed.hash);
} catch (Exception e) {}
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
HashMap<String, String> seedPropMap = seed.getMap();
synchronized(seedPropMap) {
seedPassiveDB.set(seed.hash, seedPropMap);
}
} catch (IOException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
} catch (kelondroException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
} catch (IllegalArgumentException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPassiveTable();
}
}
public synchronized void addPotential(yacySeed seed) {
if (seed == null) return;
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.remove(seed.hash);
seedPassiveDB.remove(seed.hash);
} catch (Exception e) {}
if (seed.isProper() != null) return;
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
HashMap<String, String> seedPropMap = seed.getMap();
synchronized(seedPropMap) {
seedPotentialDB.set(seed.hash, seedPropMap);
}
} catch (IOException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
} catch (kelondroException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
} catch (IllegalArgumentException e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetPotentialTable();
}
}
public synchronized void removeDisconnected(String peerHash) {
if(peerHash == null) return;
try {
seedPassiveDB.remove(peerHash);
} catch (IOException e) { }
}
public synchronized void removePotential(String peerHash) {
if(peerHash == null) return;
try {
seedPotentialDB.remove(peerHash);
} catch (IOException e) { }
}
public boolean hasConnected(String hash) {
try {
return (seedActiveDB.get(hash) != null);
} catch (IOException e) {
return false;
}
}
public boolean hasDisconnected(String hash) {
try {
return (seedPassiveDB.get(hash) != null);
} catch (IOException e) {
return false;
}
}
public boolean hasPotential(String hash) {
try {
return (seedPotentialDB.get(hash) != null);
} catch (IOException e) {
return false;
}
}
private yacySeed get(String hash, kelondroMapObjects database) {
if (hash == null) return null;
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
HashMap<String, String> entry = database.getMap(hash);
if (entry == null) return null;
return new yacySeed(hash, entry);
}
public yacySeed getConnected(String hash) {
return get(hash, seedActiveDB);
}
public yacySeed getDisconnected(String hash) {
return get(hash, seedPassiveDB);
}
public yacySeed getPotential(String hash) {
return get(hash, seedPotentialDB);
}
public yacySeed get(String hash) {
yacySeed seed = getConnected(hash);
if (seed == null) seed = getDisconnected(hash);
if (seed == null) seed = getPotential(hash);
return seed;
}
public void update(String hash, yacySeed seed) {
if (this.mySeed == null) initMySeed();
if (hash.equals(mySeed.hash)) {
mySeed = seed;
return;
}
yacySeed s = get(hash, seedActiveDB);
if (s != null) try { seedActiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
s = get(hash, seedPassiveDB);
if (s != null) try { seedPassiveDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
s = get(hash, seedPotentialDB);
if (s != null) try { seedPotentialDB.set(hash, seed.getMap()); return;} catch (IOException e) {}
}
public yacySeed lookupByName(String peerName) {
// reads a seed by searching by name
// local peer?
if (peerName.equals("localpeer")) {
if (this.mySeed == null) initMySeed();
return mySeed;
}
// then try to use the cache
yacySeed seed = (yacySeed) nameLookupCache.get(peerName);
if (seed != null) return seed;
// enumerate the cache and simultanous insert values
String name;
for (int table = 0; table < 2; table++) {
Iterator<yacySeed> e = (table == 0) ? seedsConnected(true, false, null, (float) 0.0) : seedsDisconnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
seed = (yacySeed) e.next();
if (seed != null) {
name = seed.getName().toLowerCase();
if (seed.isProper() == null) nameLookupCache.put(name, seed);
if (name.equals(peerName)) return seed;
}
}
}
// check local seed
if (this.mySeed == null) initMySeed();
name = mySeed.getName().toLowerCase();
if (mySeed.isProper() == null) nameLookupCache.put(name, mySeed);
if (name.equals(peerName)) return mySeed;
// nothing found
return null;
}
public yacySeed lookupByIP(
InetAddress peerIP,
boolean lookupConnected,
boolean lookupDisconnected,
boolean lookupPotential
) {
if (peerIP == null) return null;
yacySeed seed = null;
// local peer?
if (httpd.isThisHostIP(peerIP)) {
if (this.mySeed == null) initMySeed();
return mySeed;
}
// then try to use the cache
SoftReference<yacySeed> ref = ipLookupCache.get(peerIP);
if (ref != null) {
seed = (yacySeed) ref.get();
if (seed != null) return seed;
}
int pos = -1;
String addressStr = null;
InetAddress seedIPAddress = null;
HashSet<String> badPeerHashes = new HashSet<String>();
if (lookupConnected) {
// enumerate the cache and simultanous insert values
Iterator<yacySeed> e = seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
try {
seed = (yacySeed) e.next();
if (seed != null) {
addressStr = seed.getPublicAddress();
if (addressStr == null) {
serverLog.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
badPeerHashes.add(seed.hash);
continue;
}
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
} catch (UnknownHostException ex) {}
}
// delete bad peers
Iterator<String> i = badPeerHashes.iterator();
while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (IOException e1) {e1.printStackTrace();}
badPeerHashes.clear();
}
if (lookupDisconnected) {
// enumerate the cache and simultanous insert values
Iterator<yacySeed>e = seedsDisconnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
try {
seed = (yacySeed) e.next();
if (seed != null) {
addressStr = seed.getPublicAddress();
if (addressStr == null) {
serverLog.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null.");
badPeerHashes.add(seed.hash);
continue;
}
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
} catch (UnknownHostException ex) {}
}
// delete bad peers
Iterator<String> i = badPeerHashes.iterator();
while (i.hasNext()) try {seedActiveDB.remove(i.next());} catch (IOException e1) {e1.printStackTrace();}
badPeerHashes.clear();
}
if (lookupPotential) {
// enumerate the cache and simultanous insert values
Iterator<yacySeed> e = seedsPotential(true, false, null, (float) 0.0);
while (e.hasNext()) {
try {
seed = (yacySeed) e.next();
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) {
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (seed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
}
} catch (UnknownHostException ex) {}
}
}
try {
// check local seed
if (this.mySeed == null) return null;
addressStr = mySeed.getPublicAddress();
if (addressStr == null) return null;
if ((pos = addressStr.indexOf(":"))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = InetAddress.getByName(addressStr);
if (mySeed.isProper() == null) ipLookupCache.put(seedIPAddress, new SoftReference<yacySeed>(mySeed));
if (seedIPAddress.equals(peerIP)) return mySeed;
// nothing found
return null;
} catch (UnknownHostException e2) {
return null;
}
}
public ArrayList<String> storeCache(File seedFile) throws IOException {
return storeCache(seedFile, false);
}
private ArrayList<String> storeCache(File seedFile, boolean addMySeed) throws IOException {
PrintWriter pw = null;
ArrayList<String> v = new ArrayList<String>(seedActiveDB.size() + 1);
try {
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
// store own seed
String line;
if (this.mySeed == null) initMySeed();
if (addMySeed) {
line = mySeed.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
// store other seeds
yacySeed ys;
Iterator<yacySeed> se = seedsConnected(true, false, null, (float) 0.0);
while (se.hasNext()) {
ys = (yacySeed) se.next();
if (ys != null) {
line = ys.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
}
pw.flush();
} finally {
if (pw != null) try { pw.close(); } catch (Exception e) {}
}
return v;
}
public String uploadCache(yacySeedUploader uploader,
serverSwitch<?> sb,
yacySeedDB seedDB,
// String seedFTPServer,
// String seedFTPAccount,
// String seedFTPPassword,
// File seedFTPPath,
yacyURL seedURL) throws Exception {
// upload a seed file, if possible
if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given");
String log = null;
File seedFile = null;
try {
// create a seed file which for uploading ...
seedFile = File.createTempFile("seedFile",".txt", plasmaHTCache.cachePath);
seedFile.deleteOnExit();
serverLog.logFine("YACY","SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());
ArrayList<String> uv = storeCache(seedFile, true);
// uploading the seed file
serverLog.logFine("YACY","SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
log = uploader.uploadSeedFile(sb,seedDB,seedFile);
// test download
serverLog.logFine("YACY","SaveSeedList: Trying to download seed-file '" + seedURL + "'.");
ArrayList<String> check = downloadSeedFile(seedURL);
// Comparing if local copy and uploaded copy are equal
String errorMsg = checkCache(uv, check);
if (errorMsg == null)
log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.CRLF_STRING;
else {
throw new Exception("UPLOAD CHECK - Error: the result vector is different. " + errorMsg + serverCore.CRLF_STRING);
}
} finally {
if (seedFile != null) try { seedFile.delete(); } catch (Exception e) {/* ignore this */}
}
return log;
}
private ArrayList<String> downloadSeedFile(yacyURL seedURL) throws IOException {
httpc remote = null;
try {
// init httpc
remote = new httpc(
seedURL.getHost(),
seedURL.getHost(),
seedURL.getPort(),
10000,
seedURL.getProtocol().equalsIgnoreCase("https"),
sb.remoteProxyConfig,
null, null);
// Configure http headers
httpHeader reqHeader = new httpHeader();
reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
// send request
httpc.response res = remote.GET(seedURL.getFile(), reqHeader);
// check response code
if (res.statusCode != 200) {
throw new IOException("Server returned status: " + res.status);
}
// read byte array
byte[] content = serverFileUtils.read(res.getContentInputStream());
remote.close();
// uncompress it if it is gzipped
content = serverFileUtils.uncompressGZipArray(content);
// convert it into an array
return nxTools.strings(content,"UTF-8");
} catch (Exception e) {
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
}
}
private String checkCache(ArrayList<String> uv, ArrayList<String> check) {
if ((check == null) || (uv == null) || (uv.size() != check.size())) {
serverLog.logFine("YACY","SaveSeedList: Local and uploades seed-list " +
"contains varying numbers of entries." +
"\n\tLocal seed-list: " + ((uv == null) ? "null" : Integer.toString(uv.size())) + " entries" +
"\n\tRemote seed-list: " + ((check == null) ? "null" : Integer.toString(check.size())) + " enties");
return "Entry count is different: uv.size() = " + ((uv == null) ? "null" : Integer.toString(uv.size())) + ", check = " + ((check == null) ? "null" : Integer.toString(check.size()));
}
serverLog.logFine("YACY","SaveSeedList: Comparing local and uploades seed-list entries ...");
int i;
for (i = 0; i < uv.size(); i++) {
if (!(((String) uv.get(i)).equals((String) check.get(i)))) return "Element at position " + i + " is different.";
}
// no difference found
return null;
}
public String resolveYacyAddress(String host) {
yacySeed seed;
int p;
String subdom = null;
if (host.endsWith(".yacyh")) {
// this is not functional at the moment
// caused by lowecasing of hashes at the browser client
p = host.indexOf(".");
if ((p > 0) && (p != (host.length() - 6))) {
subdom = host.substring(0, p);
host = host.substring(p + 1);
}
// check if we have a b64-hash or a hex-hash
String hash = host.substring(0, host.length() - 6);
if (hash.length() > commonHashLength) {
// this is probably a hex-hash
hash = yacySeed.hexHash2b64Hash(hash);
}
// check remote seeds
seed = getConnected(hash); // checks only remote, not local
// check local seed
if (seed == null) {
if (this.mySeed == null) initMySeed();
if (hash.equals(mySeed.hash))
seed = mySeed;
else return null;
}
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
} else if (host.endsWith(".yacy")) {
// identify subdomain
p = host.indexOf(".");
if ((p > 0) && (p != (host.length() - 5))) {
subdom = host.substring(0, p); // no double-dot attack possible, the subdom cannot have ".." in it
host = host.substring(p + 1); // if ever, the double-dots are here but do not harm
}
// identify domain
String domain = host.substring(0, host.length() - 5).toLowerCase();
seed = lookupByName(domain);
if (seed == null) return null;
if (this.mySeed == null) initMySeed();
if ((seed == mySeed) && (!(seed.isOnline()))) {
// take local ip instead of external
return serverDomains.myPublicIP() + ":" + serverCore.getPortNr(sb.getConfig("port", "8080")) + ((subdom == null) ? "" : ("/" + subdom));
}
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
} else {
return null;
}
}
class seedEnum implements Iterator<yacySeed> {
kelondroMapObjects.mapIterator it;
yacySeed nextSeed;
kelondroMapObjects database;
float minVersion;
public seedEnum(boolean up, boolean rot, byte[] firstKey, byte[] secondKey, kelondroMapObjects database, float minVersion) {
this.database = database;
this.minVersion = minVersion;
try {
it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey);
while (true) {
nextSeed = internalNext();
if (nextSeed == null) break;
if (nextSeed.getVersion() >= this.minVersion) break;
}
} catch (IOException e) {
e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null;
} catch (kelondroException e) {
e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
it = null;
}
}
public seedEnum(boolean up, String field, kelondroMapObjects database) {
this.database = database;
try {
it = database.maps(up, field);
nextSeed = internalNext();
} catch (kelondroException e) {
e.printStackTrace();
yacyCore.log.logSevere("ERROR seedLinEnum: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
it = null;
}
}
public boolean hasNext() {
return (nextSeed != null);
}
public yacySeed internalNext() {
if ((it == null) || (!(it.hasNext()))) return null;
try {
HashMap<String, String> dna = it.next();
if (dna == null) return null;
String hash = (String) dna.remove("key");
//while (hash.length() < commonHashLength) { hash = hash + "_"; }
return new yacySeed(hash, dna);
} catch (Exception e) {
e.printStackTrace();
yacyCore.log.logSevere("ERROR internalNext: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
if (database == seedActiveDB) seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
if (database == seedPassiveDB) seedPassiveDB = resetSeedTable(seedPassiveDB, seedPassiveDBFile);
if (database == seedPotentialDB) seedPotentialDB = resetSeedTable(seedPotentialDB, seedPotentialDBFile);
return null;
}
}
public yacySeed next() {
yacySeed seed = nextSeed;
try {while (true) {
nextSeed = internalNext();
if (nextSeed == null) break;
if (nextSeed.getVersion() >= this.minVersion) break;
}} catch (kelondroException e) {
e.printStackTrace();
// eergency reset
yacyCore.log.logSevere("seed-db emergency reset", e);
try {
database.reset();
nextSeed = null;
return null;
} catch (IOException e1) {
// no recovery possible
e1.printStackTrace();
System.exit(-1);
}
}
return seed;
}
public void remove() {
throw new UnsupportedOperationException();
}
}
}