boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

redesign of fining of peers by ip: more leightweight method to read the

Browse Source 
seed databases
pull/1/head
Michael Christen 13 years ago
parent 0797b0de99
commit 20e3084bd4
5 changed files with 67 additions and 99 deletions
Show Stats Download Patch File Download Diff File

9
htroot/AccessTracker_p.java
Unescape View File

@ -24,16 +24,16 @@
// along with this program; if not, write to the Free Software // along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.text.SimpleDateFormat;
import java.util.Collection; import java.util.Collection;
import java.util.ConcurrentModificationException; import java.util.ConcurrentModificationException;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.TreeSet;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
import java.text.SimpleDateFormat;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
@ -42,11 +42,10 @@ import net.yacy.peers.Seed;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.query.AccessTracker; import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.QueryParams; import net.yacy.search.query.QueryParams;
import de.anomic.server.serverAccessTracker.Track;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.server.serverAccessTracker.Track;
public class AccessTracker_p { public class AccessTracker_p {
@ -267,7 +266,7 @@ public class AccessTracker_p {
prop.put("page_list_" + m + "_dark", ((dark) ? 1 : 0) ); dark =! dark; prop.put("page_list_" + m + "_dark", ((dark) ? 1 : 0) ); dark =! dark;
prop.putHTML("page_list_" + m + "_host", host); prop.putHTML("page_list_" + m + "_host", host);
if (page == 5) { if (page == 5) {
final Seed remotepeer = sb.peers.lookupByIP(Domains.dnsResolve(host), true, true, true); final Seed remotepeer = sb.peers.lookupByIP(Domains.dnsResolve(host), -1, true, true, true);
prop.putHTML("page_list_" + m + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName()); prop.putHTML("page_list_" + m + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName());
} }
prop.putNum("page_list_" + m + "_count", handles.size()); prop.putNum("page_list_" + m + "_count", handles.size());

7
htroot/Connections_p.java
Unescape View File

@ -39,13 +39,10 @@ import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.peers.PeerActions; import net.yacy.peers.PeerActions;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
//import de.anomic.http.client.ConnectionInfo;
//import de.anomic.http.client.Client;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverCore.Session;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
import de.anomic.server.serverCore.Session;
public final class Connections_p { public final class Connections_p {
@ -98,7 +95,7 @@ public final class Connections_p {
// determining if the source is a yacy host // determining if the source is a yacy host
Seed seed = null; Seed seed = null;
if (doNameLookup) { if (doNameLookup) {
seed = sb.peers.lookupByIP(userAddress,true,false,false); seed = sb.peers.lookupByIP(userAddress, -1, true, false, false);
if (seed != null && (seed.hash.equals(sb.peers.mySeed().hash)) && if (seed != null && (seed.hash.equals(sb.peers.mySeed().hash)) &&
(!seed.get(Seed.PORT,"").equals(Integer.toString(userPort)))) { (!seed.get(Seed.PORT,"").equals(Integer.toString(userPort)))) {
seed = null; seed = null;

2
htroot/yacy/search.java
Unescape View File

@ -417,7 +417,7 @@ public final class search {
} }
// prepare search statistics // prepare search statistics
theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), true, false, false); theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : joincount; theQuery.resultcount = (theSearch == null) ? 0 : joincount;
theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.result().getURLRetrievalTime(); theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.result().getURLRetrievalTime();

2
source/net/yacy/peers/PeerActions.java
Unescape View File

@ -83,7 +83,7 @@ public class PeerActions {
return false; return false;
} }
final Seed doubleSeed = this.seedDB.lookupByIP(seed.getInetAddress(), true, false, false); final Seed doubleSeed = this.seedDB.lookupByIP(seed.getInetAddress(), seed.getPort(), true, false, false);
if ((doubleSeed != null) && (doubleSeed.getPort() == seed.getPort()) && (!(doubleSeed.hash.equals(seed.hash)))) { if ((doubleSeed != null) && (doubleSeed.getPort() == seed.getPort()) && (!(doubleSeed.hash.equals(seed.hash)))) {
// a user frauds with his peer different peer hashes // a user frauds with his peer different peer hashes
if (Network.log.isFine()) Network.log.logFine("connect: rejecting FRAUD (double hashes " + doubleSeed.hash + "/" + seed.hash + " on same port " + seed.getPort() + ") peer " + seed.getName()); if (Network.log.isFine()) Network.log.logFine("connect: rejecting FRAUD (double hashes " + doubleSeed.hash + "/" + seed.hash + " on same port " + seed.getPort() + ") peer " + seed.getName());

146
source/net/yacy/peers/SeedDB.java
Unescape View File

@ -51,7 +51,6 @@ import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.kelondro.blob.MapDataMining; import net.yacy.kelondro.blob.MapDataMining;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.index.HandleSet;
import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Base64Order;
@ -675,13 +674,13 @@ public final class SeedDB implements AlternativeDomainNames {
public Seed lookupByIP( public Seed lookupByIP(
final InetAddress peerIP, final InetAddress peerIP,
final int port,
final boolean lookupConnected, final boolean lookupConnected,
final boolean lookupDisconnected, final boolean lookupDisconnected,
final boolean lookupPotential final boolean lookupPotential
) { ) {
if (peerIP == null) return null; if (peerIP == null) return null;
Seed seed = null;
// local peer? // local peer?
if (Domains.isThisHostIP(peerIP)) { if (Domains.isThisHostIP(peerIP)) {
@ -691,112 +690,85 @@ public final class SeedDB implements AlternativeDomainNames {
// then try to use the cache // then try to use the cache
final SoftReference<Seed> ref = this.ipLookupCache.get(peerIP); final SoftReference<Seed> ref = this.ipLookupCache.get(peerIP);
Seed seed = null;
if (ref != null) { if (ref != null) {
seed = ref.get(); seed = ref.get();
if (seed != null) return seed; if (seed != null) {
//System.out.println("*** found lookupByIP in cache: " + peerIP.toString() + " -> " + this.mySeed.getName());
return seed;
}
} }
int pos = -1; String ipString = peerIP.getHostAddress();
String addressStr = null;
InetAddress seedIPAddress = null;
final HandleSet badPeerHashes = new HandleSet(12, Base64Order.enhancedCoder, 0);
if (lookupConnected) { if (lookupConnected) {
// enumerate the cache and simultaneously insert values try {
final Iterator<Seed> e = seedsConnected(true, false, null, (float) 0.0); Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedActiveDB.entries(Seed.IP, ipString);
while (e.hasNext()) { Map.Entry<byte[], Map<String, String>> entry;
seed = e.next(); while (mmap.hasNext()) {
if (seed != null) { entry = mmap.next();
addressStr = seed.getPublicAddress(); if (entry == null) break;
if (addressStr == null) { String p = entry.getValue().get(Seed.PORT);
Log.logWarning("YACY","lookupByIP/Connected: address of seed " + seed.getName() + "/" + seed.hash + " is null."); if (p == null) continue;
try { if (port > 0 && Integer.parseInt(p) != port) continue;
badPeerHashes.put(ASCII.getBytes(seed.hash)); seed = this.getConnected(ASCII.String(entry.getKey()));
} catch (final RowSpaceExceededException e1) { if (seed == null) continue;
Log.logException(e1); this.ipLookupCache.put(peerIP, new SoftReference<Seed>(seed));
break; //System.out.println("*** found lookupByIP in connected: " + peerIP.toString() + " -> " + seed.getName());
} return seed;
continue;
}
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<Seed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
} }
} catch ( IOException e ) {
} }
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
while (i.hasNext()) try {this.seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
badPeerHashes.clear();
} }
if (lookupDisconnected) { if (lookupDisconnected) {
// enumerate the cache and simultanous insert values try {
final Iterator<Seed>e = seedsDisconnected(true, false, null, (float) 0.0); Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedPassiveDB.entries(Seed.IP, ipString);
Map.Entry<byte[], Map<String, String>> entry;
while (e.hasNext()) { while (mmap.hasNext()) {
seed = e.next(); entry = mmap.next();
if (seed != null) { if (entry == null) break;
addressStr = seed.getPublicAddress(); String p = entry.getValue().get(Seed.PORT);
if (addressStr == null) { if (p == null) continue;
Log.logWarning("YACY","lookupByIPDisconnected: address of seed " + seed.getName() + "/" + seed.hash + " is null."); if (port > 0 && Integer.parseInt(p) != port) continue;
try { seed = this.getDisconnected(ASCII.String(entry.getKey()));
badPeerHashes.put(UTF8.getBytes(seed.hash)); if (seed == null) continue;
} catch (final RowSpaceExceededException e1) { this.ipLookupCache.put(peerIP, new SoftReference<Seed>(seed));
Log.logException(e1); //System.out.println("*** found lookupByIP in disconnected: " + peerIP.toString() + " -> " + seed.getName());
break; return seed;
}
continue;
}
if ((pos = addressStr.indexOf(':'))!= -1) {
addressStr = addressStr.substring(0,pos);
}
seedIPAddress = Domains.dnsResolve(addressStr);
if (seedIPAddress == null) continue;
if (seed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<Seed>(seed));
if (seedIPAddress.equals(peerIP)) return seed;
} }
} catch ( IOException e ) {
} }
// delete bad peers
final Iterator<byte[]> i = badPeerHashes.iterator();
while (i.hasNext()) try {this.seedActiveDB.delete(i.next());} catch (final IOException e1) {Log.logException(e1);}
badPeerHashes.clear();
} }
if (lookupPotential) { if (lookupPotential) {
// enumerate the cache and simultanous insert values try {
final Iterator<Seed> e = seedsPotential(true, false, null, (float) 0.0); Iterator<Map.Entry<byte[], Map<String, String>>> mmap = this.seedPotentialDB.entries(Seed.IP, ipString);
Map.Entry<byte[], Map<String, String>> entry;
while (e.hasNext()) { while (mmap.hasNext()) {
seed = e.next(); entry = mmap.next();
if ((seed != null) && ((addressStr = seed.getPublicAddress()) != null)) { if (entry == null) break;
if ((pos = addressStr.indexOf(':'))!= -1) { String p = entry.getValue().get(Seed.PORT);
addressStr = addressStr.substring(0,pos); if (p == null) continue;
} if (port > 0 && Integer.parseInt(p) != port) continue;
seedIPAddress = Domains.dnsResolve(addressStr); seed = this.getPotential(ASCII.String(entry.getKey()));
if (seedIPAddress == null) continue; if (seed == null) continue;
if (seed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<Seed>(seed)); this.ipLookupCache.put(peerIP, new SoftReference<Seed>(seed));
if (seedIPAddress.equals(peerIP)) return seed; //System.out.println("*** found lookupByIP in potential: " + peerIP.toString() + " -> " + seed.getName());
return seed;
} }
} catch ( IOException e ) {
} }
} }
// check local seed // check local seed
if (this.mySeed == null) return null; if (this.mySeed == null) return null;
addressStr = this.mySeed.getPublicAddress(); String s = this.mySeed.getIP();
if (addressStr == null) return null; if (s == null || !ipString.equals(s)) return null;
if ((pos = addressStr.indexOf(':'))!= -1) { int p = this.mySeed.getPort();
addressStr = addressStr.substring(0,pos); if (p != port) return null;
} //System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
seedIPAddress = Domains.dnsResolve(addressStr); return this.mySeed;
if (seedIPAddress == null) return null;
if (this.mySeed.isProper(false) == null) this.ipLookupCache.put(seedIPAddress, new SoftReference<Seed>(this.mySeed));
if (seedIPAddress.equals(peerIP)) return this.mySeed;
// nothing found
return null;
} }
private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException { private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {

Write Preview
Loading…
Cancel
Save