boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added new classes to renovate the YaCy protocol based on simple data

Browse Source 
structures in cora:
- added the Peer object, which is a fresh version of Seed
- added the Peers object, which is a fresh version of Network
- added the Network api access class to retrieve a list of peers based
on the Network.xml servlet in all YaCy peers.
pull/1/head
orbiter 13 years ago
parent 14897d4bfc
commit a053b356ee
3 changed files with 409 additions and 0 deletions
Show Stats Download Patch File Download Diff File

136
source/net/yacy/cora/services/federated/yacy/Peer.java
Unescape View File

@ -0,0 +1,136 @@
package net.yacy.cora.services.federated.yacy;
/**
* Peer
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 21.09.2012 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.Serializable;
import java.util.HashMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.order.Base64Order;
/**
* new implemenentation of "Seed" objects: Peers.
* A Peer contains the attributes that a peer wants to show in public.
* This Class is simply a representation of the XML Schema that is used in the Network.xml interface.
*/
public class Peer extends HashMap<Peer.Schema, String> implements Comparable<Peer>, Serializable {
private static final long serialVersionUID = -3279480981385980050L;
public enum Schema implements Serializable {
hash,fullname,version,ppm,qph,uptime,links,
words,rurls,lastseen,sendWords,receivedWords,
sendURLs,receivedURLs,type,direct,acceptcrawl,dhtreceive,
nodestate,location,seedurl,age,seeds,connects,address,useragent;
}
private long time;
public Peer() {
super();
this.time = System.currentTimeMillis();
}
/**
* Get the number of minutes that the peer which returns the peer list knows when the peer was last seen.
* This value is only a relative to the moment when another peer was asked for fresh information.
* To compute an absolute value for last-seen, use the lastseenTime() method.
* @return time in minutes
*/
public int lastseen() {
String x = this.get(Schema.lastseen);
if (x == null) return Integer.MAX_VALUE;
try {
return Integer.parseInt(x);
} catch (NumberFormatException e) {
return Integer.MAX_VALUE;
}
}
/**
* get the absolute time when this peer was seen in the network
* @return time in milliseconds
*/
public long lastseenTime() {
return time - lastseen() * 60000;
}
/**
* get the version number of the peer
* @return
*/
public float version() {
String x = this.get(Schema.version); if (x == null) return 0.0f;
int p = x.indexOf('/'); if (p < 0) return 0.0f;
x = x.substring(0, p);
try {
return Float.parseFloat(x);
} catch (NumberFormatException e) {
return 0.0f;
}
}
/**
* check if the peer supports the solr interface
* @return true if the peer supports the solr interface
*/
public boolean supportsSolr() {
return version() >= 1.041f;
}
/**
* compare the peer to another peer.
* The comparisment is done using the base 64 order on the peer hash.
*/
@Override
public int compareTo(Peer o) {
String h0 = this.get(Schema.hash);
String h1 = o.get(Schema.hash);
return Base64Order.enhancedCoder.compare(ASCII.getBytes(h0), ASCII.getBytes(h1));
}
/**
* get the hash code of the peer.
* The hash code is a number that has the same order as the peer order.
*/
public int hashCode() {
String h = this.get(Schema.hash);
return (int) (Base64Order.enhancedCoder.cardinal(h) >> 32);
}
/**
* check if two peers are equal:
* two peers are equal if they have the same hash.
*/
public boolean equals(Object o) {
if (!(o instanceof Peer)) return false;
String h0 = this.get(Schema.hash);
String h1 = ((Peer) o).get(Schema.hash);
return h0.equals(h1);
}
public static void main(String[] args) {
String h = "____________";
long l = Base64Order.enhancedCoder.cardinal(h);
System.out.println("l = " + l + ", h = " + ((int) (l >> 32)));
System.out.println("l-maxlong = " + (l - Long.MAX_VALUE) + ", (h-maxint) = " + (((int) (l >> 32)) - Integer.MAX_VALUE));
}
}

176
source/net/yacy/cora/services/federated/yacy/Peers.java
Unescape View File

@ -0,0 +1,176 @@
/**
* Peers
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 21.09.2012 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.yacy;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
import org.apache.log4j.Logger;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.services.federated.yacy.api.Network;
public class Peers extends TreeMap<byte[], Peer> implements Serializable {
public final static String[] bootstrapPeers = new String[]{
"search.yacy.net", "yacy.dyndns.org:8000", "yacy-websuche.mxchange.org:8090",
"sokrates.homeunix.net:6070", "sokrates.homeunix.net:9090",
"141.52.175.27:8080", "62.75.214.113:8080", "141.52.175.30:8080"};
private final static Logger log = Logger.getLogger(Peers.class);
private static final long serialVersionUID = -2939656606305545080L;
private long lastBootstrap;
public Peers() {
super(Base64Order.enhancedCoder);
this.lastBootstrap = 0;
}
/**
* refresh() gets a new network list from one random remote peer once every
* minute. This method will load a remote list not more then every one minute
* and if it does, it is done concurrently. Therefore this method can be called
* every time when a process needs specific remote peers.
*/
public void refresh() {
if (System.currentTimeMillis() - this.lastBootstrap < 60000) return;
lastBootstrap = System.currentTimeMillis();
new Thread() {
public void run() {
String[] peers = bootstrapList(select(false, false));
bootstrap(peers, 1);
}
}.start();
}
/**
* this method must be called once to bootstrap a list of network peers.
* To do this, a default list of peers must be given.
* @param peers a list of known peers
* @param selection number of peers which are taken from the given list of peers for bootstraping
*/
public void bootstrap(final String[] peers, int selection) {
int loops = 0;
while (this.size() == 0 || loops++ == 0) {
if (selection > peers.length) selection = peers.length;
Set<Integer> s = new HashSet<Integer>();
Random r = new Random(System.currentTimeMillis());
while (s.size() < selection) s.add(r.nextInt(peers.length));
List<Thread> t = new ArrayList<Thread>();
for (Integer pn: s) {
final String bp = peers[pn.intValue()];
Thread t0 = new Thread() {
public void run() {
Peers ps;
try {
ps = Network.getNetwork(bp);
int c0 = Peers.this.size();
for (Peer p: ps.values()) Peers.this.add(p);
int c1 = Peers.this.size();
log.info("bootstrap with peer " + bp + ": added " + (c1 - c0) + " peers");
} catch (IOException e) {
log.info("bootstrap with peer " + bp + ": FAILED - " + e.getMessage());
}
}
};
t0.start();
t.add(t0);
}
for (Thread t0: t) try {t0.join(10000);} catch (InterruptedException e) {}
}
lastBootstrap = System.currentTimeMillis();
log.info("bootstrap finished: " + this.size() + " peers");
}
/**
* add a new peer to the list of peers
* @param peer
*/
public synchronized void add(Peer peer) {
String hash = peer.get(Peer.Schema.hash);
if (hash == null) return;
Peer p = this.put(ASCII.getBytes(hash), peer);
if (p == null) return;
if (p.lastseenTime() < peer.lastseenTime()) this.put(ASCII.getBytes(hash), p);
}
/**
* get a peer using the peer hash
* @param hash
* @return
*/
public synchronized Peer get(String hash) {
return super.get(ASCII.getBytes(hash));
}
/**
* select a list of peers according to special needs. The require parameters are combined as conjunction
* @param requireNode must be true to select only peers which are node peers
* @param requireSolr must be true to select only peers which support the solr interface
* @return
*/
public synchronized List<Peer> select(final boolean requireNode, final boolean requireSolr) {
List<Peer> l = new ArrayList<Peer>();
for (Peer p: this.values()) {
if (requireNode && !p.get(Peer.Schema.nodestate).equals("1")) continue;
if (requireSolr && !p.supportsSolr()) continue;
l.add(p);
}
return l;
}
/**
* convenient method to produce a list of bootstrap peer addresses from given peer lists
* @param peers
* @return
*/
public static synchronized String[] bootstrapList(List<Peer> peers) {
List<String> l = new ArrayList<String>();
for (Peer p: peers) l.add(p.get(Peer.Schema.address));
return l.toArray(new String[l.size()]);
}
public static void main(String[] args) {
Peers peers = new Peers();
peers.bootstrap(Peers.bootstrapPeers, 4);
//Peers peers = Network.getNetwork("sokrates.homeunix.net:9090");
/*
for (Peer p: peers.values()) {
log.info(p.get(Peer.Schema.fullname) + " - " + p.get(Peer.Schema.address));
}
*/
List<Peer> nodes = peers.select(false, true);
for (Peer p: nodes) {
log.info(p.get(Peer.Schema.fullname) + " - " + p.get(Peer.Schema.address));
}
try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {}
}
}

97
source/net/yacy/cora/services/federated/yacy/api/Network.java
Unescape View File

@ -0,0 +1,97 @@
/**
* Peers
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 21.09.2012 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.services.federated.yacy.api;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.services.federated.yacy.Peer;
import net.yacy.cora.services.federated.yacy.Peers;
/**
* discover all peers in the network when only one peer is known.
* this works only for a limited number of peers, not more than some thousands
*/
public class Network {
/**
* get the list of peers from one peer
* @param address
* @return a network as list of peers
* @throws IOException
*/
public static Peers getNetwork(final String address) throws IOException {
Peers peers = new Peers();
final HTTPClient httpclient = new HTTPClient(ClientIdentification.getUserAgent(), 15000);
final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip=");
ByteArrayInputStream bais = new ByteArrayInputStream(content);
Document doc = null;
try {
doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(bais);
} catch (Throwable e) {
throw new IOException(e.getMessage());
}
bais.close();
doc.getDocumentElement().normalize();
NodeList objects = doc.getElementsByTagName("peer");
for (int i = 0; i < objects.getLength(); i++) {
Node object = objects.item(i);
if (object.getNodeType() == Node.ELEMENT_NODE) {
Element element = (Element) object;
Peer peer = new Peer();
for (Peer.Schema attr: Peer.Schema.values()) {
peer.put(attr, getAttr(attr.name(), element));
}
peers.add(peer);
//log.info(peer.toString());
}
}
return peers;
}
private static String getAttr(String attr, Element eElement) {
NodeList nl0 = eElement.getElementsByTagName(attr);
if (nl0 == null) return "";
Node n0 = nl0.item(0);
if (n0 == null) return "";
NodeList nl1 = n0.getChildNodes();
if (nl1 == null) return "";
Node n1 = nl1.item(0);
if (n1 == null) return "";
return n1.getNodeValue();
}
public static void main(String[] args) {
//getNetwork("search.yacy.net");
try {getNetwork("sokrates.homeunix.net:9090");} catch (IOException e1) {}
try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {}
}
}
Write Preview
Loading…
Cancel
Save