boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

new network bootsraping method

Browse Source 
- no more contact to yacy.net (no remote superseed any more)
- moved superseed file into new network unit definition
- fixed build; includes new network bootstraping files now

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3922 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 2ed808366a
commit 6b4cfbd2d6
6 changed files with 18 additions and 79 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.4
javacTarget=1.4
# Release Configuration
releaseVersion=0.531
releaseVersion=0.532
releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseFileParentDir=yacy

2
build.xml
Unescape View File

@ -459,6 +459,8 @@
<include name="yacy.badwords.example"/>
<include name="yacy.stopwords"/>
<include name="yacy.parser"/>
<include name="yacy.network.group"/>
<include name="yacy.network.unit"/>
<include name="httpd.mime"/>
<include name="superseed.txt"/>
</fileset>

4
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -214,9 +214,7 @@ public class yacyCore {
loadSeedUploadMethods();
// deploy peer actions
peerActions = new yacyPeerActions(seedDB, switchboard,
new File(sb.getRootPath(), sb.getConfig("network.unit.bootstrap.file", "superseed.txt")),
switchboard.getConfig("network.unit.bootstrap.url", "http://www.yacy.net/yacy/superseed.txt"));
peerActions = new yacyPeerActions(seedDB, switchboard);
dhtAgent = new yacyDHTAction(seedDB);
peerActions.deploy(dhtAgent);
peerActions.deploy(new yacyNewsAction(newsPool));

70
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -43,11 +43,7 @@
package de.anomic.yacy;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -59,7 +55,6 @@ import de.anomic.net.URL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.tools.disorderSet;
import de.anomic.tools.nxTools;
public class yacyPeerActions {
@ -68,22 +63,17 @@ public class yacyPeerActions {
private plasmaSwitchboard sb;
private HashSet actions;
private HashMap userAgents;
private File superseedFile;
private String superseedURL;
public long juniorConnects;
public long seniorConnects;
public long principalConnects;
public long disconnects;
private int bootstrapLoadTimeout;
public yacyPeerActions(yacySeedDB seedDB, plasmaSwitchboard switchboard, File superseedFile, String superseedURL) {
public yacyPeerActions(yacySeedDB seedDB, plasmaSwitchboard switchboard) {
this.seedDB = seedDB;
this.sb = switchboard;
this.actions = new HashSet();
this.userAgents = new HashMap();
this.superseedFile = superseedFile;
this.superseedURL = superseedURL;
this.superseedURL = superseedURL;
this.juniorConnects = 0;
this.seniorConnects = 0;
this.principalConnects = 0;
@ -162,14 +152,13 @@ public class yacyPeerActions {
yacyCore.log.logInfo("BOOTSTRAP: " + sc + " seeds known from previous run");
// - load the superseed: a list of URLs
disorderSet superseed = loadSuperseed(superseedFile, superseedURL);
// - use the superseed to further fill up the seedDB
int ssc = 0;
for (int i = 0; i < superseed.size(); i++) {
int ssc = 0, c = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) break;
seedListFileURL = (String) superseed.any();
seedListFileURL = sb.getConfig("network.unit.bootstrap.seedlist" + c, "");
if (seedListFileURL.length() == 0) break;
c++;
if (
seedListFileURL.startsWith("http://") ||
seedListFileURL.startsWith("https://")
@ -216,53 +205,6 @@ public class yacyPeerActions {
yacyCore.log.logInfo("BOOTSTRAP: " + (seedDB.sizeConnected() - sc) + " new seeds while bootstraping.");
}
private disorderSet loadSuperseed(File local, String url) {
// this returns a list of locations where seed list-files can be found
disorderSet supsee = new disorderSet();
String line;
// read in local file
int lc = 0;
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(local)));
while ((line = br.readLine()) != null) {
line = line.trim();
//System.out.println("one line in file:" + line);
if (line.length() > 0) supsee.add(line);
}
br.close();
lc = supsee.size();
yacyCore.log.logInfo("BOOTSTRAP: " + lc + " seed-list URLs from superseed file " + local.toString());
} catch (IOException e) {
//e.printStackTrace();
supsee = new disorderSet();
yacyCore.log.logInfo("BOOTSTRAP: failed to load seed-list URLs from superseed file " + local.toString() + ": " + e.getMessage());
} finally {
if (br!=null)try{br.close();}catch(Exception e){}
}
// read in remote file from url
try {
URL u = new URL(url);
ArrayList remote = nxTools.strings(httpc.wget(u, u.getHost(), 5000, null, null, this.sb.remoteProxyConfig), "UTF-8");
if ((remote != null) && (remote.size() > 0)) {
Iterator e = remote.iterator();
while (e.hasNext()) {
line = (String) e.next();
if (line != null) {
line = line.trim();
supsee.add(line);
}
}
}
yacyCore.log.logInfo("BOOTSTRAP: " + (supsee.size() - lc) + " seed-list URLs from superseed URL " + url);
} catch (Exception e) {
supsee = new disorderSet();
yacyCore.log.logInfo("BOOTSTRAP: failed to load seed-list URLs from superseed URL " + url + ": " + e.getMessage());
}
return supsee;
}
private synchronized boolean connectPeer(yacySeed seed, boolean direct) {
// store a remote peer's seed
// returns true if the peer is new and previously unknown

7
superseed.txt
Unescape View File

@ -1,7 +0,0 @@
http://breyvogel.gmxhome.de/seed.txt
http://de.geocities.com/d15511z/yacy/angel/seed.txt
http://home.arcor.de/hermens/yacy/seed.txt
http://low.audioattack.de/yacy/seed.txt
http://thq.homeip.net/share/seed.txt
http://www.lulabad.de/seed.txt
http://www.yacy.net/yacy/seed.txt

12
yacy.network.unit
Unescape View File

@ -32,8 +32,7 @@
# network.unit.description = <any string, just informal; appears in network graphic>
# network.unit.domain = 'global'|'local'
# network.unit.dhtredundancy = <integer number, 0 means no DHT enabled>
# network.unit.bootstrap.file = <a file in the application directory containing urls to seedlists>
# network.unit.bootstrap.url = <a url to a bootstrap.file containing urls to seedlists>
# network.unit.bootstrap.seedlist<n> = <an url to a seedlists-file, which is stored by a principal peer>
# network.unit.protocol.control = 'uncontrolled'|'moderated'|'controlled'
# network.unit.protocol.request.authentification.method = 'salted-magic'
# network.unit.protocol.request.authentification.essentials = <any string, a secret password>
@ -55,8 +54,13 @@ network.unit.description = Public YaCy Community
#network.unit.domain = global
network.unit.dhtredundancy.junior = 1
network.unit.dhtredundancy.senior = 3
network.unit.bootstrap.file = superseed.txt
network.unit.bootstrap.url = http://www.yacy.net/superseed.txt
network.unit.bootstrap.seedlist0 = http://de.geocities.com/d15511z/yacy/angel/seed.txt
network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt
network.unit.bootstrap.seedlist2 = http://low.audioattack.de/yacy/seed.txt
network.unit.bootstrap.seedlist3 = http://www.lulabad.de/seed.txt
network.unit.bootstrap.seedlist4 = http://www.yacy.net/yacy/seed.txt
network.unit.bootstrap.seedlist5 = http://www.marcelhenseler.de/yacy/seed.txt
network.unit.bootstrap.seedlist6 = http://www.dc-art.de/yacy/seed.txt
# properties for in-protocol response authentification:
#network.unit.protocol.control = uncontrolled

Write Preview
Loading…
Cancel
Save