boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

extended seed list uploader: do not only upload all active peers but also some more peers that are passive but had been active in the last 24 hours

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7449 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 3fe03f153d
commit 0376f73fdb
8 changed files with 34 additions and 43 deletions
Show Stats Download Patch File Download Diff File

26
source/de/anomic/http/server/ChunkedInputStream.java
Unescape View File

@ -25,7 +25,7 @@
/*
* This file was imported from apache http client 3.1 library and modified
* to work for the YaCy http server when htto client library use was migrated
* to work for the YaCy http server when http client library use was migrated
* to apache http components 4.0
* by Michael Christen, 20.09.2010
*/
@ -139,15 +139,11 @@ public class ChunkedInputStream extends InputStream {
* @see java.io.InputStream#read(byte[], int, int)
* @throws IOException if an IO problem occurs.
*/
public int read (byte[] b, int off, int len) throws IOException {
public int read(byte[] b, int off, int len) throws IOException {
if (closed) {
throw new IOException("Attempted read from closed stream.");
}
if (eof) {
return -1;
}
if (closed) throw new IOException("Attempted read from closed stream.");
if (eof) return -1;
if (pos >= chunkSize) {
nextChunk();
if (eof) {
@ -168,7 +164,7 @@ public class ChunkedInputStream extends InputStream {
* @see java.io.InputStream#read(byte[])
* @throws IOException if an IO problem occurs.
*/
public int read (byte[] b) throws IOException {
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@ -178,11 +174,9 @@ public class ChunkedInputStream extends InputStream {
*/
private void readCRLF() throws IOException {
int cr = in.read();
if (cr != '\r') throw new IOException("CRLF expected at end of chunk: cr != " + cr);
int lf = in.read();
if ((cr != '\r') || (lf != '\n')) {
throw new IOException(
"CRLF expected at end of chunk: " + cr + "/" + lf);
}
if (lf != '\n') throw new IOException("CRLF expected at end of chunk: lf != " + lf);
}
@ -191,9 +185,7 @@ public class ChunkedInputStream extends InputStream {
* @throws IOException If an IO error occurs.
*/
private void nextChunk() throws IOException {
if (!bof) {
readCRLF();
}
if (!bof) readCRLF();
chunkSize = getChunkSizeFromInputStream(in);
bof = false;
pos = 0;

3
source/de/anomic/yacy/seedUpload/yacySeedUploadFile.java
Unescape View File

@ -30,14 +30,13 @@ import java.io.File;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacySeedUploader;
public class yacySeedUploadFile implements yacySeedUploader {
public static final String CONFIG_FILE_PATH = "seedFilePath";
public String uploadSeedFile(final serverSwitch sb, final yacySeedDB seedDB, final File seedFile) throws Exception {
public String uploadSeedFile(final serverSwitch sb, final File seedFile) throws Exception {
String seedFilePath = "";
try {

4
source/de/anomic/yacy/seedUpload/yacySeedUploadFtp.java
Unescape View File

@ -30,7 +30,6 @@ import java.io.File;
import net.yacy.cora.protocol.ftp.FTPClient;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacySeedUploader;
public class yacySeedUploadFtp implements yacySeedUploader {
@ -40,10 +39,9 @@ public class yacySeedUploadFtp implements yacySeedUploader {
public static final String CONFIG_FTP_PASSWORD = "seedFTPPassword";
public static final String CONFIG_FTP_PATH = "seedFTPPath";
public String uploadSeedFile (final serverSwitch sb, final yacySeedDB seedDB, final File seedFile) throws Exception {
public String uploadSeedFile (final serverSwitch sb, final File seedFile) throws Exception {
try {
if (sb == null) throw new NullPointerException("Reference to serverSwitch must not be null.");
if (seedDB == null) throw new NullPointerException("Reference to seedDB must not be null.");
if ((seedFile == null)||(!seedFile.exists())) throw new Exception("Seed file does not exist.");
if (!seedFile.isFile()) throw new Exception("Seed file is not a file.");
if (!seedFile.canRead()) throw new Exception("Seed file is not readable.");

4
source/de/anomic/yacy/seedUpload/yacySeedUploadScp.java
Unescape View File

@ -40,7 +40,6 @@ import com.jcraft.jsch.UIKeyboardInteractive;
import com.jcraft.jsch.UserInfo;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacySeedUploader;
public class yacySeedUploadScp implements yacySeedUploader {
@ -51,10 +50,9 @@ public class yacySeedUploadScp implements yacySeedUploader {
public static final String CONFIG_SCP_PASSWORD = "seedScpPassword";
public static final String CONFIG_SCP_PATH = "seedScpPath";
public String uploadSeedFile(final serverSwitch sb, final yacySeedDB seedDB, final File seedFile) throws Exception {
public String uploadSeedFile(final serverSwitch sb, final File seedFile) throws Exception {
try {
if (sb == null) throw new NullPointerException("Reference to serverSwitch nut not be null.");
if (seedDB == null) throw new NullPointerException("Reference to seedDB must not be null.");
if ((seedFile == null)||(!seedFile.exists())) throw new Exception("Seed file does not exist.");
final String seedScpServer = sb.getConfig(CONFIG_SCP_SERVER,null);

2
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -639,7 +639,7 @@ public class yacyCore {
if (log.isFine()) log.logFine("SaveSeedList: Using seed uploading method '" + seedUploadMethod + "' for seed-list uploading." +
"\n\tPrevious peerType is '" + sb.peers.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR) + "'.");
logt = sb.peers.uploadCache(uploader, sb, sb.peers, seedURL);
logt = sb.peers.uploadSeedList(uploader, sb, sb.peers, seedURL);
if (logt != null) {
if (logt.indexOf("Error") >= 0) {
sb.peers.mySeed().put(yacySeed.PEERTYPE, prevStatus);

9
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -500,15 +500,6 @@ public class yacySeed implements Cloneable {
return System.currentTimeMillis() - AbstractFormatter.dayMillis;
}
}
/**
* @see #getLastSeenUTC()
* @return the last seen value as string representation in the following format: YearMonthDayHoursMinutesSeconds
* or <code>20040101000000</code> if not present
*/
public final String getLastSeenString() {
return get(yacySeed.LASTSEEN, "20040101000000");
}
/** @return the age of the seed in number of days */
public final int getAge() {

27
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -764,14 +764,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
return null;
}
private ArrayList<String> storeCache(final File seedFile, final boolean addMySeed) throws IOException {
private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {
PrintWriter pw = null;
final ArrayList<String> v = new ArrayList<String>(seedActiveDB.size() + 1);
try {
pw = new PrintWriter(new BufferedWriter(new FileWriter(seedFile)));
// store own seed
// store own peer seed
String line;
if (this.mySeed == null) initMySeed();
if (addMySeed) {
@ -780,9 +780,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
pw.print(line + serverCore.CRLF_STRING);
}
// store other seeds
// store active peer seeds
yacySeed ys;
final Iterator<yacySeed> se = seedsConnected(true, false, null, (float) 0.0);
Iterator<yacySeed> se = seedsConnected(true, false, null, (float) 0.0);
while (se.hasNext()) {
ys = se.next();
if (ys != null) {
@ -791,6 +791,19 @@ public final class yacySeedDB implements AlternativeDomainNames {
pw.print(line + serverCore.CRLF_STRING);
}
}
// store some of the not-so-old passive peer seeds (limit: 1 day)
se = seedsDisconnected(true, false, null, (float) 0.0);
long timeout = System.currentTimeMillis() - (1000L * 60L * 60L * 24L);
while (se.hasNext()) {
ys = se.next();
if (ys != null) {
if (ys.getLastSeenUTC() < timeout) continue;
line = ys.genSeedStr(null);
v.add(line);
pw.print(line + serverCore.CRLF_STRING);
}
}
pw.flush();
} finally {
if (pw != null) try { pw.close(); } catch (final Exception e) {}
@ -798,7 +811,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
return v;
}
protected String uploadCache(final yacySeedUploader uploader,
protected String uploadSeedList(final yacySeedUploader uploader,
final serverSwitch sb,
final yacySeedDB seedDB,
final DigestURI seedURL) throws Exception {
@ -813,11 +826,11 @@ public final class yacySeedDB implements AlternativeDomainNames {
seedFile = File.createTempFile("seedFile",".txt", seedDB.myOwnSeedFile.getParentFile());
seedFile.deleteOnExit();
if (Log.isFine("YACY")) Log.logFine("YACY", "SaveSeedList: Storing seedlist into tempfile " + seedFile.toString());
final ArrayList<String> uv = storeCache(seedFile, true);
final ArrayList<String> uv = storeSeedList(seedFile, true);
// uploading the seed file
if (Log.isFine("YACY")) Log.logFine("YACY", "SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
log = uploader.uploadSeedFile(sb,seedDB,seedFile);
log = uploader.uploadSeedFile(sb, seedFile);
// test download
if (Log.isFine("YACY")) Log.logFine("YACY", "SaveSeedList: Trying to download seed-file '" + seedURL + "'.");

2
source/de/anomic/yacy/yacySeedUploader.java
Unescape View File

@ -29,6 +29,6 @@ import java.io.File;
import de.anomic.server.serverSwitch;
public interface yacySeedUploader {
public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception;
public String uploadSeedFile(serverSwitch sb, File seedFile) throws Exception;
public String[] getConfigurationOptions();
}

Write Preview
Loading…
Cancel
Save