boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

ftp client shall be able to open non-anonymous ftp servers if login

Browse Source 
details are given
pull/1/head
orbiter 11 years ago
parent 3d913558ab
commit f3ac923a7e
6 changed files with 27 additions and 16 deletions
Show Stats Download Patch File Download Diff File

2
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -927,7 +927,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
u.append(this.protocol);
u.append("://");
if (h != null) {
if (this.userInfo != null) {
if (this.userInfo != null && !(this.isFTP() && this.userInfo.startsWith(FTPClient.ANONYMOUS))) {
u.append(this.userInfo);
u.append("@");
}

2
source/net/yacy/cora/protocol/Scanner.java
Unescape View File

@ -131,7 +131,7 @@ public class Scanner {
final FTPClient ftpClient = new FTPClient();
try {
ftpClient.open(this.getInetAddress().getHostAddress(), this.getProtocol().port);
ftpClient.login("anonymous", "anomic@");
ftpClient.login(FTPClient.ANONYMOUS, "anomic@");
final List<String> list = ftpClient.list("/", false);
ftpClient.CLOSE();
access = list == null || list.isEmpty() ? Access.empty : Access.granted;

17
source/net/yacy/cora/protocol/ftp/FTPClient.java
Unescape View File

@ -71,6 +71,7 @@ import net.yacy.cora.util.ConcurrentLog;
public class FTPClient {
public static final String ANONYMOUS = "anonymous";
private static final ConcurrentLog log = new ConcurrentLog("FTPClient");
private static final String vDATE = "20100823";
@ -2537,10 +2538,10 @@ public class FTPClient {
* @return a list of entryInfo from all files of the ftp server
* @throws IOException
*/
public static BlockingQueue<entryInfo> sitelist(final String host, final int port) throws IOException {
public static BlockingQueue<entryInfo> sitelist(final String host, final int port, final String user, final String pw) throws IOException {
final FTPClient ftpClient = new FTPClient();
ftpClient.open(host, port);
ftpClient.login("anonymous", "anomic@");
ftpClient.login(user, pw);
final LinkedBlockingQueue<entryInfo> queue = new LinkedBlockingQueue<entryInfo>();
new Thread() {
@Override
@ -2603,7 +2604,7 @@ public class FTPClient {
final String pwd = pwd();
final List<String> list = list(remotePath, true);
if (this.remotesystem == null) try {sys();} catch (final IOException e) {}
final String base = "ftp://" + ((this.account.equals("anonymous")) ? "" : (this.account + ":" + this.password + "@"))
final String base = "ftp://" + ((this.account.equals(ANONYMOUS)) ? "" : (this.account + ":" + this.password + "@"))
+ this.host + ((this.port == 21) ? "" : (":" + this.port)) + ((remotePath.length() > 0 && remotePath.charAt(0) == '/') ? "" : pwd + "/")
+ remotePath;
@ -2742,7 +2743,7 @@ public class FTPClient {
}
public static void getAnonymous(final String host, final String remoteFile, final File localPath) {
get(host, remoteFile, localPath, "anonymous", "anomic");
get(host, remoteFile, localPath, ANONYMOUS, "anomic");
}
/**
@ -2812,7 +2813,7 @@ public class FTPClient {
final FTPClient ftpClient = new FTPClient();
try {
ftpClient.open("192.168.1.90", 21);
ftpClient.login("anonymous", "anomic@");
ftpClient.login(ANONYMOUS, "anomic@");
final byte[] b = ftpClient.get("/Movie/ATest Ordner/Unterordner/test file.txt");
System.out.println(UTF8.String(b));
} catch (final IOException e) {
@ -2823,10 +2824,10 @@ public class FTPClient {
printHelp();
} else if (args.length == 3) {
if (args[0].equals("-dir")) {
dir(args[1], args[2], "anonymous", "anomic@");
dir(args[1], args[2], ANONYMOUS, "anomic@");
} else if (args[0].equals("-htmldir")) {
try {
final StringBuilder page = dirhtml(args[1], 21, args[2], "anonymous", "anomic@");
final StringBuilder page = dirhtml(args[1], 21, args[2], ANONYMOUS, "anomic@");
final File file = new File("dirindex.html");
FileOutputStream fos;
fos = new FileOutputStream(file);
@ -2839,7 +2840,7 @@ public class FTPClient {
}
} else if (args[0].equals("-sitelist")) {
try {
final BlockingQueue<entryInfo> q = sitelist(args[1], Integer.parseInt(args[2]));
final BlockingQueue<entryInfo> q = sitelist(args[1], Integer.parseInt(args[2]), ANONYMOUS, "anomic");
entryInfo entry;
while ((entry = q.take()) != FTPClient.POISON_entryInfo) {
System.out.println(entry.toString());

13
source/net/yacy/crawler/CrawlStacker.java
Unescape View File

@ -212,7 +212,11 @@ public final class CrawlStacker {
if (url.getProtocol().equals("ftp")) {
// put the whole ftp site on the crawl stack
enqueueEntriesFTP(initiator, profileHandle, url.getHost(), url.getPort(), replace);
String userInfo = url.getUserInfo();
int p = userInfo == null ? -1 : userInfo.indexOf(':');
String user = userInfo == null ? FTPClient.ANONYMOUS : userInfo.substring(0, p);
String pw = userInfo == null || p == -1 ? "anomic" : userInfo.substring(p + 1);
enqueueEntriesFTP(initiator, profileHandle, url.getHost(), url.getPort(), user, pw, replace);
} else {
// put entry on crawl stack
enqueueEntry(new Request(
@ -231,7 +235,7 @@ public final class CrawlStacker {
}
}
public void enqueueEntriesFTP(final byte[] initiator, final String profileHandle, final String host, final int port, final boolean replace) {
public void enqueueEntriesFTP(final byte[] initiator, final String profileHandle, final String host, final int port, final String user, final String pw, final boolean replace) {
final CrawlQueues cq = this.nextQueue;
new Thread() {
@Override
@ -239,14 +243,14 @@ public final class CrawlStacker {
Thread.currentThread().setName("enqueueEntriesFTP");
BlockingQueue<FTPClient.entryInfo> queue;
try {
queue = FTPClient.sitelist(host, port);
queue = FTPClient.sitelist(host, port, user, pw);
FTPClient.entryInfo entry;
while ((entry = queue.take()) != FTPClient.POISON_entryInfo) {
// delete old entry, if exists to force a re-load of the url (thats wanted here)
DigestURL url = null;
try {
url = new DigestURL("ftp://" + host + (port == 21 ? "" : ":" + port) + MultiProtocolURL.escape(entry.name));
url = new DigestURL("ftp://" + user + ":" + pw + "@" + host + (port == 21 ? "" : ":" + port) + MultiProtocolURL.escape(entry.name));
} catch (final MalformedURLException e) {
continue;
}
@ -271,6 +275,7 @@ public final class CrawlStacker {
));
}
} catch (final IOException e1) {
ConcurrentLog.logException(e1);
} catch (final InterruptedException e) {
}
}

2
source/net/yacy/crawler/retrieval/FTPLoader.java
Unescape View File

@ -179,7 +179,7 @@ public class FTPLoader {
private boolean openConnection(final FTPClient ftpClient, final DigestURL entryUrl) {
// get username and password
final String userInfo = entryUrl.getUserInfo();
String userName = "anonymous", userPwd = "anonymous";
String userName = FTPClient.ANONYMOUS, userPwd = "anomic";
if (userInfo != null) {
final int pos = userInfo.indexOf(':',0);
if (pos != -1) {

7
source/net/yacy/search/Switchboard.java
Unescape View File

@ -115,6 +115,7 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.protocol.ftp.FTPClient;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.protocol.http.ProxySettings;
import net.yacy.cora.util.ConcurrentLog;
@ -2962,7 +2963,11 @@ public final class Switchboard extends serverSwitch {
if (url.isFTP()) {
try {
this.crawler.putActive(handle, profile);
this.crawlStacker.enqueueEntriesFTP(this.peers.mySeed().hash.getBytes(), profile.handle(), url.getHost(), url.getPort(), false);
String userInfo = url.getUserInfo();
int p = userInfo == null ? -1 : userInfo.indexOf(':');
String user = userInfo == null ? FTPClient.ANONYMOUS : userInfo.substring(0, p);
String pw = userInfo == null || p == -1 ? "anomic" : userInfo.substring(p + 1);
this.crawlStacker.enqueueEntriesFTP(this.peers.mySeed().hash.getBytes(), profile.handle(), url.getHost(), url.getPort(), user, pw, false);
return null;
} catch (final Exception e) {
// mist

Write Preview
Loading…
Cancel
Save