From f3ac923a7e6623c6f2fc7b5f1416936abf7b19f2 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 28 Dec 2013 22:42:02 +0100 Subject: [PATCH] ftp client shall be able to open non-anonymous ftp servers if login details are given --- .../yacy/cora/document/id/MultiProtocolURL.java | 2 +- source/net/yacy/cora/protocol/Scanner.java | 2 +- .../net/yacy/cora/protocol/ftp/FTPClient.java | 17 +++++++++-------- source/net/yacy/crawler/CrawlStacker.java | 13 +++++++++---- .../net/yacy/crawler/retrieval/FTPLoader.java | 2 +- source/net/yacy/search/Switchboard.java | 7 ++++++- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index c1b2000bf..b901c0d7c 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -927,7 +927,7 @@ public class MultiProtocolURL implements Serializable, Comparable list = ftpClient.list("/", false); ftpClient.CLOSE(); access = list == null || list.isEmpty() ? Access.empty : Access.granted; diff --git a/source/net/yacy/cora/protocol/ftp/FTPClient.java b/source/net/yacy/cora/protocol/ftp/FTPClient.java index 47448191b..2e3acf185 100644 --- a/source/net/yacy/cora/protocol/ftp/FTPClient.java +++ b/source/net/yacy/cora/protocol/ftp/FTPClient.java @@ -71,6 +71,7 @@ import net.yacy.cora.util.ConcurrentLog; public class FTPClient { + public static final String ANONYMOUS = "anonymous"; private static final ConcurrentLog log = new ConcurrentLog("FTPClient"); private static final String vDATE = "20100823"; @@ -2537,10 +2538,10 @@ public class FTPClient { * @return a list of entryInfo from all files of the ftp server * @throws IOException */ - public static BlockingQueue sitelist(final String host, final int port) throws IOException { + public static BlockingQueue sitelist(final String host, final int port, final String user, final String pw) throws IOException { final FTPClient ftpClient = new FTPClient(); ftpClient.open(host, port); - ftpClient.login("anonymous", "anomic@"); + ftpClient.login(user, pw); final LinkedBlockingQueue queue = new LinkedBlockingQueue(); new Thread() { @Override @@ -2603,7 +2604,7 @@ public class FTPClient { final String pwd = pwd(); final List list = list(remotePath, true); if (this.remotesystem == null) try {sys();} catch (final IOException e) {} - final String base = "ftp://" + ((this.account.equals("anonymous")) ? "" : (this.account + ":" + this.password + "@")) + final String base = "ftp://" + ((this.account.equals(ANONYMOUS)) ? "" : (this.account + ":" + this.password + "@")) + this.host + ((this.port == 21) ? "" : (":" + this.port)) + ((remotePath.length() > 0 && remotePath.charAt(0) == '/') ? "" : pwd + "/") + remotePath; @@ -2742,7 +2743,7 @@ public class FTPClient { } public static void getAnonymous(final String host, final String remoteFile, final File localPath) { - get(host, remoteFile, localPath, "anonymous", "anomic"); + get(host, remoteFile, localPath, ANONYMOUS, "anomic"); } /** @@ -2812,7 +2813,7 @@ public class FTPClient { final FTPClient ftpClient = new FTPClient(); try { ftpClient.open("192.168.1.90", 21); - ftpClient.login("anonymous", "anomic@"); + ftpClient.login(ANONYMOUS, "anomic@"); final byte[] b = ftpClient.get("/Movie/ATest Ordner/Unterordner/test file.txt"); System.out.println(UTF8.String(b)); } catch (final IOException e) { @@ -2823,10 +2824,10 @@ public class FTPClient { printHelp(); } else if (args.length == 3) { if (args[0].equals("-dir")) { - dir(args[1], args[2], "anonymous", "anomic@"); + dir(args[1], args[2], ANONYMOUS, "anomic@"); } else if (args[0].equals("-htmldir")) { try { - final StringBuilder page = dirhtml(args[1], 21, args[2], "anonymous", "anomic@"); + final StringBuilder page = dirhtml(args[1], 21, args[2], ANONYMOUS, "anomic@"); final File file = new File("dirindex.html"); FileOutputStream fos; fos = new FileOutputStream(file); @@ -2839,7 +2840,7 @@ public class FTPClient { } } else if (args[0].equals("-sitelist")) { try { - final BlockingQueue q = sitelist(args[1], Integer.parseInt(args[2])); + final BlockingQueue q = sitelist(args[1], Integer.parseInt(args[2]), ANONYMOUS, "anomic"); entryInfo entry; while ((entry = q.take()) != FTPClient.POISON_entryInfo) { System.out.println(entry.toString()); diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index 12123e3f8..461b923c9 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -212,7 +212,11 @@ public final class CrawlStacker { if (url.getProtocol().equals("ftp")) { // put the whole ftp site on the crawl stack - enqueueEntriesFTP(initiator, profileHandle, url.getHost(), url.getPort(), replace); + String userInfo = url.getUserInfo(); + int p = userInfo == null ? -1 : userInfo.indexOf(':'); + String user = userInfo == null ? FTPClient.ANONYMOUS : userInfo.substring(0, p); + String pw = userInfo == null || p == -1 ? "anomic" : userInfo.substring(p + 1); + enqueueEntriesFTP(initiator, profileHandle, url.getHost(), url.getPort(), user, pw, replace); } else { // put entry on crawl stack enqueueEntry(new Request( @@ -231,7 +235,7 @@ public final class CrawlStacker { } } - public void enqueueEntriesFTP(final byte[] initiator, final String profileHandle, final String host, final int port, final boolean replace) { + public void enqueueEntriesFTP(final byte[] initiator, final String profileHandle, final String host, final int port, final String user, final String pw, final boolean replace) { final CrawlQueues cq = this.nextQueue; new Thread() { @Override @@ -239,14 +243,14 @@ public final class CrawlStacker { Thread.currentThread().setName("enqueueEntriesFTP"); BlockingQueue queue; try { - queue = FTPClient.sitelist(host, port); + queue = FTPClient.sitelist(host, port, user, pw); FTPClient.entryInfo entry; while ((entry = queue.take()) != FTPClient.POISON_entryInfo) { // delete old entry, if exists to force a re-load of the url (thats wanted here) DigestURL url = null; try { - url = new DigestURL("ftp://" + host + (port == 21 ? "" : ":" + port) + MultiProtocolURL.escape(entry.name)); + url = new DigestURL("ftp://" + user + ":" + pw + "@" + host + (port == 21 ? "" : ":" + port) + MultiProtocolURL.escape(entry.name)); } catch (final MalformedURLException e) { continue; } @@ -271,6 +275,7 @@ public final class CrawlStacker { )); } } catch (final IOException e1) { + ConcurrentLog.logException(e1); } catch (final InterruptedException e) { } } diff --git a/source/net/yacy/crawler/retrieval/FTPLoader.java b/source/net/yacy/crawler/retrieval/FTPLoader.java index aaf3b6c2a..02e4b371b 100644 --- a/source/net/yacy/crawler/retrieval/FTPLoader.java +++ b/source/net/yacy/crawler/retrieval/FTPLoader.java @@ -179,7 +179,7 @@ public class FTPLoader { private boolean openConnection(final FTPClient ftpClient, final DigestURL entryUrl) { // get username and password final String userInfo = entryUrl.getUserInfo(); - String userName = "anonymous", userPwd = "anonymous"; + String userName = FTPClient.ANONYMOUS, userPwd = "anomic"; if (userInfo != null) { final int pos = userInfo.indexOf(':',0); if (pos != -1) { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 1c164816d..52783614e 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -115,6 +115,7 @@ import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.protocol.TimeoutRequest; +import net.yacy.cora.protocol.ftp.FTPClient; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.protocol.http.ProxySettings; import net.yacy.cora.util.ConcurrentLog; @@ -2962,7 +2963,11 @@ public final class Switchboard extends serverSwitch { if (url.isFTP()) { try { this.crawler.putActive(handle, profile); - this.crawlStacker.enqueueEntriesFTP(this.peers.mySeed().hash.getBytes(), profile.handle(), url.getHost(), url.getPort(), false); + String userInfo = url.getUserInfo(); + int p = userInfo == null ? -1 : userInfo.indexOf(':'); + String user = userInfo == null ? FTPClient.ANONYMOUS : userInfo.substring(0, p); + String pw = userInfo == null || p == -1 ? "anomic" : userInfo.substring(p + 1); + this.crawlStacker.enqueueEntriesFTP(this.peers.mySeed().hash.getBytes(), profile.handle(), url.getHost(), url.getPort(), user, pw, false); return null; } catch (final Exception e) { // mist