boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

FTP:

Browse Source 
- report connection status (to break if no connection possible)
- fixed isFolder()
- additional error output
- fixed paths with encoded symbols (ie. a%20file.txt)
- refactoring


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4567 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent 820641938e
commit f51bad8ae5
2 changed files with 187 additions and 117 deletions
Show Stats Download Patch File Download Diff File

149
source/de/anomic/net/ftpc.java
Unescape View File

@ -249,11 +249,11 @@ public class ftpc {
.booleanValue()); .booleanValue());
} catch (final InvocationTargetException e) { } catch (final InvocationTargetException e) {
if (e.getMessage() == null) { if (e.getMessage() == null) {
} else if (ControlSocket == null) { } else if (notConnected()) {
// the error was probably caused because there is no // the error was probably caused because there is no
// connection // connection
errPrintln("not connected. no effect."); errPrintln("not connected. no effect.");
e.printStackTrace(); e.printStackTrace(err);
return ret; return ret;
} else { } else {
errPrintln("ftp internal exception: target exception " + e); errPrintln("ftp internal exception: target exception " + e);
@ -266,7 +266,7 @@ public class ftpc {
// consider first that the user attempted to execute a java // consider first that the user attempted to execute a java
// command from // command from
// the current path; either local or remote // the current path; either local or remote
if (ControlSocket == null) { if (notConnected()) {
// try a local exec // try a local exec
try { try {
javaexec(cmd); javaexec(cmd);
@ -470,7 +470,7 @@ public class ftpc {
errPrintln("Syntax: CD <path>"); errPrintln("Syntax: CD <path>");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LCD(); return LCD();
} }
try { try {
@ -531,7 +531,7 @@ public class ftpc {
errPrintln("Syntax: DEL <file>"); errPrintln("Syntax: DEL <file>");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LDEL(); return LDEL();
} }
try { try {
@ -551,7 +551,7 @@ public class ftpc {
errPrintln("Syntax: DIR [<path>|<file>]"); errPrintln("Syntax: DIR [<path>|<file>]");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LDIR(); return LDIR();
} }
try { try {
@ -575,18 +575,21 @@ public class ftpc {
} catch (final IOException e) { } catch (final IOException e) {
errPrintln("Connection to server lost."); errPrintln("Connection to server lost.");
} }
ControlSocket = null; try {
DataSocketActive = null; closeConnection();
DataSocketPassive = null; } catch (final IOException e) {
clientInput = null; ControlSocket = null;
clientOutput = null; DataSocketActive = null;
DataSocketPassive = null;
clientInput = null;
clientOutput = null;
}
prompt = "ftp [local]>"; prompt = "ftp [local]>";
return true; return true;
} }
private String quit() throws IOException { private String quit() throws IOException {
// send delete command
send("QUIT"); send("QUIT");
// read status reply // read status reply
@ -595,24 +598,7 @@ public class ftpc {
throw new IOException(reply); throw new IOException(reply);
} }
// cleanup closeConnection();
if (ControlSocket != null) {
clientOutput.close();
clientInput.close();
ControlSocket.close();
ControlSocket = null;
}
if (DataSocketActive != null) {
DataSocketActive.close();
DataSocketActive = null;
}
if (DataSocketPassive != null) {
DataSocketPassive.close();
DataSocketPassive = null; // "Once a socket has been closed, it is
// not available for further networking
// use"
}
return reply; return reply;
} }
@ -633,8 +619,8 @@ public class ftpc {
final File local = absoluteLocalFile(localFilename); final File local = absoluteLocalFile(localFilename);
if (local.exists()) { if (local.exists()) {
errPrintln("Error: local file " + local.toString() + " already exists."); errPrintln("Error: local file " + local.toString() + " already exists.\n" + " File " + remote
errPrintln(logPrefix + " File " + remote + " not retrieved. Local file unchanged."); + " not retrieved. Local file unchanged.");
} else { } else {
if (withoutLocalFile) { if (withoutLocalFile) {
retrieveFilesRecursively(remote, false); retrieveFilesRecursively(remote, false);
@ -732,7 +718,8 @@ public class ftpc {
} }
// check if we actually changed into the folder // check if we actually changed into the folder
final String changedPath = pwd(); final String changedPath = pwd();
if (!(changedPath.equals(path) || changedPath.equals(currentFolder + "/" + path))) { if (!(changedPath.equals(path) || changedPath.equals(currentFolder
+ (currentFolder.endsWith("/") ? "" : "/") + path))) {
throw new IOException("folder is '" + changedPath + "' should be '" + path + "'"); throw new IOException("folder is '" + changedPath + "' should be '" + path + "'");
} }
// return to last folder // return to last folder
@ -1172,7 +1159,7 @@ public class ftpc {
final String dateString = tokens.group(3) + " " + tokens.group(4) + " " + year + " " + time; final String dateString = tokens.group(3) + " " + tokens.group(4) + " " + year + " " + time;
try { try {
date = lsDateFormat.parse(dateString); date = lsDateFormat.parse(dateString);
} catch (ParseException e) { } catch (final ParseException e) {
errPrintln(logPrefix + "---- Error: not ls date-format '" + dateString + "': " + e.getMessage()); errPrintln(logPrefix + "---- Error: not ls date-format '" + dateString + "': " + e.getMessage());
date = new Date(); date = new Date();
} }
@ -1391,7 +1378,7 @@ public class ftpc {
errPrintln("Syntax: LS [<path>|<file>]"); errPrintln("Syntax: LS [<path>|<file>]");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LLS(); return LLS();
} }
try { try {
@ -1415,7 +1402,6 @@ public class ftpc {
outPrintln("---- v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v"); outPrintln("---- v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v---v");
for (final String element : list) { for (final String element : list) {
outPrintln(element); outPrintln(element);
outPrintln("--> " + parseListData(element));
} }
outPrintln("---- ^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^"); outPrintln("---- ^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^---^");
} }
@ -1477,7 +1463,7 @@ public class ftpc {
errPrintln("Syntax: MKDIR <folder-name>"); errPrintln("Syntax: MKDIR <folder-name>");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LMKDIR(); return LMKDIR();
} }
try { try {
@ -1545,7 +1531,7 @@ public class ftpc {
errPrintln("Syntax: MV <from> <to>"); errPrintln("Syntax: MV <from> <to>");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LMV(); return LMV();
} }
try { try {
@ -1604,7 +1590,7 @@ public class ftpc {
outPrintln("---- Connection to " + cmd[1] + " established."); outPrintln("---- Connection to " + cmd[1] + " established.");
prompt = "ftp [" + cmd[1] + "]>"; prompt = "ftp [" + cmd[1] + "]>";
} catch (final IOException e) { } catch (final IOException e) {
errPrintln("Error: connecting " + cmd[1] + " on port " + port + " failed."); errPrintln("Error: connecting " + cmd[1] + " on port " + port + " failed: " + e.getMessage());
} }
return true; return true;
} }
@ -1614,17 +1600,56 @@ public class ftpc {
exec("close", false); // close any existing connections first exec("close", false); // close any existing connections first
} }
ControlSocket = new Socket(host, port); try {
ControlSocket.setSoTimeout(getTimeout()); ControlSocket = new Socket(host, port);
clientInput = new BufferedReader(new InputStreamReader(ControlSocket.getInputStream())); ControlSocket.setSoTimeout(getTimeout());
clientOutput = new DataOutputStream(new BufferedOutputStream(ControlSocket.getOutputStream())); clientInput = new BufferedReader(new InputStreamReader(ControlSocket.getInputStream()));
clientOutput = new DataOutputStream(new BufferedOutputStream(ControlSocket.getOutputStream()));
// read and return server message
this.host = host;
this.port = port;
remotemessage = receive();
if ((remotemessage != null) && (remotemessage.length() > 3)) {
remotemessage = remotemessage.substring(4);
}
} catch (final IOException e) {
// if a connection was opened, it should not be used
closeConnection();
throw new IOException(e);
}
}
/**
* @return
*/
public boolean notConnected() {
return ControlSocket == null;
}
// read and return server message /**
this.host = host; * close all sockets
this.port = port; *
remotemessage = receive(); * @throws IOException
if ((remotemessage != null) && (remotemessage.length() > 3)) { */
remotemessage = remotemessage.substring(4); private void closeConnection() throws IOException {
// cleanup
if (ControlSocket != null) {
clientOutput.close();
clientInput.close();
ControlSocket.close();
ControlSocket = null;
}
if (DataSocketActive != null) {
DataSocketActive.close();
DataSocketActive = null;
}
if (DataSocketPassive != null) {
DataSocketPassive.close();
DataSocketPassive = null; // "Once a socket has been closed, it is
// not available for further networking
// use"
} }
} }
@ -1658,7 +1683,7 @@ public class ftpc {
errPrintln("Syntax: PWD (no parameter)"); errPrintln("Syntax: PWD (no parameter)");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LPWD(); return LPWD();
} }
try { try {
@ -1701,7 +1726,7 @@ public class ftpc {
errPrintln("Syntax: RMDIR <folder-name>"); errPrintln("Syntax: RMDIR <folder-name>");
return true; return true;
} }
if (ControlSocket == null) { if (notConnected()) {
return LRMDIR(); return LRMDIR();
} }
try { try {
@ -1713,7 +1738,7 @@ public class ftpc {
} }
public boolean QUIT() { public boolean QUIT() {
if (ControlSocket != null) { if (!notConnected()) {
exec("close", false); exec("close", false);
} }
return false; return false;
@ -1780,7 +1805,7 @@ public class ftpc {
login(cmd[1], cmd[2]); login(cmd[1], cmd[2]);
outPrintln("---- Granted access for user " + cmd[1] + "."); outPrintln("---- Granted access for user " + cmd[1] + ".");
} catch (final IOException e) { } catch (final IOException e) {
errPrintln("Error: authorization of user " + cmd[1] + " failed."); errPrintln("Error: authorization of user " + cmd[1] + " failed: " + e.getMessage());
} }
return true; return true;
} }
@ -2354,6 +2379,7 @@ public class ftpc {
* @throws IOException * @throws IOException
*/ */
private void login(final String account, final String password) throws IOException { private void login(final String account, final String password) throws IOException {
unsetLoginData();
// send user name // send user name
send("USER " + account); send("USER " + account);
@ -2380,6 +2406,15 @@ public class ftpc {
setLoginData(account, password, reply); setLoginData(account, password, reply);
} }
/**
* we are authorized to use the server
*
* @return
*/
public boolean isLoggedIn() {
return (account != null && password != null && remotegreeting != null);
}
/** /**
* remember username and password which were used to login * remember username and password which were used to login
* *
@ -2394,6 +2429,12 @@ public class ftpc {
remotegreeting = reply; remotegreeting = reply;
} }
private void unsetLoginData() {
account = null;
password = null;
remotegreeting = null;
}
public void sys() throws IOException { public void sys() throws IOException {
// send system command // send system command
send("SYST"); send("SYST");
@ -2451,7 +2492,7 @@ public class ftpc {
* @param timeout * @param timeout
* in seconds, 0 = infinite * in seconds, 0 = infinite
*/ */
public void setDataSocketTimeout(int timeout) { public void setDataSocketTimeout(final int timeout) {
DataSocketTimeout = timeout; DataSocketTimeout = timeout;
try { try {

155
source/de/anomic/plasma/crawler/plasmaFTPLoader.java
Unescape View File

@ -90,7 +90,11 @@ public class plasmaFTPLoader {
*/ */
public plasmaHTCache.Entry load(final plasmaCrawlEntry entry) { public plasmaHTCache.Entry load(final plasmaCrawlEntry entry) {
final yacyURL entryUrl = entry.url(); final yacyURL entryUrl = entry.url();
final String fullPath = entryUrl.getPath(); final String fullPath = getPath(entryUrl);
final File cacheFile = createCachefile(entryUrl);
// the return value
plasmaHTCache.Entry htCache = null;
// determine filename and path // determine filename and path
String file, path; String file, path;
@ -113,72 +117,78 @@ public class plasmaFTPLoader {
final ByteArrayOutputStream berr = new ByteArrayOutputStream(); final ByteArrayOutputStream berr = new ByteArrayOutputStream();
final ftpc ftpClient = createFTPClient(berr); final ftpc ftpClient = createFTPClient(berr);
plasmaHTCache.Entry htCache = null; if (openConnection(ftpClient, entryUrl)) {
try { // ftp stuff
openConnection(ftpClient, entryUrl); try {
// testing if the specified file is a directory
// testing if the specified file is a directory if (file.length() > 0) {
if (file.length() > 0) { ftpClient.exec("cd \"" + path + "\"", false);
ftpClient.exec("cd \"" + path + "\"", false);
final boolean isFolder = ftpClient.isFolder(file);
// testing if the current name is a directoy if (isFolder) {
final boolean isFolder = ftpClient.isFolder(file); path = fullPath + "/";
if (isFolder) { file = "";
path = fullPath + "/"; }
file = "";
} }
}
// creating a cache file object
final File cacheFile = plasmaHTCache.getCachePath(entryUrl);
// TODO: invalid file path check
// testing if the file already exists
if (cacheFile.isFile()) {
// delete the file if it already exists
plasmaHTCache.deleteURLfromCache(entryUrl);
} else {
// create parent directories
cacheFile.getParentFile().mkdirs();
}
if (file.length() == 0) { if (file.length() == 0) {
// directory -> get list of files // directory -> get list of files
// create a htcache entry // create a htcache entry
htCache = createCacheEntry(entry, "text/html", new Date()); htCache = createCacheEntry(entry, "text/html", new Date());
if (!generateDirlist(ftpClient, entry, path, cacheFile)) { if (!generateDirlist(ftpClient, entry, path, cacheFile)) {
htCache = null; htCache = null;
} }
} else { } else {
// file -> download // file -> download
try { try {
htCache = getFile(ftpClient, entry, cacheFile); htCache = getFile(ftpClient, entry, cacheFile);
} catch (final Exception e) { } catch (final Exception e) {
// add message to errorLog
(new PrintStream(berr)).print(e.getMessage());
}
} }
} finally {
closeConnection(ftpClient);
} }
}
// pass the downloaded resource to the cache manager // pass the downloaded resource to the cache manager
if (berr.size() > 0 || htCache == null) { if (berr.size() > 0 || htCache == null) {
// some error logging // some error logging
final String detail = (berr.size() > 0) ? "\n Errorlog: " + berr.toString() : ""; final String detail = (berr.size() > 0) ? "\n Errorlog: " + berr.toString() : "";
log.logWarning("Unable to download URL " + entry.url().toString() + detail); log.logWarning("Unable to download URL " + entry.url().toString() + detail);
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
// an error has occured. cleanup // an error has occured. cleanup
if (cacheFile.exists()) { if (cacheFile.exists()) {
cacheFile.delete(); cacheFile.delete();
}
} else {
// announce the file
plasmaHTCache.writeFileAnnouncement(cacheFile);
} }
} else {
// announce the file
plasmaHTCache.writeFileAnnouncement(cacheFile);
}
return htCache; return htCache;
} finally { }
closeConnection(ftpClient);
/**
* creating a cache file object
*
* @param entryUrl
* @return
*/
private File createCachefile(final yacyURL entryUrl) {
final File cacheFile = plasmaHTCache.getCachePath(entryUrl);
// testing if the file already exists
if (cacheFile.isFile()) {
// delete the file if it already exists
plasmaHTCache.deleteURLfromCache(entryUrl);
} else {
// create parent directories
cacheFile.getParentFile().mkdirs();
} }
return cacheFile;
} }
/** /**
@ -196,8 +206,9 @@ public class plasmaFTPLoader {
* @param ftpClient * @param ftpClient
* @param host * @param host
* @param port * @param port
* @return success
*/ */
private void openConnection(final ftpc ftpClient, final yacyURL entryUrl) { private boolean openConnection(final ftpc ftpClient, final yacyURL entryUrl) {
// get username and password // get username and password
final String userInfo = entryUrl.getUserInfo(); final String userInfo = entryUrl.getUserInfo();
String userName = "anonymous", userPwd = "anonymous"; String userName = "anonymous", userPwd = "anonymous";
@ -218,12 +229,20 @@ public class plasmaFTPLoader {
} else { } else {
ftpClient.exec("open " + host + " " + port, false); ftpClient.exec("open " + host + " " + port, false);
} }
if (ftpClient.notConnected()) {
return false;
}
// login to the server // login to the server
ftpClient.exec("user " + userName + " " + userPwd, false); ftpClient.exec("user " + userName + " " + userPwd, false);
// change transfer mode to binary if (ftpClient.isLoggedIn()) {
ftpClient.exec("binary", false); // change transfer mode to binary
ftpClient.exec("binary", false);
} else {
return false;
}
return true;
} }
/** /**
@ -240,7 +259,7 @@ public class plasmaFTPLoader {
final yacyURL entryUrl = entry.url(); final yacyURL entryUrl = entry.url();
final String extension = plasmaParser.getFileExt(entryUrl); final String extension = plasmaParser.getFileExt(entryUrl);
final String mimeType = plasmaParser.getMimeTypeByFileExt(extension); final String mimeType = plasmaParser.getMimeTypeByFileExt(extension);
final String path = entryUrl.getPath(); final String path = getPath(entryUrl);
// if the mimetype and file extension is supported we start to download // if the mimetype and file extension is supported we start to download
// the file // the file
@ -264,7 +283,7 @@ public class plasmaFTPLoader {
log.logInfo("REJECTED TOO BIG FILE with size " + size + " Bytes for URL " + entry.url().toString()); log.logInfo("REJECTED TOO BIG FILE with size " + size + " Bytes for URL " + entry.url().toString());
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED); plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
throw new Exception("filesize too big: " + size + " bytes"); throw new Exception("file size exceeds limit");
} }
} else { } else {
// if the response has not the right file type then reject file // if the response has not the right file type then reject file
@ -275,6 +294,16 @@ public class plasmaFTPLoader {
return htCache; return htCache;
} }
/**
* gets path suitable for FTP (url-decoded, double-quotes escaped)
*
* @param entryUrl
* @return
*/
private String getPath(final yacyURL entryUrl) {
return yacyURL.unescape(entryUrl.getPath()).replace("\"", "\"\"");
}
/** /**
* @param ftpClient * @param ftpClient
* @param entry * @param entry

Write Preview
Loading…
Cancel
Save