boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

FTP: source cleanup (added finals, indention for easier diffs)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4559 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
danielr 17 years ago
parent fa1090113d
commit 1a7870df0d
2 changed files with 2422 additions and 2095 deletions
Show Stats Download Patch File Download Diff File

4368
source/de/anomic/net/ftpc.java
View File

File diff suppressed because it is too large Load Diff

149
source/de/anomic/plasma/crawler/plasmaFTPLoader.java
Unescape View File

@ -44,7 +44,6 @@
// the intact and unchanged copyright notice. // the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such. // Contributions and changes to the program code must be marked as such.
package de.anomic.plasma.crawler; package de.anomic.plasma.crawler;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@ -65,96 +64,85 @@ import de.anomic.server.logging.serverLog;
public class plasmaFTPLoader { public class plasmaFTPLoader {
private plasmaSwitchboard sb; private final plasmaSwitchboard sb;
private serverLog log; private final serverLog log;
public plasmaFTPLoader(plasmaSwitchboard sb, serverLog log) { public plasmaFTPLoader(final plasmaSwitchboard sb, final serverLog log) {
this.sb = sb; this.sb = sb;
this.log = log; this.log = log;
} }
protected plasmaHTCache.Entry createCacheEntry(plasmaCrawlEntry entry, String mimeType, Date fileDate) { protected plasmaHTCache.Entry createCacheEntry(final plasmaCrawlEntry entry, final String mimeType,
return plasmaHTCache.newEntry( final Date fileDate) {
new Date(), return plasmaHTCache.newEntry(new Date(), entry.depth(), entry.url(), entry.name(), "OK", new ResourceInfo(
entry.depth(), entry.url(), sb.getURL(entry.referrerhash()), mimeType, fileDate), entry.initiator(),
entry.url(), sb.profilesActiveCrawls.getEntry(entry.profileHandle()));
entry.name(),
"OK",
new ResourceInfo(
entry.url(),
sb.getURL(entry.referrerhash()),
mimeType,
fileDate
),
entry.initiator(),
sb.profilesActiveCrawls.getEntry(entry.profileHandle())
);
} }
public plasmaHTCache.Entry load(plasmaCrawlEntry entry) { public plasmaHTCache.Entry load(final plasmaCrawlEntry entry) {
ByteArrayOutputStream bout = new ByteArrayOutputStream(); final ByteArrayOutputStream bout = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bout); final PrintStream out = new PrintStream(bout);
ByteArrayOutputStream berr = new ByteArrayOutputStream(); final ByteArrayOutputStream berr = new ByteArrayOutputStream();
PrintStream err = new PrintStream(berr); final PrintStream err = new PrintStream(berr);
// create a new ftp client // create a new ftp client
ftpc ftpClient = new ftpc(System.in, out, err); final ftpc ftpClient = new ftpc(System.in, out, err);
// get username and password // get username and password
String userInfo = entry.url().getUserInfo(); final String userInfo = entry.url().getUserInfo();
String userName = "anonymous", userPwd = "anonymous"; String userName = "anonymous", userPwd = "anonymous";
if (userInfo != null) { if (userInfo != null) {
int pos = userInfo.indexOf(":"); final int pos = userInfo.indexOf(":");
if (pos != -1) { if (pos != -1) {
userName = userInfo.substring(0,pos); userName = userInfo.substring(0, pos);
userPwd = userInfo.substring(pos+1); userPwd = userInfo.substring(pos + 1);
} }
} }
// get server name, port and file path // get server name, port and file path
String host = entry.url().getHost(); final String host = entry.url().getHost();
String fullPath = entry.url().getPath(); String fullPath = entry.url().getPath();
int port = entry.url().getPort(); final int port = entry.url().getPort();
plasmaHTCache.Entry htCache = null; plasmaHTCache.Entry htCache = null;
try { try {
// open a connection to the ftp server // open a connection to the ftp server
if (port == -1) { if (port == -1) {
ftpClient.exec("open " + host, false); ftpClient.exec("open " + host, false);
} else { } else {
ftpClient.exec("open " + host + " " + port, false); ftpClient.exec("open " + host + " " + port, false);
} }
// login to the server // login to the server
ftpClient.exec("user " + userName + " " + userPwd, false); ftpClient.exec("user " + userName + " " + userPwd, false);
// change transfer mode to binary // change transfer mode to binary
ftpClient.exec("binary", false); ftpClient.exec("binary", false);
// determine filename and path // determine filename and path
String file, path; String file, path;
if (fullPath.endsWith("/")) { if (fullPath.endsWith("/")) {
file = ""; file = "";
path = fullPath; path = fullPath;
} else { } else {
int pos = fullPath.lastIndexOf("/"); final int pos = fullPath.lastIndexOf("/");
if (pos == -1) { if (pos == -1) {
file = fullPath; file = fullPath;
path = "/"; path = "/";
} else { } else {
path = fullPath.substring(0,pos+1); path = fullPath.substring(0, pos + 1);
file = fullPath.substring(pos+1); file = fullPath.substring(pos + 1);
} }
} }
// testing if the specified file is a directory // testing if the specified file is a directory
if (file.length() > 0) { if (file.length() > 0) {
ftpClient.exec("cd \"" + path + "\"", false); ftpClient.exec("cd \"" + path + "\"", false);
// testing if the current name is a directoy // testing if the current name is a directoy
boolean isFolder = ftpClient.isFolder(file); final boolean isFolder = ftpClient.isFolder(file);
if (isFolder) { if (isFolder) {
fullPath = fullPath + "/"; fullPath = fullPath + "/";
file = ""; file = "";
@ -162,11 +150,11 @@ public class plasmaFTPLoader {
} }
// creating a cache file object // creating a cache file object
File cacheFile = plasmaHTCache.getCachePath(entry.url()); final File cacheFile = plasmaHTCache.getCachePath(entry.url());
// TODO: aborting download if content is to long ... // TODO: aborting download if content is to long ...
// TODO: invalid file path check // TODO: invalid file path check
// testing if the file already exists // testing if the file already exists
if (cacheFile.isFile()) { if (cacheFile.isFile()) {
@ -179,7 +167,7 @@ public class plasmaFTPLoader {
String mimeType; String mimeType;
Date fileDate; Date fileDate;
if (file.length() == 0) { if (file.length() == 0) {
// getting the dirlist // getting the dirlist
mimeType = "text/html"; mimeType = "text/html";
fileDate = new Date(); fileDate = new Date();
@ -188,24 +176,27 @@ public class plasmaFTPLoader {
htCache = createCacheEntry(entry, mimeType, fileDate); htCache = createCacheEntry(entry, mimeType, fileDate);
// generate the dirlist // generate the dirlist
StringBuffer dirList = ftpClient.dirhtml(fullPath); final StringBuffer dirList = ftpClient.dirhtml(fullPath);
if (dirList != null && dirList.length() > 0) try { if (dirList != null && dirList.length() > 0) {
// write it into a file try {
PrintWriter writer = new PrintWriter(new FileOutputStream(cacheFile),false); // write it into a file
writer.write(dirList.toString()); final PrintWriter writer = new PrintWriter(new FileOutputStream(cacheFile), false);
writer.flush(); writer.write(dirList.toString());
writer.close(); writer.flush();
} catch (Exception e) { writer.close();
this.log.logInfo("Unable to write dirlist for URL " + entry.url().toString()); } catch (final Exception e) {
htCache = null; log.logInfo("Unable to write dirlist for URL " + entry.url().toString());
htCache = null;
}
} }
} else { } else {
// determine the mimetype of the resource // determine the mimetype of the resource
String extension = plasmaParser.getFileExt(entry.url()); final String extension = plasmaParser.getFileExt(entry.url());
mimeType = plasmaParser.getMimeTypeByFileExt(extension); mimeType = plasmaParser.getMimeTypeByFileExt(extension);
// if the mimetype and file extension is supported we start to download the file // if the mimetype and file extension is supported we start to
// download the file
if (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER, entry.url(), mimeType)) { if (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER, entry.url(), mimeType)) {
// TODO: determine the real file date // TODO: determine the real file date
@ -215,14 +206,16 @@ public class plasmaFTPLoader {
htCache = createCacheEntry(entry, mimeType, fileDate); htCache = createCacheEntry(entry, mimeType, fileDate);
// change into working directory // change into working directory
//ftpClient.exec("cd \"" + path + "\"", false); // ftpClient.exec("cd \"" + path + "\"", false);
// download the remote file // download the remote file
ftpClient.exec("get \"" + fullPath + "\" \"" + cacheFile.getAbsolutePath() + "\"", false); ftpClient.exec("get \"" + fullPath + "\" \"" + cacheFile.getAbsolutePath() + "\"", false);
} else { } else {
// if the response has not the right file type then reject file // if the response has not the right file type then reject
this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + mimeType + " for URL " + entry.url().toString()); // file
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT); log.logInfo("REJECTED WRONG MIME/EXT TYPE " + mimeType + " for URL " + entry.url().toString());
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
return null; return null;
} }
} }
@ -230,23 +223,25 @@ public class plasmaFTPLoader {
// pass the downloaded resource to the cache manager // pass the downloaded resource to the cache manager
if (berr.size() > 0 || htCache == null) { if (berr.size() > 0 || htCache == null) {
// if the response has not the right file type then reject file // if the response has not the right file type then reject file
this.log.logWarning("Unable to download URL " + entry.url().toString() + "\nErrorlog: " + berr.toString()); log.logWarning("Unable to download URL " + entry.url().toString() + "\nErrorlog: " + berr.toString());
sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR); sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
// an error has occured. cleanup // an error has occured. cleanup
if (cacheFile.exists()) cacheFile.delete(); if (cacheFile.exists()) {
cacheFile.delete();
}
} else { } else {
// announce the file // announce the file
plasmaHTCache.writeFileAnnouncement(cacheFile); plasmaHTCache.writeFileAnnouncement(cacheFile);
} }
return htCache; return htCache;
} finally { } finally {
// closing connection // closing connection
ftpClient.exec("close", false); ftpClient.exec("close", false);
ftpClient.exec("exit", false); ftpClient.exec("exit", false);
} }
} }
} }

Write Preview
Loading…
Cancel
Save