boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) better error handling for seed upload - test download - problems

Browse Source 
See: http://www.yacy-forum.de/viewtopic.php?p=26814#26814

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2812 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 19 years ago
parent a5b9b514c1
commit df49724f28
3 changed files with 139 additions and 69 deletions
Show Stats Download Patch File Download Diff File

28
source/de/anomic/http/httpc.java
Unescape View File

@ -324,6 +324,8 @@ public final class httpc {
boolean ssl,
httpRemoteProxyConfig remoteProxyConfig
) throws IOException {
if (remoteProxyConfig == null) throw new NullPointerException("Proxy object must not be null.");
return getInstance(server,vhost,port,timeout,ssl,remoteProxyConfig,null,null);
}
@ -1338,29 +1340,9 @@ do upload
if (a == null) return null;
// support of gzipped data (requested by roland)
if ((a.length > 1) && (((a[1] << 8) | a[0]) == GZIPInputStream.GZIP_MAGIC)) {
try {
ByteArrayInputStream byteInput = new ByteArrayInputStream(a);
ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
GZIPInputStream zippedContent = new GZIPInputStream(byteInput);
byte[] data = new byte[1024];
int read = 0;
// reading gzip file and store it uncompressed
while((read = zippedContent.read(data, 0, 1024)) != -1) {
byteOutput.write(data, 0, read);
}
zippedContent.close();
byteOutput.close();
a = byteOutput.toByteArray();
} catch (Exception e) {
if (!e.getMessage().equals("Not in GZIP format")) {
throw new IOException(e.getMessage());
}
}
}
a = serverFileUtils.uncompressGZipArray(a);
// return result
return a;
}

41
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -64,6 +64,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@ -283,7 +284,9 @@ public final class serverFileUtils {
: new ByteArrayOutputStream();
copy(source, baos, count);
baos.close();
return baos.toByteArray();
// convert Stream into array
return baos.toByteArray();
}
public static byte[] read(File source) throws IOException {
@ -346,6 +349,42 @@ public final class serverFileUtils {
public static void write(byte[] source, File dest) throws IOException {
copy(new ByteArrayInputStream(source), dest);
}
/**
* This function determines if a byte array is gzip compressed and uncompress it
* @param source properly gzip compressed byte array
* @return uncompressed byte array
* @throws IOException
*/
public static byte[] uncompressGZipArray(byte[] source) throws IOException {
if (source == null) return null;
// support of gzipped data (requested by roland)
if ((source.length > 1) && (((source[1] << 8) | source[0]) == GZIPInputStream.GZIP_MAGIC)) {
try {
ByteArrayInputStream byteInput = new ByteArrayInputStream(source);
ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
GZIPInputStream zippedContent = new GZIPInputStream(byteInput);
byte[] data = new byte[1024];
int read = 0;
// reading gzip file and store it uncompressed
while((read = zippedContent.read(data, 0, 1024)) != -1) {
byteOutput.write(data, 0, read);
}
zippedContent.close();
byteOutput.close();
source = byteOutput.toByteArray();
} catch (Exception e) {
if (!e.getMessage().equals("Not in GZIP format")) {
throw new IOException(e.getMessage());
}
}
}
return source;
}
public static HashSet loadList(File file) {
HashSet set = new HashSet();

139
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -69,6 +69,7 @@ import de.anomic.kelondro.kelondroRecords;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.nxTools;
@ -680,69 +681,117 @@ public final class yacySeedDB {
serverLog.logFine("YACY","SaveSeedList: Trying to upload seed-file, " + seedFile.length() + " bytes, " + uv.size() + " entries.");
log = uploader.uploadSeedFile(sb,seedDB,seedFile);
// check also if the result can be retrieved again
serverLog.logFine("YACY","SaveSeedList: Checking uploading success ...");
if (checkCache(uv, seedURL))
// test download
serverLog.logFine("YACY","SaveSeedList: Trying to download seed-file '" + seedURL + "'.");
ArrayList check = downloadSeedFile(seedURL);
// Comparing if local copy and uploaded copy are equal
String errorMsg = checkCache(uv, check);
if (errorMsg == null)
log = log + "UPLOAD CHECK - Success: the result vectors are equal" + serverCore.crlfString;
else {
throw new Exception("UPLOAD CHECK - Error: the result vector is different" + serverCore.crlfString);
throw new Exception("UPLOAD CHECK - Error: the result vector is different. " + errorMsg + serverCore.crlfString);
}
} finally {
if (seedFile != null) seedFile.delete();
if (seedFile != null) try { seedFile.delete(); } catch (Exception e) {/* ignore this */}
}
return log;
}
private ArrayList downloadSeedFile(URL seedURL) throws IOException {
httpc remote = null;
try {
// init httpc
if ((sb.remoteProxyConfig == null)||(!sb.remoteProxyConfig.useProxy())) {
remote = httpc.getInstance(
seedURL.getHost(),
seedURL.getHost(),
seedURL.getPort(),
10000,
seedURL.getProtocol().equalsIgnoreCase("https"));
} else {
remote = httpc.getInstance(
seedURL.getHost(),
seedURL.getHost(),
seedURL.getPort(),
10000,
seedURL.getProtocol().equalsIgnoreCase("https"),
sb.remoteProxyConfig);
}
// Configure http headers
httpHeader reqHeader = new httpHeader();
reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
// send request
httpc.response res = remote.GET(seedURL.getFile(), reqHeader);
// check response code
if (res.statusCode != 200) {
throw new IOException("Server returned status: " + res.status);
}
// read byte array
byte[] content = serverFileUtils.read(res.getContentInputStream());
// uncompress it if it is gzipped
content = serverFileUtils.uncompressGZipArray(content);
// convert it into an array
return nxTools.strings(content,"UTF-8");
} catch (Exception e) {
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
} finally {
if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {}
}
}
/**
* @deprecated: Function seems to be unused
*/
public String copyCache(File seedFile, URL seedURL) throws IOException {
if (seedURL == null) return "COPY - Error: URL not given";
ArrayList uv = storeCache(seedFile, true);
try {
// check also if the result can be retrieved again
if (checkCache(uv, seedURL))
return "COPY CHECK - Success: the result vectors are equal" + serverCore.crlfString;
else
return "COPY CHECK - Error: the result vector is different" + serverCore.crlfString;
} catch (IOException e) {
return "COPY CHECK - Error: IO problem " + e.getMessage() + serverCore.crlfString;
}
if (seedURL == null) return "COPY - Error: URL not given";
try {
// getting the current list
ArrayList uv = storeCache(seedFile, true);
// test download
serverLog.logFine("YACY","Trying to download seed-file '" + seedURL + "'.");
ArrayList check = downloadSeedFile(seedURL);
// Comparing if local copy and uploaded copy are equal
String errorMsg = checkCache(uv, check);
if (errorMsg == null) {
return "COPY CHECK - Success: the result vectors are equal" + serverCore.crlfString;
} else {
return "COPY CHECK - Error: the result vector is different. " + errorMsg + serverCore.crlfString;
}
} catch (IOException e) {
return "COPY CHECK - Error: IO problem " + e.getMessage() + serverCore.crlfString;
}
}
private boolean checkCache(ArrayList uv, URL seedURL) throws IOException {
// check if the result can be retrieved again
// TODO: should we check the useProxy4Yacy option here???
httpHeader reqHeader = new httpHeader();
reqHeader.put(httpHeader.PRAGMA, "no-cache");
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
ArrayList check = nxTools.strings(httpc.wget(
seedURL,
seedURL.getHost(),
10000,
null,
null,
sb.remoteProxyConfig,
reqHeader
), "UTF-8");
if (check == null) {
serverLog.logFine("YACY","SaveSeedList: Testing download failed ...");
}
private String checkCache(ArrayList uv, ArrayList check) throws IOException {
if ((check == null) || (uv == null) || (uv.size() != check.size())) {
serverLog.logFine("YACY","SaveSeedList: Local and uploades seed-list " +
"contains varying numbers of entries." +
"\n\tLocal seed-list: " + uv.size() + " entries" +
"\n\tRemote seed-list: " + check.size() + " enties");
return false;
} else {
serverLog.logFine("YACY","SaveSeedList: Comparing local and uploades seed-list entries ...");
int i;
for (i = 0; i < uv.size(); i++) {
if (!(((String) uv.get(i)).equals((String) check.get(i)))) return false;
}
if (i == uv.size()) return true;
return "Entry count is different";
}
serverLog.logFine("YACY","SaveSeedList: Comparing local and uploades seed-list entries ...");
int i;
for (i = 0; i < uv.size(); i++) {
if (!(((String) uv.get(i)).equals((String) check.get(i)))) return "Element at position " + i + " is different.";
}
return false;
// no difference found
return null;
}
public String resolveYacyAddress(String host) {

Write Preview
Loading…
Cancel
Save