if maxFileSize < 0 then the file size limit is without limit.

pull/1/head
orbiter 12 years ago
parent f86d469973
commit 712cc37c40

@ -565,7 +565,7 @@ public class HTTPClient {
// get the response body
final HttpEntity httpEntity = this.httpResponse.getEntity();
if (httpEntity != null) {
if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) {
if (getStatusCode() == 200 && (maxBytes < 0 || httpEntity.getContentLength() < maxBytes)) {
return getByteArray(httpEntity, maxBytes);
}
// Ensures that the entity content is fully consumed and the content stream, if exists, is closed.
@ -615,7 +615,7 @@ public class HTTPClient {
return null;
}
try {
int i = Math.min(maxBytes, (int)entity.getContentLength());
int i = maxBytes < 0 ? (int)entity.getContentLength() : Math.min(maxBytes, (int)entity.getContentLength());
if (i < 0) {
i = 4096;
}
@ -624,7 +624,7 @@ public class HTTPClient {
int l, sum = 0;
while((l = instream.read(tmp)) != -1) {
sum += l;
if (sum > maxBytes) throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes");
if (maxBytes >= 0 && sum > maxBytes) throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes");
buffer.append(tmp, 0, l);
}
return buffer.toByteArray();

@ -362,7 +362,7 @@ public final class CrawlStacker {
// check availability of parser and maxfilesize
String warning = null;
if (entry.size() > maxFileSize ||
if ((maxFileSize >= 0 && entry.size() > maxFileSize) ||
entry.url().getContentDomain() == ContentDomain.APP ||
entry.url().getContentDomain() == ContentDomain.IMAGE ||
entry.url().getContentDomain() == ContentDomain.AUDIO ||

@ -197,7 +197,7 @@ public final class HTTPLoader {
ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength);
// check length again in case it was not possible to get the length before loading
if (maxFileSize > 0 && contentLength > maxFileSize) {
if (maxFileSize >= 0 && contentLength > maxFileSize) {
this.sb.crawlQueues.errorURL.push(request, myHash, new Date(), 1, FailCategory.FINAL_PROCESS_CONTEXT, "file size limit exceeded", statusCode);
throw new IOException("REJECTED URL " + request.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + maxFileSize + " bytes. (GET)");
}

@ -787,7 +787,7 @@ public class ArrayStack implements BLOB {
else if (bi.location.length() > this.fileSizeLimit)
System.out.println("bi.location.length() > this.maxsize");
*/
if ((bi == null) || (System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit) || (bi.location.length() > this.fileSizeLimit)) {
if ((bi == null) || (System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit) || (bi.location.length() > this.fileSizeLimit && this.fileSizeLimit >= 0)) {
// add a new blob to the array
bi = new blobItem(this.buffersize);
this.blobs.add(bi);

Loading…
Cancel
Save