diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 60fe7c94a..45415f6f1 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -565,7 +565,7 @@ public class HTTPClient { // get the response body final HttpEntity httpEntity = this.httpResponse.getEntity(); if (httpEntity != null) { - if (getStatusCode() == 200 && httpEntity.getContentLength() < maxBytes) { + if (getStatusCode() == 200 && (maxBytes < 0 || httpEntity.getContentLength() < maxBytes)) { return getByteArray(httpEntity, maxBytes); } // Ensures that the entity content is fully consumed and the content stream, if exists, is closed. @@ -615,7 +615,7 @@ public class HTTPClient { return null; } try { - int i = Math.min(maxBytes, (int)entity.getContentLength()); + int i = maxBytes < 0 ? (int)entity.getContentLength() : Math.min(maxBytes, (int)entity.getContentLength()); if (i < 0) { i = 4096; } @@ -624,7 +624,7 @@ public class HTTPClient { int l, sum = 0; while((l = instream.read(tmp)) != -1) { sum += l; - if (sum > maxBytes) throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes"); + if (maxBytes >= 0 && sum > maxBytes) throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes"); buffer.append(tmp, 0, l); } return buffer.toByteArray(); diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index 87b235279..77f3e59df 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -362,7 +362,7 @@ public final class CrawlStacker { // check availability of parser and maxfilesize String warning = null; - if (entry.size() > maxFileSize || + if ((maxFileSize >= 0 && entry.size() > maxFileSize) || entry.url().getContentDomain() == ContentDomain.APP || entry.url().getContentDomain() == ContentDomain.IMAGE || entry.url().getContentDomain() == ContentDomain.AUDIO || diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java index 3e515c0a7..9c9d73ec1 100644 --- a/source/net/yacy/crawler/retrieval/HTTPLoader.java +++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java @@ -197,7 +197,7 @@ public final class HTTPLoader { ByteCount.addAccountCount(ByteCount.CRAWLER, contentLength); // check length again in case it was not possible to get the length before loading - if (maxFileSize > 0 && contentLength > maxFileSize) { + if (maxFileSize >= 0 && contentLength > maxFileSize) { this.sb.crawlQueues.errorURL.push(request, myHash, new Date(), 1, FailCategory.FINAL_PROCESS_CONTEXT, "file size limit exceeded", statusCode); throw new IOException("REJECTED URL " + request.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + maxFileSize + " bytes. (GET)"); } diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 65532d570..09aed8c74 100644 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -787,7 +787,7 @@ public class ArrayStack implements BLOB { else if (bi.location.length() > this.fileSizeLimit) System.out.println("bi.location.length() > this.maxsize"); */ - if ((bi == null) || (System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit) || (bi.location.length() > this.fileSizeLimit)) { + if ((bi == null) || (System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit) || (bi.location.length() > this.fileSizeLimit && this.fileSizeLimit >= 0)) { // add a new blob to the array bi = new blobItem(this.buffersize); this.blobs.add(bi);