- 
+
#[title]#
#(heuristic)#::
diff --git a/lib/imageio-bmp-3.2.jar b/lib/imageio-bmp-3.2.jar
new file mode 100755
index 000000000..1946caefe
Binary files /dev/null and b/lib/imageio-bmp-3.2.jar differ
diff --git a/pom.xml b/pom.xml
index baa8f332e..13eed6181 100644
--- a/pom.xml
+++ b/pom.xml
@@ -430,6 +430,11 @@
icu4j
56.1
+
+ com.twelvemonkeys.imageio
+ imageio-bmp
+ 3.2
+
com.twelvemonkeys.imageio
imageio-tiff
diff --git a/readme.mediawiki b/readme.mediawiki
index c3f8a887d..7848a276f 100644
--- a/readme.mediawiki
+++ b/readme.mediawiki
@@ -55,7 +55,7 @@ all these locations into one search result.
You need java 1.7 or later to run YaCy, nothing else.
Please download it from http://www.java.com
-YaCy also runs on IcedTea6.
+YaCy also runs on IcedTea7.
See http://icedtea.classpath.org
NO OTHER SOFTWARE IS REQUIRED!
diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java
index 7a9161149..f2f757335 100644
--- a/source/net/yacy/cora/protocol/http/HTTPClient.java
+++ b/source/net/yacy/cora/protocol/http/HTTPClient.java
@@ -712,31 +712,62 @@ public class HTTPClient {
}
}
- private static byte[] getByteArray(final HttpEntity entity, final int maxBytes) throws IOException {
- final InputStream instream = entity.getContent();
- if (instream == null) {
- return null;
- }
- try {
- int i = maxBytes < 0 ? (int)entity.getContentLength() : Math.min(maxBytes, (int)entity.getContentLength());
- if (i < 0) {
- i = 4096;
- }
- final ByteArrayBuffer buffer = new ByteArrayBuffer(i);
- byte[] tmp = new byte[4096];
- int l, sum = 0;
- while((l = instream.read(tmp)) != -1) {
- sum += l;
- if (maxBytes >= 0 && sum > maxBytes) throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes");
- buffer.append(tmp, 0, l);
- }
- return buffer.toByteArray();
- } catch (final OutOfMemoryError e) {
- throw new IOException(e.toString());
- } finally {
- instream.close();
- }
- }
+ /**
+ * Return entity content loaded as a byte array
+ * @param entity HTTP entity
+ * @param maxBytes maximum bytes to read. -1 means no maximum limit.
+ * @return content bytes or null when entity content is null.
+ * @throws IOException when a read error occured or content length is over maxBytes
+ */
+ public static byte[] getByteArray(final HttpEntity entity, int maxBytes) throws IOException {
+ final InputStream instream = entity.getContent();
+ if (instream == null) {
+ return null;
+ }
+ try {
+ long contentLength = entity.getContentLength();
+ /*
+ * When no maxBytes is specified, the default limit is
+ * Integer.MAX_VALUE as a byte array size can not be over
+ */
+ if (maxBytes < 0) {
+ maxBytes = Integer.MAX_VALUE;
+ }
+ /*
+ * Content length may already be known now : check it before
+ * downloading
+ */
+ if (contentLength > maxBytes) {
+ throw new IOException("Content to download exceed maximum value of " + maxBytes + " bytes");
+ }
+ int initialSize = Math.min(maxBytes, (int) contentLength);
+ /* ContentLenght may be negative because unknown for now */
+ if (initialSize < 0) {
+ initialSize = 4096;
+ }
+ final ByteArrayBuffer buffer = new ByteArrayBuffer(initialSize);
+ byte[] tmp = new byte[4096];
+ int l = 0;
+ /* Sum is a long to enable check against Integer.MAX_VALUE */
+ long sum = 0;
+ while ((l = instream.read(tmp)) != -1) {
+ sum += l;
+ /*
+ * Check total length while downloading as content lenght might
+ * not be known at beginning
+ */
+ if (sum > maxBytes) {
+ throw new IOException("Download exceeded maximum value of " + maxBytes + " bytes");
+ }
+ buffer.append(tmp, 0, l);
+ }
+ return buffer.toByteArray();
+ } catch (final OutOfMemoryError e) {
+ throw new IOException(e.toString());
+ } finally {
+ instream.close();
+ }
+ }
private void setHeaders(final HttpUriRequest httpUriRequest) {
if (this.headers != null) {
diff --git a/source/net/yacy/cora/util/HTTPInputStream.java b/source/net/yacy/cora/util/HTTPInputStream.java
new file mode 100755
index 000000000..035edfad9
--- /dev/null
+++ b/source/net/yacy/cora/util/HTTPInputStream.java
@@ -0,0 +1,125 @@
+/**
+ * HTTPInputStream
+ * Copyright 2014 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+ * First published 26.11.2014 on http://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see failures,
+ File inFile) throws IOException {
+ /* Delete eventual previous result file */
+ System.out
+ .println("Measuring ViewImage render with file : " + inFile.getAbsolutePath() + " encoded To : " + ext);
+ File outFile = new File(outDir, inFile.getName() + "." + ext);
+ if (outFile.exists()) {
+ outFile.delete();
}
- post.put("maxwidth", String.valueOf(maxWidth));
- int maxHeight = DEFAULT_MAX_HEIGHT;
- if (args != null && args.length > 2) {
- maxHeight = Integer.parseInt(args[2]);
+ String urlString = inFile.getAbsolutePath();
+ EncodedImage img = null;
+ Exception error = null;
+ long beginTime = System.nanoTime(), time, minTime = Long.MAX_VALUE, maxTime = 0, meanTime = 0, totalTime = 0;
+ int step = 0;
+ for (step = 0; (totalTime / 1000000000) < this.minMeasureTime; step++) {
+ beginTime = System.nanoTime();
+ ImageInputStream inStream = ImageIO.createImageInputStream(inFile);
+ try {
+ img = ViewImage.parseAndScale(post, true, urlString, ext, inStream);
+ } catch (Exception e) {
+ error = e;
+ }
+ time = System.nanoTime() - beginTime;
+ minTime = Math.min(minTime, time);
+ maxTime = Math.max(maxTime, time);
+ totalTime += time;
+ }
+ if (step > 0) {
+ meanTime = totalTime / step;
+ } else {
+ meanTime = totalTime;
+ }
+ PrintWriter resultsWriter = new PrintWriter(new FileWriter(new File(outDir, "results_perfs.txt"), true));
+ try {
+ writeMessage("Measured ViewImage render with file : " + inFile.getAbsolutePath() + " encoded To : " + ext,
+ resultsWriter);
+ if(img == null) {
+ writeMessage("Image could not be rendered! Measurement show time needed to read and parse image data until error detection.", resultsWriter);
+ }
+ writeMessage("Render total time (ms) : " + (totalTime) / 1000000 + " on " + step + " steps.",
+ resultsWriter);
+ writeMessage("Render mean time (ms) : " + (meanTime) / 1000000, resultsWriter);
+ writeMessage("Render min time (ms) : " + (minTime) / 1000000, resultsWriter);
+ writeMessage("Render max time (ms) : " + (maxTime) / 1000000, resultsWriter);
+ } finally {
+ resultsWriter.close();
}
- post.put("maxheight", String.valueOf(maxHeight));
- /* Make it square by default */
- post.put("quadratic", "");
- return post;
- }
- /**
- *
- * @param args
- * main parameters : fourth item may contain extension
- * @return extension to use for encoding
- */
- private static String getEncodingExt(String args[]) {
- String ext = DEFAUL_EXT;
- if (args != null && args.length > 3) {
- ext = args[3];
+ if (img == null) {
+ failures.put(urlString, error);
+ } else {
+ FileOutputStream outFileStream = null;
+ try {
+ outFileStream = new FileOutputStream(outFile);
+ img.getImage().writeTo(outFileStream);
+ } finally {
+ if (outFileStream != null) {
+ outFileStream.close();
+ }
+ img.getImage().close();
+ }
}
- return ext;
}
/**
- * Test image is parsed and rendered again and again until 20 seconds
- * elapsed. Then measured statistics are displayed.
+ * Test image(s) (default : classpath resource folder /viewImageTest/test/)
+ * are parsed and rendered again and again until specified time (default :
+ * 10 seconds) elapsed. Then rendered image is written to outDir for visual
+ * check and measured statistics are displayed.
*
* @param args
* may be empty or contain parameters to override defaults :
* processedFiles = new ArrayList();
+ Map failures = new TreeMap<>();
try {
- /* Max test total time (s) */
- int maxTotalTime = 20;
- long beginTime, time, minTime = Long.MAX_VALUE, maxTime = 0, meanTime = 0, totalTime = 0;
- int step = 0;
- for (step = 0; (totalTime / 1000000000) < maxTotalTime; step++) {
- beginTime = System.nanoTime();
- EncodedImage img = ViewImage.parseAndScale(post, true, urlString, ext, false, resourceb);
- time = System.nanoTime() - beginTime;
- minTime = Math.min(minTime, time);
- maxTime = Math.max(maxTime, time);
- totalTime += time;
- if (img == null) {
- throw new IOException("Image render failed");
- }
- }
- meanTime = totalTime / step;
- System.out.println("Render total time (ms) : " + (totalTime) / 1000000 + " on " + step + " steps.");
- System.out.println("Render mean time (ms) : " + (meanTime) / 1000000);
- System.out.println("Render min time (ms) : " + (minTime) / 1000000);
- System.out.println("Render max time (ms) : " + (maxTime) / 1000000);
+ long time = System.nanoTime();
+ test.processFiles(ext, recursive, outDir, post, inFiles, processedFiles, failures);
+ time = System.nanoTime() - time;
+ test.displayResults(processedFiles, failures, time, outDir);
} finally {
ConcurrentLog.shutdown();
}
diff --git a/test/ViewImageTest.java b/test/ViewImageTest.java
index 84319969a..a80a7d588 100755
--- a/test/ViewImageTest.java
+++ b/test/ViewImageTest.java
@@ -1,15 +1,17 @@
import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
-import java.io.InputStream;
+import java.io.PrintWriter;
import java.net.URL;
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
-import org.apache.commons.io.filefilter.FileFileFilter;
+import javax.imageio.ImageIO;
+import javax.imageio.stream.ImageInputStream;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.peers.graphics.EncodedImage;
@@ -51,24 +53,6 @@ public class ViewImageTest {
/** Default output encoding format */
private static final String DEFAULT_OUT_EXT = "png";
- /**
- * @param testFile
- * file to load
- * @return testFile content as a bytes array
- * @throws IOException
- * when an error occured while loading
- */
- private static byte[] getBytes(File testFile) throws IOException {
- InputStream inStream = new FileInputStream(testFile);
- byte[] res = new byte[inStream.available()];
- try {
- inStream.read(res);
- } finally {
- inStream.close();
- }
- return res;
- }
-
/**
* @param args
* main parameters. first item may contain input file or folder
@@ -76,7 +60,7 @@ public class ViewImageTest {
* @return file or folder to be used : specified as first in args or default
* one
*/
- private static File getInputURL(String args[]) {
+ protected File getInputURL(String args[]) {
String fileURL;
if (args != null && args.length > 0) {
fileURL = args[0];
@@ -98,7 +82,7 @@ public class ViewImageTest {
* @throws IllegalArgumentException
* when args[2] is not set and default is not found
*/
- private static File getOuputDir(String[] args) {
+ protected File getOuputDir(String[] args) {
File outDir;
if (args.length > 2) {
outDir = new File(args[2]);
@@ -107,7 +91,7 @@ public class ViewImageTest {
if (tmpDir == null) {
throw new IllegalArgumentException("No destination dir specified, and default not found");
}
- outDir = new File(tmpDir + File.separator + ViewImageTest.class.getCanonicalName());
+ outDir = new File(tmpDir + File.separator + this.getClass().getCanonicalName());
}
return outDir;
}
@@ -117,10 +101,12 @@ public class ViewImageTest {
*
* @param args
* main parameters : args[3] and args[4] may respectively contain
- * max width and max height
+ * max width and max height. Set it to zero so there is no max
+ * width and no max height when processing. args[5] may be set to
+ * "quadratic" to render output images as squares.
* @return a serverObjects instance
*/
- private static serverObjects makePostParams(String args[]) {
+ protected serverObjects makePostParams(String args[]) {
serverObjects post = new serverObjects();
if (args != null && args.length > 3) {
int maxWidth = Integer.parseInt(args[3]);
@@ -132,51 +118,198 @@ public class ViewImageTest {
post.put("maxheight", String.valueOf(maxHeight));
}
+ boolean quadratic = isQuadratic(args);
+ if (quadratic) {
+ post.put("quadratic", "");
+ }
+
return post;
}
/**
*
* @param args
- * main parameters : fourth item may contain extension
+ * main parameters : second item may contain extension
* @return extension to use for encoding
*/
- private static String getEncodingExt(String args[]) {
+ protected String getEncodingExt(String args[]) {
String ext = DEFAULT_OUT_EXT;
- if (args != null && args.length > 3) {
- ext = args[3];
+ if (args != null && args.length > 1) {
+ ext = args[1];
}
return ext;
}
/**
- * Display detailed results. All parametrers required not to be null.
*
- * @param inFiles
- * input image files
+ * @param args
+ * main parameters. args[5] may be set to "quadratic"
+ * @return true when image are supposed to be rendered as squares.
+ */
+ protected boolean isQuadratic(String args[]) {
+ boolean recursive = false;
+ if (args != null && args.length > 5) {
+ recursive = "quadratic".equals(args[5]);
+ }
+ return recursive;
+ }
+
+ /**
+ *
+ * @param args
+ * main parameters. args[6] may be set to "recursive"
+ * @return true when folders are supposed to processed recursively
+ */
+ protected boolean isRecursive(String args[]) {
+ boolean recursive = false;
+ if (args != null && args.length > 6) {
+ recursive = "recursive".equals(args[6]);
+ }
+ return recursive;
+ }
+
+ /**
+ * Write same message to both system standard output and to outWriter.
+ *
+ * @param message
+ * message to write
+ * @param outWriter
+ * PrintWriter writer. Must not be null.
+ * @throws IOException
+ * in case of write error
+ */
+ protected void writeMessage(String message, PrintWriter outWriter) throws IOException {
+ System.out.println(message);
+ outWriter.println(message);
+ }
+
+ /**
+ * Display detailed results and produce a results.txt file in outDir. All
+ * parametrers required not to be null.
+ *
+ * @param processedFiles
+ * all processed image files
* @param failures
- * map input file url which failed with eventual cause exception
+ * map input file url which failed with eventual cause error
+ * @param time
+ * total processing time in nanoseconds
+ * @param outDir
+ * directory to write results file
+ * @throws IOException
+ * when a write error occured writing the results file
*/
- private static void displayResults(File[] inFiles, Map failures) {
- if (failures.size() > 0) {
- if (failures.size() == inFiles.length) {
- System.out.println("No input files could be processed :");
+ protected void displayResults(List processedFiles, Map failures, long time, File outDir)
+ throws IOException {
+ PrintWriter resultsWriter = new PrintWriter(new FileWriter(new File(outDir, "results.txt")));
+ try {
+ writeMessage(processedFiles.size() + " files processed in " + (time / 1000000) + " ms", resultsWriter);
+ if (failures.size() > 0) {
+ if (failures.size() == processedFiles.size()) {
+ writeMessage("No input files could be processed :", resultsWriter);
+ } else {
+ writeMessage("Some input files could not be processed :", resultsWriter);
+ }
+ for (Entry entry : failures.entrySet()) {
+ writeMessage(entry.getKey(), resultsWriter);
+ if (entry.getValue() != null) {
+ writeMessage("cause : " + entry.getValue(), resultsWriter);
+ }
+ }
} else {
- System.out.println("Some input files could not be processed :");
+ if (processedFiles.size() > 0) {
+ writeMessage("All input files were successfully processed.", resultsWriter);
+ } else {
+ writeMessage("No input file was provided.", resultsWriter);
+ }
}
- for (Entry entry : failures.entrySet()) {
- System.out.println(entry.getKey());
- if (entry.getValue() != null) {
- System.out.println("cause : " + entry.getValue());
+ } finally {
+ resultsWriter.close();
+ }
+ }
+
+ /**
+ * Process inFiles and update processedFiles list and failures map. All
+ * parameters must not be null.
+ *
+ * @param ext
+ * output encoding image format
+ * @param recursive
+ * when true, also process inFiles directories
+ * @param outDir
+ * output directory
+ * @param post
+ * ViewImage post parameters
+ * @param inFiles
+ * files or directories to process
+ * @param processedFiles
+ * list of processed files
+ * @param failures
+ * map failed file urls to eventual exception
+ * @throws IOException
+ * when an read/write error occured
+ */
+ protected void processFiles(String ext, boolean recursive, File outDir, serverObjects post, File[] inFiles,
+ List processedFiles, Map failures) throws IOException {
+ for (File inFile : inFiles) {
+ if (inFile.isDirectory()) {
+ if (recursive) {
+ File subDir = new File(outDir, inFile.getName());
+ subDir.mkdirs();
+ processFiles(ext, recursive, subDir, post, inFile.listFiles(), processedFiles, failures);
}
+ } else {
+ processedFiles.add(inFile);
+ processFile(ext, outDir, post, failures, inFile);
}
+ }
+ }
+
+ /**
+ * Process inFile image and update processedFiles list and failures map. All
+ * parameters must not be null.
+ * @param ext output encoding image format
+ * @param outDir output directory
+ * @param post ViewImage post parameters
+ * @param failures map failed file urls to eventual exception
+ * @param inFile file image to process
+ * @throws IOException when an read/write error occured
+ */
+ protected void processFile(String ext, File outDir, serverObjects post, Map failures, File inFile)
+ throws IOException {
+ /* Delete eventual previous result file */
+ File outFile = new File(outDir, inFile.getName() + "." + ext);
+ if (outFile.exists()) {
+ outFile.delete();
+ }
+
+ ImageInputStream inStream = ImageIO.createImageInputStream(inFile);
+ String urlString = inFile.getAbsolutePath();
+ EncodedImage img = null;
+ Throwable error = null;
+ try {
+ img = ViewImage.parseAndScale(post, true, urlString, ext, inStream);
+ } catch (Throwable e) {
+ error = e;
+ }
+
+ if (img == null) {
+ failures.put(urlString, error);
} else {
- System.out.println("All input files were successfully processed.");
+ FileOutputStream outFileStream = null;
+ try {
+ outFileStream = new FileOutputStream(outFile);
+ img.getImage().writeTo(outFileStream);
+ } finally {
+ if (outFileStream != null) {
+ outFileStream.close();
+ }
+ img.getImage().close();
+ }
}
}
/**
- * Test image(s) (default : JPEG_example_JPG_RIP_100.jpg) are parsed and
+ * Test image(s) (default : classpath resource folder /viewImageTest/test/) are parsed and
* rendered to an output foler. Result can then be checked with program of
* your choice.
*
@@ -184,80 +317,61 @@ public class ViewImageTest {
* may be empty or contain parameters to override defaults :
* failures = new HashMap();
+ List processedFiles = new ArrayList();
+ Map failures = new TreeMap<>();
try {
- for (File inFile : inFiles) {
- /* Delete eventual previous result file */
- File outFile = new File(outDir, inFile.getName() + "." + ext);
- if (outFile.exists()) {
- outFile.delete();
- }
-
- byte[] resourceb = getBytes(inFile);
- String urlString = inFile.getAbsolutePath();
- EncodedImage img = null;
- Exception error = null;
- try {
- img = ViewImage.parseAndScale(post, true, urlString, ext, false, resourceb);
- } catch (Exception e) {
- error = e;
- }
-
- if (img == null) {
- failures.put(urlString, error);
- } else {
- FileOutputStream outFileStream = null;
- try {
- outFileStream = new FileOutputStream(outFile);
- img.getImage().writeTo(outFileStream);
- } finally {
- if (outFileStream != null) {
- outFileStream.close();
- }
- img.getImage().close();
- }
- }
- }
- displayResults(inFiles, failures);
+ long time = System.nanoTime();
+ test.processFiles(ext, recursive, outDir, post, inFiles, processedFiles, failures);
+ time = System.nanoTime() - time;
+ test.displayResults(processedFiles, failures, time, outDir);
} finally {
ConcurrentLog.shutdown();
}
diff --git a/test/viewImageTest/ViewImageTest.html b/test/viewImageTest/ViewImageTest.html
index 547152cc7..efdfd96da 100644
--- a/test/viewImageTest/ViewImageTest.html
+++ b/test/viewImageTest/ViewImageTest.html
@@ -70,7 +70,8 @@ td {
.png
Portable Network Graphics
RFC 2083
- Pngsuite
+ Pngsuite,
+ imagetestsuite
APNG
Specification
- APNG tests
+ APNG
+ tests
Joint Photographic Experts Group
imagetestsuite
Graphics Interchange Format
imagetestsuite
Windows Bitmap
bmpsuite,
+ The Bitmap Test
+ suite
libtiff,
TwelveMonkeys
+ href="https://github.com/haraldk/TwelveMonkeys/tree/master/imageio/imageio-tiff/src/test/resources/tiff">TwelveMonkeys,
+ imagetestsuite
-
- *
- args[0] : input image file URL. Default : - * viewImageTest/test/JPEG_example_JPG_RIP_100.jpg - *
- args[1] : max width (in pixels) for rendered image. - * Default : default image width divided by 10. - *
- args[2] : max height (in pixels) for rendered image. - * Default : default image height divided by 10. - *
- args[3] : output format name. Default : "png". + *
- args[0] : input image file URL or folder containing image + * files URL. Default : classpath resource /viewImageTest/test/ + * + *
- args[1] : output format name (for example : "jpg") for + * rendered image. Defaut : "png". + *
- args[2] : ouput folder URL. Default : + * "[system tmp dir]/ViewImageTest". + *
- args[3] : max width (in pixels) for rendered image. May be + * set to zero to specify no max width. Default : no value. + *
- args[4] : max height (in pixels) for rendered image. May + * be set to zero to specify no max height. Default : no value. + * + *
- args[5] : set to "quadratic" to render square output + * image. May be set to any string to specify no quadratic shape. + * Default : false. + *
- args[6] : set to "recursive" to process recursively sub + * folders. Default : false. + *
- args[7] : minimum measurement time in secondes. Default : + * 10. *
-
*
- args[0] : input image file URL or folder containing image - * files URL. Default : - * viewImageTest/test/JPEG_example_JPG_RIP_100.jpg + * files URL. Default : classpath resource + * /viewImageTest/test/ *
- args[1] : output format name (for example : "jpg") for - * rendered image - *
- args[2] : ouput folder URL - *
- args[3] : max width (in pixels) for rendered image. - * Default : no value. - *
- args[4] : max height (in pixels) for rendered image. - * Default : no value. + * rendered image. Defaut : "png". + *
- args[2] : ouput folder URL. Default : + * "[system tmp dir]/ViewImageTest". + *
- args[3] : max width (in pixels) for rendered image. May be + * set to zero to specify no max width. Default : no value. + *
- args[4] : max height (in pixels) for rendered image. May + * be set to zero to specify no max height. Default : no value. + * + *
- args[5] : set to "quadratic" to render square output + * image. May be set to any string to specify no quadratic shape. + * Default : false. + *
- args[6] : set to "recursive" to process recursively sub + * folders. Default : false. *