diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.java
index baf8d76be..8e437a2fb 100644
--- a/source/net/yacy/cora/util/Html2Image.java
+++ b/source/net/yacy/cora/util/Html2Image.java
@@ -43,6 +43,10 @@ import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.ImageView;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.rendering.ImageType;
+import org.apache.pdfbox.rendering.PDFRenderer;
+
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
@@ -51,57 +55,53 @@ import net.yacy.document.ImageParser;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.OS;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.rendering.ImageType;
-import org.apache.pdfbox.rendering.PDFRenderer;
-
/**
* Convert html to an copy on disk-image in a other file format
* currently (pdf and/or jpg)
*/
public class Html2Image {
-
+
// Mac
- /**
- * Path to wkhtmltopdf executable on Mac OS when installed using
- * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
- * This can also be a path on Debian or another Gnu/Linux distribution.
- */
- private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
-
+ /**
+ * Path to wkhtmltopdf executable on Mac OS when installed using
+ * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
+ * This can also be a path on Debian or another Gnu/Linux distribution.
+ */
+ private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
+
// to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip
// the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/
-
+
private final static File convertMac1 = new File("/opt/local/bin/convert");
private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert");
-
+
/* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript
The imagemagick policy at /etc should also be checked :
if it contains a line such as it must be edited with rights="read" at minimum
- */
+ */
private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images
private final static File convertDebian = new File("/usr/bin/convert");
-
- /**
- * Path to wkhtmltopdf executable on Windows, when installed with default
- * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
- * https://wkhtmltopdf.org/downloads.html
- */
- private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
-
- /**
- * Path to wkhtmltopdf executable on Windows, when installed with default
- * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
- * https://wkhtmltopdf.org/downloads.html
- */
- private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
- "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
-
- /** Command to use when wkhtmltopdf is included in the system Path */
- private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
-
- /** Command to use when imagemagick convert is included in the system Path */
- private static final String CONVERT_COMMAND = "convert";
+
+ /**
+ * Path to wkhtmltopdf executable on Windows, when installed with default
+ * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
+ * https://wkhtmltopdf.org/downloads.html
+ */
+ private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
+
+ /**
+ * Path to wkhtmltopdf executable on Windows, when installed with default
+ * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
+ * https://wkhtmltopdf.org/downloads.html
+ */
+ private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
+ "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
+
+ /** Command to use when wkhtmltopdf is included in the system Path */
+ private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
+
+ /** Command to use when imagemagick convert is included in the system Path */
+ private static final String CONVERT_COMMAND = "convert";
private static boolean usexvfb = false;
@@ -109,109 +109,111 @@ public class Html2Image {
* @return when the wkhtmltopdf command is detected as available in the system
*/
public static boolean wkhtmltopdfAvailable() {
- /* Check wkhtmltopdf common installation paths and system Path */
- return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
+ /* Check wkhtmltopdf common installation paths and system Path */
+ return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
}
-
+
/**
* @return a wkhtmltopdf executable file when one can be found, null otherwise
*/
private static File wkhtmltopdfExecutable() {
- File executable = null;
- if(OS.isWindows) {
- if(WKHTMLTOPDF_WINDOWS.exists()) {
- executable = WKHTMLTOPDF_WINDOWS;
- } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
- executable = WKHTMLTOPDF_WINDOWS_X86;
- }
- } else {
- if(wkhtmltopdfMac.exists()) {
- executable = wkhtmltopdfMac;
- } else if(wkhtmltopdfDebian.exists()) {
- executable = wkhtmltopdfDebian;
- }
- }
- return executable;
+ File executable = null;
+ if(OS.isWindows) {
+ if(WKHTMLTOPDF_WINDOWS.exists()) {
+ executable = WKHTMLTOPDF_WINDOWS;
+ } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
+ executable = WKHTMLTOPDF_WINDOWS_X86;
+ }
+ } else {
+ if(wkhtmltopdfMac.exists()) {
+ executable = wkhtmltopdfMac;
+ } else if(wkhtmltopdfDebian.exists()) {
+ executable = wkhtmltopdfDebian;
+ }
+ }
+ return executable;
}
/**
* @return true when wkhtmltopdf is available in system path
*/
- private static boolean wkhtmltopdfAvailableInPath() {
- boolean available = false;
- try {
- final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
- available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
- } catch (final IOException e) {
- ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt(); // preserve thread interrupted state
- }
- return available;
- }
-
+ private static boolean wkhtmltopdfAvailableInPath() {
+ boolean available = false;
+ try {
+ @SuppressWarnings("deprecation")
+ final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
+ available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
+ } catch (final IOException e) {
+ ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt(); // preserve thread interrupted state
+ }
+ return available;
+ }
+
/**
* @return a imagemagick convert executable file when one can be found, null otherwise
*/
private static File convertExecutable() {
- File executable = null;
- if(!OS.isWindows) {
- if(convertMac1.exists()) {
- executable = convertMac1;
- } else if(convertMac2.exists()) {
- executable = convertMac2;
- } else if(convertDebian.exists()) {
- executable = convertDebian;
- }
- }
- return executable;
+ File executable = null;
+ if(!OS.isWindows) {
+ if(convertMac1.exists()) {
+ executable = convertMac1;
+ } else if(convertMac2.exists()) {
+ executable = convertMac2;
+ } else if(convertDebian.exists()) {
+ executable = convertDebian;
+ }
+ }
+ return executable;
}
-
+
/**
* @return when the imagemagick convert command is detected as available in the system
*/
public static boolean convertAvailable() {
- /* Check convert common installation paths and system Path */
+ /* Check convert common installation paths and system Path */
return convertExecutable() != null || convertAvailableInPath();
}
/**
* @return when imagemagick convert is available in system path
*/
- private static boolean convertAvailableInPath() {
- boolean available = false;
- if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
- try {
- final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
- available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
- } catch (final IOException e) {
- ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt(); // preserve thread interrupted state
- }
- }
- return available;
- }
-
- /**
- * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
- * wKhtmltopdf may be called multiple times with various parameters flavors in
- * case of failure.
- *
- * @param url the URL of a web resource to fetch, render and convert to
- * a pdf file. Must not be null.
- * @param proxy the eventual proxy address to use. Can be null. Must be of
- * the form http://host:port; use YaCy here as proxy which is
- * mostly http://localhost:8090
- * @param destination the destination PDF file that should be written. Must not
- * be null.
- * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf
- * call termination. Beyond this limit the process is killed.
- * @return true when the destination file was successfully written
- */
+ private static boolean convertAvailableInPath() {
+ boolean available = false;
+ if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
+ try {
+ @SuppressWarnings("deprecation")
+ final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
+ available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
+ } catch (final IOException e) {
+ ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt(); // preserve thread interrupted state
+ }
+ }
+ return available;
+ }
+
+ /**
+ * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
+ * wKhtmltopdf may be called multiple times with various parameters flavors in
+ * case of failure.
+ *
+ * @param url the URL of a web resource to fetch, render and convert to
+ * a pdf file. Must not be null.
+ * @param proxy the eventual proxy address to use. Can be null. Must be of
+ * the form http://host:port; use YaCy here as proxy which is
+ * mostly http://localhost:8090
+ * @param destination the destination PDF file that should be written. Must not
+ * be null.
+ * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf
+ * call termination. Beyond this limit the process is killed.
+ * @return true when the destination file was successfully written
+ */
public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {
boolean success = false;
- for (boolean ignoreErrors: new boolean[]{false, true}) {
+ for (final boolean ignoreErrors: new boolean[]{false, true}) {
success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
if (success) break;
if (!success && proxy != null) {
@@ -227,49 +229,49 @@ public class Html2Image {
}
return success;
}
-
- /**
- * Run wkhtmltopdf in a separate process to fetch and render to PDF a web
- * resource.
- *
- * @param url the URL of a web resource to fetch, render and convert to
- * a pdf file. Must not be null.
- * @param proxy the eventual proxy address to use. Can be null.
- * @param destination the destination PDF file that should be written. Must not
- * be null.
- * @param userAgent TODO: implement
- * @param acceptLanguage TODO: implement
- * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
- * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf
- * dedicated process termination. Beyond this limit the
- * process is killed.
- * @return true when the destination file was successfully written
- */
- private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
- final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
- final String wkhtmltopdfCmd;
- final File wkhtmltopdf = wkhtmltopdfExecutable();
- if(wkhtmltopdf != null) {
- wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
- } else if(wkhtmltopdfAvailableInPath()) {
- wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
- } else {
- ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
- return false;
- }
+
+ /**
+ * Run wkhtmltopdf in a separate process to fetch and render to PDF a web
+ * resource.
+ *
+ * @param url the URL of a web resource to fetch, render and convert to
+ * a pdf file. Must not be null.
+ * @param proxy the eventual proxy address to use. Can be null.
+ * @param destination the destination PDF file that should be written. Must not
+ * be null.
+ * @param userAgent TODO: implement
+ * @param acceptLanguage TODO: implement
+ * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
+ * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf
+ * dedicated process termination. Beyond this limit the
+ * process is killed.
+ * @return true when the destination file was successfully written
+ */
+ private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
+ final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
+ final String wkhtmltopdfCmd;
+ final File wkhtmltopdf = wkhtmltopdfExecutable();
+ if(wkhtmltopdf != null) {
+ wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
+ } else if(wkhtmltopdfAvailableInPath()) {
+ wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
+ } else {
+ ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
+ return false;
+ }
String commandline =
- wkhtmltopdfCmd + " -q --title '" + url + "' " +
- //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
- //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
- (proxy == null ? "" : "--proxy " + proxy + " ") +
- (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
- //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
- "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
- url + " " + destination.getAbsolutePath();
+ wkhtmltopdfCmd + " -q --title '" + url + "' " +
+ //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
+ //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
+ (proxy == null ? "" : "--proxy " + proxy + " ") +
+ (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
+ //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
+ "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
+ url + " " + destination.getAbsolutePath();
try {
- ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
+ ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) {
- return true;
+ return true;
}
// if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers
commandline = "xvfb-run -a " + commandline;
@@ -280,46 +282,46 @@ public class Html2Image {
}
}
- /**
- * Run a wkhtmltopdf commandline in a separate process.
- *
- * @param proxy the eventual proxy address to use. Can be null.
- * @param destination the destination PDF file that should be written. Must not
- * be null.
- * @param commandline the wkhtmltopdf command line to execute. Must not be null.
- * @param maxSeconds the maximum time in seconds to wait for the process
- * termination. Beyond this limit the process is killed.
- * @return true when the destination file was successfully written
- * @throws IOException when an unexpected error occurred
- */
- private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds)
- throws IOException {
- final Process p = Runtime.getRuntime().exec(commandline);
-
- try {
- p.waitFor(maxSeconds, TimeUnit.SECONDS);
- } catch (final InterruptedException e) {
- p.destroyForcibly();
- ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
- Thread.currentThread().interrupt(); // Keep the thread interrupted state
- return false;
- }
- if(p.isAlive()) {
- ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
- p.destroyForcibly();
- return false;
- }
- if (p.exitValue() == 0 && destination.exists()) {
- return true;
- }
- final List messages = OS.readStreams(p);
- ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
- for (final String message : messages) {
- ConcurrentLog.warn("Html2Image", ">> " + message);
- }
- return false;
- }
-
+ /**
+ * Run a wkhtmltopdf commandline in a separate process.
+ *
+ * @param proxy the eventual proxy address to use. Can be null.
+ * @param destination the destination PDF file that should be written. Must not
+ * be null.
+ * @param commandline the wkhtmltopdf command line to execute. Must not be null.
+ * @param maxSeconds the maximum time in seconds to wait for the process
+ * termination. Beyond this limit the process is killed.
+ * @return true when the destination file was successfully written
+ * @throws IOException when an unexpected error occurred
+ */
+ private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException {
+ @SuppressWarnings("deprecation")
+ final Process p = Runtime.getRuntime().exec(commandline);
+
+ try {
+ p.waitFor(maxSeconds, TimeUnit.SECONDS);
+ } catch (final InterruptedException e) {
+ p.destroyForcibly();
+ ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
+ Thread.currentThread().interrupt(); // Keep the thread interrupted state
+ return false;
+ }
+ if(p.isAlive()) {
+ ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
+ p.destroyForcibly();
+ return false;
+ }
+ if (p.exitValue() == 0 && destination.exists()) {
+ return true;
+ }
+ final List messages = OS.readStreams(p);
+ ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
+ for (final String message : messages) {
+ ConcurrentLog.warn("Html2Image", ">> " + message);
+ }
+ return false;
+ }
+
/**
* Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75
* using internal pdf library or external command line tool on linux or mac
@@ -332,34 +334,34 @@ public class Html2Image {
* @return true when the ouput image file was successfully written.
*/
public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) {
- /* Deduce the ouput image format from the file extension */
- String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
- if(imageFormat.isEmpty()) {
- /* Use JPEG as a default fallback */
- imageFormat = "jpg";
- }
- String convertCmd = null;
- final File convert = convertExecutable();
- if(convert != null) {
- convertCmd = convert.getAbsolutePath();
- } else if(convertAvailableInPath()) {
- convertCmd = CONVERT_COMMAND;
- } else {
- ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
- }
+ /* Deduce the ouput image format from the file extension */
+ String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
+ if(imageFormat.isEmpty()) {
+ /* Use JPEG as a default fallback */
+ imageFormat = "jpg";
+ }
+ String convertCmd = null;
+ final File convert = convertExecutable();
+ if(convert != null) {
+ convertCmd = convert.getAbsolutePath();
+ } else if(convertAvailableInPath()) {
+ convertCmd = CONVERT_COMMAND;
+ } else {
+ ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
+ }
// convert pdf to jpg using internal pdfbox capability
if (convertCmd == null) {
try (final PDDocument pdoc = PDDocument.load(pdf);) {
-
- BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
+
+ final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
return ImageIO.write(bi, imageFormat, image);
} catch (final IOException ex) {
- ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
- + (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
- return false;
+ ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
+ + (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
+ return false;
}
}
@@ -367,18 +369,18 @@ public class Html2Image {
try {
// i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg
// note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf
- String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
+ final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
List message = OS.execSynchronous(command);
if (image.exists()) return true;
ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command);
- for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
-
+ for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
+
// another try for mac: use Image Events using AppleScript in osacript commands...
// the following command overwrites a pdf with an png, so we must make a copy first
if (!OS.isMacArchitecture) return false;
- File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
+ final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
org.apache.commons.io.FileUtils.copyFile(pdf, pngFile);
- String[] commandx = {"osascript",
+ final String[] commandx = {"osascript",
"-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"",
"-e", "tell application \"Image Events\"",
"-e", "set Img to open file ImgFile",
@@ -386,15 +388,15 @@ public class Html2Image {
"-e", "end tell"};
//ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx);
message = OS.execSynchronous(commandx);
- for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
+ for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
// now we must read and convert this file to the target format with the target size 1024x1024
try {
- File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
+ final File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
pngFile.renameTo(newPngFile);
final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile));
if(img == null) {
- /* Should not happen. If so, ImageParser.parse() should already have logged about the error */
- return false;
+ /* Should not happen. If so, ImageParser.parse() should already have logged about the error */
+ return false;
}
final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
final MediaTracker mediaTracker = new MediaTracker(new Container());
@@ -406,24 +408,24 @@ public class Html2Image {
ImageIO.write(bi, imageFormat, image);
newPngFile.delete();
return image.exists();
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.logException(e);
return false;
}
- } catch (IOException e) {
+ } catch (final IOException e) {
e.printStackTrace();
return false;
}
}
-
+
/**
* render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported!
* @param url
* @param size
- * @throws IOException
+ * @throws IOException
*/
public static void writeSwingImage(String url, Dimension size, File destination) throws IOException {
-
+
// set up a pane for rendering
final JEditorPane htmlPane = new JEditorPane();
htmlPane.setSize(size);
@@ -434,7 +436,7 @@ public class Html2Image {
@Override
public Document createDefaultDocument() {
- HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
+ final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
doc.setAsynchronousLoadPriority(-1);
return doc;
}
@@ -444,7 +446,7 @@ public class Html2Image {
return new HTMLFactory() {
@Override
public View create(Element elem) {
- View view = super.create(elem);
+ final View view = super.create(elem);
if (view instanceof ImageView) {
((ImageView) view).setLoadsSynchronously(true);
}
@@ -460,23 +462,23 @@ public class Html2Image {
public void propertyChange(PropertyChangeEvent evt) {
}
});
-
+
// load the page
try {
htmlPane.setPage(url);
- } catch (IOException e) {
+ } catch (final IOException e) {
e.printStackTrace();
}
-
+
// render the page
- Dimension prefSize = htmlPane.getPreferredSize();
- BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
- Graphics graphics = img.getGraphics();
+ final Dimension prefSize = htmlPane.getPreferredSize();
+ final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
+ final Graphics graphics = img.getGraphics();
htmlPane.setSize(prefSize);
htmlPane.paint(graphics);
ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination);
}
-
+
/**
* Test PDF or image snapshot generation for a given URL.
* @param args main arguments list:
@@ -491,88 +493,88 @@ public class Html2Image {
*
*
*/
- public static void main(String[] args) {
- final String usageMessage = "Usage : java " + Html2Image.class.getName()
- + " [wkhtmltopdf|swing]";
- int exitStatus = 0;
- try {
- if (args.length < 2) {
- System.out.println("Missing required parameter(s).");
- System.out.println(usageMessage);
- exitStatus = 1;
- return;
- }
- final String targetPath = args[1];
- if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
- if(Html2Image.wkhtmltopdfAvailable()) {
- final File targetPdfFile;
- if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
- targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
- } else if(targetPath.endsWith(".pdf")) {
- targetPdfFile = new File(targetPath);
- } else {
- System.out.println("Unsupported output format");
- System.out.println(usageMessage);
- exitStatus = 1;
- return;
- }
- if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,
- "en-us,en;q=0.5", targetPdfFile, 30)) {
- if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
- if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
- ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
- } else {
- exitStatus = 1;
- return;
- }
- }
- } else {
- exitStatus = 1;
- return;
- }
- } else {
- System.out.println("Unable to locate wkhtmltopdf executable on this system!");
- exitStatus = 1;
- return;
- }
- } else if ("swing".equals(args[2])) {
- if(targetPath.endsWith(".pdf")) {
- System.out.println("Pdf output format is not supported with swing method.");
- exitStatus = 1;
- return;
- }
- if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
- System.out.println("Unsupported output format");
- System.out.println(usageMessage);
- exitStatus = 1;
- return;
- }
-
- try {
- Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
- } catch (final IOException e) {
- e.printStackTrace();
- exitStatus = 1;
- return;
- }
- } else {
- System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
- exitStatus = 1;
- return;
- }
- } finally {
- /* Shutdown running threads */
- Domains.close();
- try {
- HTTPClient.closeConnectionManager();
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt(); // restore interrupted state
- }
- ConcurrentLog.shutdown();
- if(exitStatus != 0) {
- System.exit(exitStatus);
- }
- }
- }
-
+ public static void main(String[] args) {
+ final String usageMessage = "Usage : java " + Html2Image.class.getName()
+ + " [wkhtmltopdf|swing]";
+ int exitStatus = 0;
+ try {
+ if (args.length < 2) {
+ System.out.println("Missing required parameter(s).");
+ System.out.println(usageMessage);
+ exitStatus = 1;
+ return;
+ }
+ final String targetPath = args[1];
+ if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
+ if(Html2Image.wkhtmltopdfAvailable()) {
+ final File targetPdfFile;
+ if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
+ targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
+ } else if(targetPath.endsWith(".pdf")) {
+ targetPdfFile = new File(targetPath);
+ } else {
+ System.out.println("Unsupported output format");
+ System.out.println(usageMessage);
+ exitStatus = 1;
+ return;
+ }
+ if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,
+ "en-us,en;q=0.5", targetPdfFile, 30)) {
+ if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
+ if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
+ ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
+ } else {
+ exitStatus = 1;
+ return;
+ }
+ }
+ } else {
+ exitStatus = 1;
+ return;
+ }
+ } else {
+ System.out.println("Unable to locate wkhtmltopdf executable on this system!");
+ exitStatus = 1;
+ return;
+ }
+ } else if ("swing".equals(args[2])) {
+ if(targetPath.endsWith(".pdf")) {
+ System.out.println("Pdf output format is not supported with swing method.");
+ exitStatus = 1;
+ return;
+ }
+ if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
+ System.out.println("Unsupported output format");
+ System.out.println(usageMessage);
+ exitStatus = 1;
+ return;
+ }
+
+ try {
+ Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
+ } catch (final IOException e) {
+ e.printStackTrace();
+ exitStatus = 1;
+ return;
+ }
+ } else {
+ System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
+ exitStatus = 1;
+ return;
+ }
+ } finally {
+ /* Shutdown running threads */
+ Domains.close();
+ try {
+ HTTPClient.closeConnectionManager();
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt(); // restore interrupted state
+ }
+ ConcurrentLog.shutdown();
+ if(exitStatus != 0) {
+ System.exit(exitStatus);
+ }
+ }
+ }
+
}
diff --git a/source/net/yacy/cora/util/Memory.java b/source/net/yacy/cora/util/Memory.java
index 3a1334d91..eb47f0908 100644
--- a/source/net/yacy/cora/util/Memory.java
+++ b/source/net/yacy/cora/util/Memory.java
@@ -115,8 +115,9 @@ public class Memory {
* @return the "recent cpu usage" for the whole operating environment;
* a negative value if not available.
*/
+ @SuppressWarnings("deprecation")
public static double getSystemCpuLoad() {
- com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+ final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
return operatingSystemMXBean.getSystemCpuLoad();
}
@@ -136,13 +137,13 @@ public class Memory {
* a negative value if not available.
*/
public static double getProcessCpuLoad() {
- com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+ final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
return operatingSystemMXBean.getProcessCpuLoad();
}
public static Map status() {
- Runtime runtime = Runtime.getRuntime();
- Map status = new LinkedHashMap<>();
+ final Runtime runtime = Runtime.getRuntime();
+ final Map status = new LinkedHashMap<>();
status.put("service", "Peer");
status.put("assigned_memory", runtime.maxMemory());
status.put("used_memory", runtime.totalMemory() - runtime.freeMemory());
@@ -153,7 +154,7 @@ public class Memory {
status.put("load_system_load_average", Memory.getSystemLoadAverage());
status.put("load_system_cpu_load", Memory.getSystemCpuLoad());
status.put("load_process_cpu_load", Memory.getProcessCpuLoad());
- YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();
+ final YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();
status.put("server_threads", server == null ? 0 : server.getServerThreads());
return status;
}
@@ -163,7 +164,7 @@ public class Memory {
* @return the number of deadlocked threads
*/
public static long deadlocks() {
- long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
+ final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
if (deadlockIDs == null) return 0;
return deadlockIDs.length;
}
@@ -172,10 +173,10 @@ public class Memory {
* write deadlocked threads as to the log as warning
*/
public static void logDeadlocks() {
- long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
+ final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
if (deadlockIDs == null) return;
- ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);
- for (ThreadInfo ti : infos) {
+ final ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);
+ for (final ThreadInfo ti : infos) {
ConcurrentLog.warn("DEADLOCKREPORT", ti.toString());
}
}
diff --git a/source/net/yacy/data/list/XMLBlacklistImporter.java b/source/net/yacy/data/list/XMLBlacklistImporter.java
index 6f72f091d..20155df3c 100644
--- a/source/net/yacy/data/list/XMLBlacklistImporter.java
+++ b/source/net/yacy/data/list/XMLBlacklistImporter.java
@@ -33,7 +33,6 @@ import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
-import org.xml.sax.helpers.XMLReaderFactory;
/**
* This class provides methods to import blacklists from an XML file (see
@@ -54,13 +53,15 @@ public class XMLBlacklistImporter extends DefaultHandler {
* @throws java.io.IOException if input can't be read
* @throws org.xml.sax.SAXException if XML can't be parsed
*/
+ @SuppressWarnings("deprecation")
public synchronized ListAccumulator parse(InputSource input) throws IOException, SAXException {
- XMLReader reader = XMLReaderFactory.createXMLReader();
+ @SuppressWarnings("deprecation")
+ final XMLReader reader = org.xml.sax.helpers.XMLReaderFactory.createXMLReader();
reader.setContentHandler(this);
reader.parse(input);
- return ba;
+ return this.ba;
}
/**
@@ -104,7 +105,7 @@ public class XMLBlacklistImporter extends DefaultHandler {
*/
@Override
public void startDocument() {
- ba = new ListAccumulator();
+ this.ba = new ListAccumulator();
}
/**
@@ -127,24 +128,24 @@ public class XMLBlacklistImporter extends DefaultHandler {
public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) {
if (qName.equalsIgnoreCase("list")) {
- currentListName = attributes.getValue("name");
- ba.addList(currentListName);
-
+ this.currentListName = attributes.getValue("name");
+ this.ba.addList(this.currentListName);
+
int attributesLength = 0;
if ((attributesLength = attributes.getLength()) > 1) {
for (int i = 0; i < attributesLength; i++) {
if (!attributes.getQName(i).equals("name")) {
- ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));
+ this.ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));
}
}
}
}
-
+
if (qName.equalsIgnoreCase("item")) {
- lastText = new StringBuilder();
+ this.lastText = new StringBuilder();
}
-
+
}
/**
@@ -162,7 +163,7 @@ public class XMLBlacklistImporter extends DefaultHandler {
@Override
public void endElement(final String uri, final String localName, final String qName) throws SAXException {
if (qName.equalsIgnoreCase("item")) {
- ba.addEntryToCurrent(lastText.toString());
+ this.ba.addEntryToCurrent(this.lastText.toString());
}
}
@@ -175,8 +176,8 @@ public class XMLBlacklistImporter extends DefaultHandler {
*/
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
- if (lastText == null) lastText = new StringBuilder();
- lastText.append(ch, start, length);
+ if (this.lastText == null) this.lastText = new StringBuilder();
+ this.lastText.append(ch, start, length);
}
}
diff --git a/source/net/yacy/gui/framework/Browser.java b/source/net/yacy/gui/framework/Browser.java
index 0b967c6ac..741a30047 100644
--- a/source/net/yacy/gui/framework/Browser.java
+++ b/source/net/yacy/gui/framework/Browser.java
@@ -37,7 +37,7 @@ import net.yacy.cora.util.ConcurrentLog;
public class Browser {
- // constants for system identification
+ // constants for system identification
public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.*
public static final int systemMacOSX = 1; // all Mac OS X
public static final int systemUnix = 2; // all Unix/Linux type systems
@@ -100,7 +100,7 @@ public class Browser {
openBrowserClassic(url);
}
}
-
+
public static void openBrowserClassic(final String url) {
try {
if (systemOS == systemMacOSX) {
@@ -113,12 +113,12 @@ public class Browser {
throw new RuntimeException("System unknown");
}
} catch (final Throwable e) {
- ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());
+ ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());
}
}
private static void openBrowserMac(final String url) throws Exception {
- Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});
+ final Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});
p.waitFor();
if (p.exitValue() != 0) {
throw new RuntimeException("Mac Exec Error: " + errorResponse(p));
@@ -131,12 +131,13 @@ public class Browser {
* @throws Exception when an error occured
*/
private static void openDefaultUnixBrowser(final String url) throws Exception {
- /* Use the freedesktop xdg-open to open url with the default browser.
- * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)
- * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it
- * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */
- String cmd = "xdg-open " + url;
- Process p = Runtime.getRuntime().exec(cmd);
+ /* Use the freedesktop xdg-open to open url with the default browser.
+ * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)
+ * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it
+ * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */
+ final String cmd = "xdg-open " + url;
+ @SuppressWarnings("deprecation")
+ final Process p = Runtime.getRuntime().exec(cmd);
p.waitFor();
if (p.exitValue() != 0) {
throw new RuntimeException("Unix Exec Error/xdg-open: " + errorResponse(p));
@@ -152,7 +153,8 @@ public class Browser {
cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\"";
}
//cmd = "cmd.exe /c start javascript:document.location='" + url + "'";
- Process p = Runtime.getRuntime().exec(cmd);
+ @SuppressWarnings("deprecation")
+ final Process p = Runtime.getRuntime().exec(cmd);
p.waitFor();
if (p.exitValue() != 0) {
throw new RuntimeException("EXEC ERROR: " + errorResponse(p));
@@ -183,15 +185,15 @@ public class Browser {
* @param args
*/
public static void main(final String[] args) {
- try {
- if (args.length > 0 && "-u".equals(args[0])) {
- openBrowser(args[1]);
- } else {
- System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");
- }
- } finally {
- ConcurrentLog.shutdown();
- }
- System.out.println("The End!");
- }
+ try {
+ if (args.length > 0 && "-u".equals(args[0])) {
+ openBrowser(args[1]);
+ } else {
+ System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");
+ }
+ } finally {
+ ConcurrentLog.shutdown();
+ }
+ System.out.println("The End!");
+ }
}
diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java
index 711d0c71a..408ee459d 100644
--- a/source/net/yacy/kelondro/util/FileUtils.java
+++ b/source/net/yacy/kelondro/util/FileUtils.java
@@ -56,15 +56,14 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream;
-import net.yacy.cora.document.encoding.UTF8;
-import net.yacy.cora.storage.Files;
-import net.yacy.cora.util.ConcurrentLog;
-
import org.apache.commons.lang.StringUtils;
-
import org.mozilla.intl.chardet.nsDetector;
import org.mozilla.intl.chardet.nsPSMDetector;
+import net.yacy.cora.document.encoding.UTF8;
+import net.yacy.cora.storage.Files;
+import net.yacy.cora.util.ConcurrentLog;
+
public final class FileUtils {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
@@ -98,7 +97,7 @@ public final class FileUtils {
* @see #copy(File source, File dest)
*/
public static long copy(final InputStream source, final OutputStream dest, final long count)
- throws IOException {
+ throws IOException {
assert count < 0 || count > 0 : "precondition violated: count == " + count + " (nothing to copy)";
if ( count == 0 ) {
// no bytes to copy
@@ -106,15 +105,15 @@ public final class FileUtils {
}
final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
- int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
+ final int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
int c;
long total = 0;
long remaining;
if(count > 0) {
- remaining = count;
+ remaining = count;
} else {
- remaining = Long.MAX_VALUE;
+ remaining = Long.MAX_VALUE;
}
while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) {
dest.write(buffer, 0, c);
@@ -158,7 +157,7 @@ public final class FileUtils {
* @throws NullPointerException when a parameter is null
*/
public static int copy(final InputStream source, final Writer dest, final Charset inputCharset)
- throws IOException {
+ throws IOException {
final InputStreamReader reader = new InputStreamReader(source, inputCharset);
return copy(reader, dest);
}
@@ -212,8 +211,8 @@ public final class FileUtils {
// an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here
//Log.logException(e);
throw new IOException(
- e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),
- e);
+ e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),
+ e);
}
return count;
}
@@ -260,8 +259,8 @@ public final class FileUtils {
fos.close();
} catch (final Exception e ) {
ConcurrentLog.warn(
- "FileUtils",
- "cannot close FileOutputStream for " + dest + "! " + e.getMessage());
+ "FileUtils",
+ "cannot close FileOutputStream for " + dest + "! " + e.getMessage());
}
}
}
@@ -282,17 +281,17 @@ public final class FileUtils {
* @see #copy(File source, File dest)
*/
public static void copyRange(final File source, final OutputStream dest, final int start)
- throws IOException {
+ throws IOException {
InputStream fis = null;
try {
fis = new FileInputStream(source);
final long skipped = fis.skip(start);
if ( skipped != start ) {
throw new IllegalStateException("Unable to skip '"
- + start
- + "' bytes. Only '"
- + skipped
- + "' bytes skipped.");
+ + start
+ + "' bytes. Only '"
+ + skipped
+ + "' bytes skipped.");
}
copy(fis, dest, -1);
} finally {
@@ -300,7 +299,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
}
@@ -328,7 +327,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
}
@@ -368,17 +367,17 @@ public final class FileUtils {
* @throws NullPointerException when source parameter is null
*/
public static byte[] read(final InputStream source) throws IOException {
- byte[] content;
- try {
- content = read(source, -1);
- } finally {
- /* source input stream must be closed here in all cases */
- try {
- source.close();
- } catch(IOException ignoredException) {
- }
- }
- return content;
+ byte[] content;
+ try {
+ content = read(source, -1);
+ } finally {
+ /* source input stream must be closed here in all cases */
+ try {
+ source.close();
+ } catch(final IOException ignoredException) {
+ }
+ }
+ return content;
}
/**
@@ -392,7 +391,7 @@ public final class FileUtils {
*/
public static byte[] read(final InputStream source, final int count) throws IOException {
if(count == 0) {
- return new byte[0];
+ return new byte[0];
}
final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
copy(source, baos, count);
@@ -414,7 +413,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
fis = null;
@@ -470,17 +469,17 @@ public final class FileUtils {
return source;
}
- /**
- * Generate a set of strings matching each line of the given file. Lines are
- * lower cased and any eventual surrounding space characters are removed. Empty
- * lines and lines starting with the '#' character are ignored.
- *
- * @param file
- * a file to load
- * @return a set of strings eventually empty
- */
+ /**
+ * Generate a set of strings matching each line of the given file. Lines are
+ * lower cased and any eventual surrounding space characters are removed. Empty
+ * lines and lines starting with the '#' character are ignored.
+ *
+ * @param file
+ * a file to load
+ * @return a set of strings eventually empty
+ */
public static HashSet loadList(final File file) {
- final HashSet set = new HashSet();
+ final HashSet set = new HashSet<>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
@@ -497,7 +496,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
}
}
}
@@ -516,10 +515,10 @@ public final class FileUtils {
}
public static ConcurrentHashMap loadMapB(final File f) {
- ConcurrentHashMap m = loadMap(f);
+ final ConcurrentHashMap m = loadMap(f);
if (m == null) return null;
- ConcurrentHashMap mb = new ConcurrentHashMap();
- for (Map.Entry e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));
+ final ConcurrentHashMap mb = new ConcurrentHashMap<>();
+ for (final Map.Entry e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));
return mb;
}
@@ -529,7 +528,7 @@ public final class FileUtils {
private final static String[] unescaped_strings_out = {"\\", "\n", "="};
public static void saveMap(final File file, final Map props, final String comment) {
- boolean err = false;
+ boolean err = false;
PrintWriter pw = null;
final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000));
try {
@@ -550,10 +549,7 @@ public final class FileUtils {
pw.println(key + "=" + value);
}
pw.println("# EOF");
- } catch (final FileNotFoundException e ) {
- ConcurrentLog.warn("FileUtils", e.getMessage(), e);
- err = true;
- } catch (final UnsupportedEncodingException e ) {
+ } catch (final FileNotFoundException | UnsupportedEncodingException e ) {
ConcurrentLog.warn("FileUtils", e.getMessage(), e);
err = true;
} finally {
@@ -568,10 +564,10 @@ public final class FileUtils {
// ignore
}
}
-
+
public static void saveMapB(final File file, final Map props, final String comment) {
- HashMap m = new HashMap();
- for (Map.Entry e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));
+ final HashMap m = new HashMap<>();
+ for (final Map.Entry e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));
saveMap(file, m, comment);
}
@@ -582,7 +578,7 @@ public final class FileUtils {
public static ConcurrentHashMap table(final Iterator li) {
String line;
- final ConcurrentHashMap props = new ConcurrentHashMap();
+ final ConcurrentHashMap props = new ConcurrentHashMap<>();
while ( li.hasNext() ) {
int pos = 0;
line = li.next().trim();
@@ -594,8 +590,8 @@ public final class FileUtils {
pos = line.indexOf('=', pos + 1);
} while ( pos > 0 && line.charAt(pos - 1) == '\\' );
if ( pos > 0 ) try {
- String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);
- String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);
+ final String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);
+ final String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);
//System.out.println("key = " + key + ", value = " + value);
props.put(key, value);
} catch (final IndexOutOfBoundsException e) {
@@ -606,7 +602,7 @@ public final class FileUtils {
}
public static Map table(final byte[] a) {
- if (a == null) return new ConcurrentHashMap();
+ if (a == null) return new ConcurrentHashMap<>();
//System.out.println("***TABLE: a.size = " + a.length);
return table(strings(a));
}
@@ -627,7 +623,7 @@ public final class FileUtils {
*/
public static ArrayList getListArray(final File listFile) {
String line;
- final ArrayList list = new ArrayList();
+ final ArrayList list = new ArrayList<>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile), StandardCharsets.UTF_8));
@@ -642,7 +638,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
}
}
}
@@ -711,7 +707,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
}
}
}
@@ -760,7 +756,7 @@ public final class FileUtils {
* @return array of file names
*/
public static List getDirListing(final File dir, final String filter) {
- final List ret = new LinkedList();
+ final List ret = new LinkedList<>();
File[] fileList;
if ( dir != null ) {
if ( !dir.exists() ) {
@@ -781,39 +777,39 @@ public final class FileUtils {
public static ArrayList getDirsRecursive(final File dir, final String notdir) {
return getDirsRecursive(dir, notdir, true);
}
-
+
/**
* @param sourceDir source directory. Must be not null.
* @param notdir name of dir to exlcude. Can be null
* @param fileNameFilter filter to apply on file names. Can be null.
- * @return list of all files passing fileFilter under sourceDir including sub directories
+ * @return list of all files passing fileFilter under sourceDir including sub directories
*/
public static List getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) {
- List dirList = getDirsRecursive(sourceDir,
- notdir);
- dirList.add(sourceDir);
- List files = new ArrayList<>();
- for (final File dir : dirList) {
- Collections.addAll(files, dir.listFiles(fileNameFilter));
- }
- return files;
+ final List dirList = getDirsRecursive(sourceDir,
+ notdir);
+ dirList.add(sourceDir);
+ final List files = new ArrayList<>();
+ for (final File dir : dirList) {
+ Collections.addAll(files, dir.listFiles(fileNameFilter));
+ }
+ return files;
}
/**
* Returns a List of all dirs and subdirs as File Objects Warning: untested
*/
private static ArrayList getDirsRecursive(
- final File dir,
- final String notdir,
- final boolean excludeDotfiles) {
+ final File dir,
+ final String notdir,
+ final boolean excludeDotfiles) {
final File[] dirList = dir.listFiles();
- final ArrayList resultList = new ArrayList();
+ final ArrayList resultList = new ArrayList<>();
ArrayList recursive;
Iterator iter;
for ( int i = 0; i < dirList.length; i++ ) {
if ( dirList[i].isDirectory()
- && (!excludeDotfiles || !dirList[i].getName().startsWith("."))
- && !dirList[i].getName().equals(notdir) ) {
+ && (!excludeDotfiles || !dirList[i].getName().startsWith("."))
+ && !dirList[i].getName().equals(notdir) ) {
resultList.add(dirList[i]);
recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles);
iter = recursive.iterator();
@@ -927,12 +923,12 @@ public final class FileUtils {
// create the temp file
final File tempFile =
- File.createTempFile(
- parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),
- (!fileExt.isEmpty()) ? "." + fileExt : fileExt);
+ File.createTempFile(
+ parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),
+ (!fileExt.isEmpty()) ? "." + fileExt : fileExt);
return tempFile;
}
-
+
/**
* delete files and directories if a directory is not empty, delete also everything inside because
* deletion sometimes fails on windows, there is also a windows exec included
@@ -973,7 +969,7 @@ public final class FileUtils {
break;
}
}
- */
+ */
if ( path.exists() ) {
path.deleteOnExit();
String p = "";
@@ -986,6 +982,7 @@ public final class FileUtils {
// deleting files on windows sometimes does not work with java
try {
final String command = "cmd /C del /F /Q \"" + p + "\"";
+ @SuppressWarnings("deprecation")
final Process r = Runtime.getRuntime().exec(command);
if ( r == null ) {
ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);
@@ -1002,7 +999,7 @@ public final class FileUtils {
}
}
}
-
+
/**
* Checks if a certain file is in a given directory.
* @param file the file to check
@@ -1010,9 +1007,9 @@ public final class FileUtils {
* @return true if file is contained in directory
*/
public static boolean isInDirectory(final File file, final File directory) {
-
+
boolean inDirectory;
-
+
try {
inDirectory = (
directory != null
@@ -1024,10 +1021,10 @@ public final class FileUtils {
} catch (final IOException e) {
inDirectory = false;
}
-
+
return inDirectory;
}
-
+
/**
* Auto-detect the charset of content in a stream.
* Used code from http://jchardet.sourceforge.net/.
@@ -1040,28 +1037,28 @@ public final class FileUtils {
public static List detectCharset(final InputStream inStream) throws IOException {
// auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.html
List result;
- nsDetector det = new nsDetector(nsPSMDetector.ALL);
- byte[] buf = new byte[1024] ;
+ final nsDetector det = new nsDetector(nsPSMDetector.ALL);
+ final byte[] buf = new byte[1024] ;
int len;
boolean done = false ;
boolean isAscii = true ;
while ((len = inStream.read(buf,0,buf.length)) != -1) {
if (isAscii) {
- isAscii = det.isAscii(buf,len);
+ isAscii = det.isAscii(buf,len);
}
if (!isAscii && !done) {
- done = det.DoIt(buf,len, false);
+ done = det.DoIt(buf,len, false);
}
} det.DataEnd();
result = new ArrayList<>();
if (isAscii) {
result.add(StandardCharsets.US_ASCII.name());
} else {
- for (String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"
+ for (final String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"
}
return result;
}
-
+
/**
* Because the checking of very large files for their charset may take some time, we do this concurrently in this method
* This method does not return anything but it logs an info line if the charset is a good choice
@@ -1071,19 +1068,19 @@ public final class FileUtils {
* @param concurrent if this shall run concurrently
*/
public static void checkCharset(final File file, final String givenCharset, final boolean concurrent) {
- Thread t = new Thread("FileUtils.checkCharset") {
+ final Thread t = new Thread("FileUtils.checkCharset") {
@Override
public void run() {
- try (final FileInputStream fileStream = new FileInputStream(file);
- final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources
- List charsets = FileUtils.detectCharset(imp);
+ try (final FileInputStream fileStream = new FileInputStream(file);
+ final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources
+ final List charsets = FileUtils.detectCharset(imp);
if (charsets.contains(givenCharset)) {
ConcurrentLog.info("checkCharset", "appropriate charset '" + givenCharset + "' for import of " + file + ", is part one detected " + charsets);
} else {
ConcurrentLog.warn("checkCharset", "possibly wrong charset '" + givenCharset + "' for import of " + file + ", use one of " + charsets);
}
- } catch (IOException e) {}
-
+ } catch (final IOException e) {}
+
}
};
if (concurrent) t.start(); else t.run();
diff --git a/source/net/yacy/kelondro/util/OS.java b/source/net/yacy/kelondro/util/OS.java
index beca081a1..abf7d1695 100644
--- a/source/net/yacy/kelondro/util/OS.java
+++ b/source/net/yacy/kelondro/util/OS.java
@@ -41,7 +41,7 @@ import net.yacy.server.serverCore;
public final class OS {
- // constants for system identification
+ // constants for system identification
public enum System {
MacOSC, // 'classic' Mac OS 7.6.1/8.*/9.*
MacOSX, // all Mac OS X
@@ -50,125 +50,129 @@ public final class OS {
Unknown; // any other system
}
- // constants for file type identification (Mac only)
- public static final String blankTypeString = "____";
-
- // system-identification statics
- private static final System systemOS;
- public static final boolean isMacArchitecture;
- private static final boolean isUnixFS;
- public static final boolean canExecUnix;
- public static final boolean isWindows;
- public static final boolean isWin32;
-
- // calculated system constants
- public static int maxPathLength = 65535;
-
- // Macintosh-specific statics
- public static final Map macFSTypeCache = new HashMap();
- public static final Map macFSCreatorCache = new HashMap();
-
- // static initialization
- static {
- // check operation system type
- final Properties sysprop = java.lang.System.getProperties();
- final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
- if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;
- else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;
- else if (sysname.startsWith("windows")) systemOS = System.Windows;
- else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;
- else systemOS = System.Unknown;
-
- isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));
- isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));
- canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));
- isWindows = (systemOS == System.Windows);
- isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));
-
- // set up maximum path length according to system
- if (isWindows) maxPathLength = 255; else maxPathLength = 65535;
- }
-
- public static String infoString() {
- String s = "System=";
- if (systemOS == System.Unknown) s += "unknown";
- else if (systemOS == System.MacOSC) s += "Mac OS Classic";
- else if (systemOS == System.MacOSX) s += "Mac OS X";
- else if (systemOS == System.Unix) s += "Unix/Linux";
- else if (systemOS == System.Windows) s += "Windows";
- else s += "unknown";
- if (isMacArchitecture) s += ", Mac System Architecture";
- if (isUnixFS) s += ", has Unix-like File System";
- if (canExecUnix) s += ", can execute Unix-Shell Commands";
- return s;
- }
-
- /** generates a 2-character string containing information about the OS-type*/
- public static String infoKey() {
- String s = "";
- if (systemOS == System.Unknown) s += "o";
- else if (systemOS == System.MacOSC) s += "c";
- else if (systemOS == System.MacOSX) s += "x";
- else if (systemOS == System.Unix) s += "u";
- else if (systemOS == System.Windows) s += "w";
- else s += "o";
- if (isMacArchitecture) s += "m";
- if (isUnixFS) s += "f";
- if (canExecUnix) s += "e";
- return s;
- }
-
- public static void deployScript(final File scriptFile, final String theScript) throws IOException {
- FileUtils.copy(UTF8.getBytes(theScript), scriptFile);
- if(!isWindows){ // set executable
- try {
- Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
- } catch (final InterruptedException e) {
- ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
- throw new IOException(e.getMessage());
- }
- }
- }
+ // constants for file type identification (Mac only)
+ public static final String blankTypeString = "____";
+
+ // system-identification statics
+ private static final System systemOS;
+ public static final boolean isMacArchitecture;
+ private static final boolean isUnixFS;
+ public static final boolean canExecUnix;
+ public static final boolean isWindows;
+ public static final boolean isWin32;
+
+ // calculated system constants
+ public static int maxPathLength = 65535;
+
+ // Macintosh-specific statics
+ public static final Map macFSTypeCache = new HashMap<>();
+ public static final Map macFSCreatorCache = new HashMap<>();
+
+ // static initialization
+ static {
+ // check operation system type
+ final Properties sysprop = java.lang.System.getProperties();
+ final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
+ if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;
+ else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;
+ else if (sysname.startsWith("windows")) systemOS = System.Windows;
+ else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;
+ else systemOS = System.Unknown;
+
+ isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));
+ isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));
+ canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));
+ isWindows = (systemOS == System.Windows);
+ isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));
+
+ // set up maximum path length according to system
+ if (isWindows) maxPathLength = 255; else maxPathLength = 65535;
+ }
+
+ public static String infoString() {
+ String s = "System=";
+ if (systemOS == System.Unknown) s += "unknown";
+ else if (systemOS == System.MacOSC) s += "Mac OS Classic";
+ else if (systemOS == System.MacOSX) s += "Mac OS X";
+ else if (systemOS == System.Unix) s += "Unix/Linux";
+ else if (systemOS == System.Windows) s += "Windows";
+ else s += "unknown";
+ if (isMacArchitecture) s += ", Mac System Architecture";
+ if (isUnixFS) s += ", has Unix-like File System";
+ if (canExecUnix) s += ", can execute Unix-Shell Commands";
+ return s;
+ }
+
+ /** generates a 2-character string containing information about the OS-type*/
+ public static String infoKey() {
+ String s = "";
+ if (systemOS == System.Unknown) s += "o";
+ else if (systemOS == System.MacOSC) s += "c";
+ else if (systemOS == System.MacOSX) s += "x";
+ else if (systemOS == System.Unix) s += "u";
+ else if (systemOS == System.Windows) s += "w";
+ else s += "o";
+ if (isMacArchitecture) s += "m";
+ if (isUnixFS) s += "f";
+ if (canExecUnix) s += "e";
+ return s;
+ }
- /**
- * use a hack to get the current process PID
- * @return the PID of the current java process or -1 if the PID cannot be obtained
- */
- public static int getPID() {
+ @SuppressWarnings("deprecation")
+ public static void deployScript(final File scriptFile, final String theScript) throws IOException {
+ FileUtils.copy(UTF8.getBytes(theScript), scriptFile);
+ if(!isWindows){ // set executable
+ try {
+ Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
+ } catch (final InterruptedException e) {
+ ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
+ throw new IOException(e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * use a hack to get the current process PID
+ * @return the PID of the current java process or -1 if the PID cannot be obtained
+ */
+ public static int getPID() {
final String pids = ManagementFactory.getRuntimeMXBean().getName();
final int p = pids.indexOf('@');
return p >= 0 ? NumberTools.parseIntDecSubstring(pids, 0, p) : -1;
- }
-
- public static void execAsynchronous(final File scriptFile) throws IOException {
- // runs a script as separate thread
- String starterFileExtension = null;
- String script = null;
- if(isWindows){
- starterFileExtension = ".starter.bat";
- // use /K to debug, /C for release
- script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";
- } else { // unix/linux
- starterFileExtension = ".starter.sh";
- script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;
- }
- final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);
- deployScript(starterFile, script);
- try {
- Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
- } catch (final InterruptedException e) {
- throw new IOException(e.getMessage());
- }
- FileUtils.deletedelete(starterFile);
- }
+ }
+ @SuppressWarnings("deprecation")
+ public static void execAsynchronous(final File scriptFile) throws IOException {
+ // runs a script as separate thread
+ String starterFileExtension = null;
+ String script = null;
+ if(isWindows){
+ starterFileExtension = ".starter.bat";
+ // use /K to debug, /C for release
+ script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";
+ } else { // unix/linux
+ starterFileExtension = ".starter.sh";
+ script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;
+ }
+ final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);
+ deployScript(starterFile, script);
+ try {
+ Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
+ } catch (final InterruptedException e) {
+ throw new IOException(e.getMessage());
+ }
+ FileUtils.deletedelete(starterFile);
+ }
+
+ @SuppressWarnings("deprecation")
public static List execSynchronous(final String command) throws IOException {
// runs a unix/linux command and returns output as Vector of Strings
// this method blocks until the command is executed
final Process p = Runtime.getRuntime().exec(command);
return readStreams(p);
}
-
+
+ @SuppressWarnings("deprecation")
public static List execSynchronous(final String[] command) throws IOException {
// runs a unix/linux command and returns output as Vector of Strings
// this method blocks until the command is executed
@@ -184,40 +188,40 @@ public final class OS {
*/
public static List readStreams(final Process p) throws IOException {
String line;
- final List output = new ArrayList<>();
-
- if(p == null) {
- return output;
- }
-
- try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());
- final BufferedReader in = new BufferedReader(streamReader);) {
- while ((line = in.readLine()) != null) {
- output.add(line);
- }
- }
-
- try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());
- final BufferedReader in = new BufferedReader(streamReader);) {
- while ((line = in.readLine()) != null) {
- output.add(line);
- }
- }
+ final List output = new ArrayList<>();
+
+ if(p == null) {
+ return output;
+ }
+
+ try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());
+ final BufferedReader in = new BufferedReader(streamReader);) {
+ while ((line = in.readLine()) != null) {
+ output.add(line);
+ }
+ }
+
+ try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());
+ final BufferedReader in = new BufferedReader(streamReader);) {
+ while ((line = in.readLine()) != null) {
+ output.add(line);
+ }
+ }
return output;
}
- public static void main(final String[] args) {
+ public static void main(final String[] args) {
try {
- List v = execSynchronous("/usr/local/bin/wkhtmltoimage");
- for (String r: v) java.lang.System.out.println(r);
- } catch (IOException e) {
+ final List v = execSynchronous("/usr/local/bin/wkhtmltoimage");
+ for (final String r: v) java.lang.System.out.println(r);
+ } catch (final IOException e) {
}
/*
if (args[0].equals("-m")) {
java.lang.System.out.println("Maximum possible memory: " + Integer.toString(getWin32MaxHeap()) + "m");
}
- */
- }
+ */
+ }
}
diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java
index 744c4515c..e300ec9e2 100644
--- a/source/net/yacy/server/http/HTTPDProxyHandler.java
+++ b/source/net/yacy/server/http/HTTPDProxyHandler.java
@@ -37,7 +37,7 @@
support gzip-ed encoding. We also do not support unrealistic
'expires' values that would force a cache to be flushed immediately
pragma non-cache attributes are supported
-*/
+ */
package net.yacy.server.http;
@@ -67,6 +67,7 @@ import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.Logger;
+
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -89,6 +90,7 @@ import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
+@SuppressWarnings("deprecation")
public final class HTTPDProxyHandler {
@@ -108,7 +110,7 @@ public final class HTTPDProxyHandler {
// creating a logger
private static final ConcurrentLog log = new ConcurrentLog("PROXY");
- /**
+ /**
* Do logging configuration for special proxy access log file
*/
static {
@@ -117,77 +119,77 @@ public final class HTTPDProxyHandler {
sb = Switchboard.getSwitchboard();
if (sb != null) {
- // set timeout
- timeout = sb.getConfigInt("proxy.clientTimeout", 60000);
+ // set timeout
+ timeout = sb.getConfigInt("proxy.clientTimeout", 60000);
- // do logger initialization
- try {
- log.info("Configuring proxy access logging ...");
-
- // getting the logging manager
- final LogManager manager = LogManager.getLogManager();
- final String className = HTTPDProxyHandler.class.getName();
-
- // determining if proxy access logging is enabled
- final String enabled = manager.getProperty(className + ".logging.enabled");
- if ("true".equalsIgnoreCase(enabled)) {
-
- // reading out some needed configuration properties
- int limit = 1024*1024, count = 20;
- String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");
- if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";
- // make pattern absolute
- if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();
-
- final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");
- if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}
-
- final String countStr = manager.getProperty(className + ".logging.FileHandler.count");
- if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}
-
- // creating the proxy access logger
- final Logger proxyLogger = Logger.getLogger("PROXY.access");
- proxyLogger.setUseParentHandlers(false);
- proxyLogger.setLevel(Level.FINEST);
-
- final FileHandler txtLog = new FileHandler(pattern, limit, count, true);
- txtLog.setFormatter(new ProxyLogFormatter());
- txtLog.setLevel(Level.FINEST);
- proxyLogger.addHandler(txtLog);
-
- log.info("Proxy access logging configuration done." +
- "\n\tFilename: " + pattern +
- "\n\tLimit: " + limitStr +
- "\n\tCount: " + countStr);
- } else {
- log.info("Proxy access logging is deactivated.");
+ // do logger initialization
+ try {
+ log.info("Configuring proxy access logging ...");
+
+ // getting the logging manager
+ final LogManager manager = LogManager.getLogManager();
+ final String className = HTTPDProxyHandler.class.getName();
+
+ // determining if proxy access logging is enabled
+ final String enabled = manager.getProperty(className + ".logging.enabled");
+ if ("true".equalsIgnoreCase(enabled)) {
+
+ // reading out some needed configuration properties
+ int limit = 1024*1024, count = 20;
+ String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");
+ if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";
+ // make pattern absolute
+ if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();
+
+ final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");
+ if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}
+
+ final String countStr = manager.getProperty(className + ".logging.FileHandler.count");
+ if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}
+
+ // creating the proxy access logger
+ final Logger proxyLogger = Logger.getLogger("PROXY.access");
+ proxyLogger.setUseParentHandlers(false);
+ proxyLogger.setLevel(Level.FINEST);
+
+ final FileHandler txtLog = new FileHandler(pattern, limit, count, true);
+ txtLog.setFormatter(new ProxyLogFormatter());
+ txtLog.setLevel(Level.FINEST);
+ proxyLogger.addHandler(txtLog);
+
+ log.info("Proxy access logging configuration done." +
+ "\n\tFilename: " + pattern +
+ "\n\tLimit: " + limitStr +
+ "\n\tCount: " + countStr);
+ } else {
+ log.info("Proxy access logging is deactivated.");
+ }
+ } catch (final Exception e) {
+ log.severe("Unable to configure proxy access logging.",e);
}
- } catch (final Exception e) {
- log.severe("Unable to configure proxy access logging.",e);
- }
- // load the yellow-list
- final String f = sb.getConfig("proxyYellowList", null);
- if (f != null) {
- yellowList = FileUtils.loadList(new File(f));
- log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
- } else {
- yellowList = null;
- }
+ // load the yellow-list
+ final String f = sb.getConfig("proxyYellowList", null);
+ if (f != null) {
+ yellowList = FileUtils.loadList(new File(f));
+ log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
+ } else {
+ yellowList = null;
+ }
- final String redirectorPath = sb.getConfig("externalRedirector", "");
- if (redirectorPath.length() > 0 && !redirectorEnabled) {
- try {
- redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);
- redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
- redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));
- redirectorEnabled=true;
- } catch (final IOException e) {
- System.out.println("redirector not Found");
+ final String redirectorPath = sb.getConfig("externalRedirector", "");
+ if (redirectorPath.length() > 0 && !redirectorEnabled) {
+ try {
+ redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);
+ redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
+ redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));
+ redirectorEnabled=true;
+ } catch (final IOException e) {
+ System.out.println("redirector not Found");
+ }
}
- }
} else {
- yellowList = null;
+ yellowList = null;
}
}
@@ -342,25 +344,25 @@ public final class HTTPDProxyHandler {
// case 1 and case 3
if (cachedResponseHeader == null) {
if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web");
- fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
+ fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
} else {
- final Request request = new Request(
- null,
+ final Request request = new Request(
+ null,
url,
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
- "",
- cachedResponseHeader.lastModified(),
- sb.crawler.defaultProxyProfile.handle(),
- 0,
- sb.crawler.defaultProxyProfile.timezoneOffset());
+ "",
+ cachedResponseHeader.lastModified(),
+ sb.crawler.defaultProxyProfile.handle(),
+ 0,
+ sb.crawler.defaultProxyProfile.timezoneOffset());
final Response response = new Response(
- request,
+ request,
requestHeader,
cachedResponseHeader,
sb.crawler.defaultProxyProfile,
true,
null
- );
+ );
final byte[] cacheContent = Cache.getContent(url.hash());
if (cacheContent != null && response.isFreshForProxy()) {
if (log.isFinest()) log.finest(reqID + " fulfill request from cache");
@@ -433,9 +435,9 @@ public final class HTTPDProxyHandler {
// remove yacy-subdomain-path, when accessing /env
if ((yAddress != null)
- && (remotePath.startsWith("/env"))
- && (yAddress.indexOf('/') != -1)
- ) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
+ && (remotePath.startsWith("/env"))
+ && (yAddress.indexOf('/') != -1)
+ ) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
modifyProxyHeaders(requestHeader, httpVer);
@@ -448,17 +450,17 @@ public final class HTTPDProxyHandler {
try (final HTTPClient client = new HTTPClient(agent, timeout)) {
client.setHeader(requestHeader.entrySet());
client.setRedirecting(false);
- client.GET(getUrl, false);
+ client.GET(getUrl, false);
if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
- int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
+ final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
- throw new Exception(client.getHttpResponse().getStatusLine().toString());
+ throw new Exception(client.getHttpResponse().getStatusLine().toString());
}
- ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);
+ final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);
// the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1;
@@ -475,19 +477,19 @@ public final class HTTPDProxyHandler {
// reserver cache entry
final Request request = new Request(
- null,
+ null,
url,
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
- "",
- responseHeader.lastModified(),
- sb.crawler.defaultProxyProfile.handle(),
- 0,
- sb.crawler.defaultProxyProfile.timezoneOffset());
+ "",
+ responseHeader.lastModified(),
+ sb.crawler.defaultProxyProfile.handle(),
+ 0,
+ sb.crawler.defaultProxyProfile.timezoneOffset());
// handle incoming cookies
handleIncomingCookies(responseHeader, host, ip);
-// prepareResponseHeader(responseHeader, res.getHttpVer());
+ // prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client
@@ -506,7 +508,7 @@ public final class HTTPDProxyHandler {
if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
- OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
+ final OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
final Response response = new Response(
request,
requestHeader,
@@ -514,7 +516,7 @@ public final class HTTPDProxyHandler {
sb.crawler.defaultProxyProfile,
true,
null
- );
+ );
final String storeError = response.shallStoreCacheForProxy();
final boolean storeHTCache = response.profile().storeHTCache();
final String supportError = TextParser.supports(response.url(), response.getMimeType());
@@ -530,14 +532,14 @@ public final class HTTPDProxyHandler {
* c) the content should be indexed
*/
((storeHTCache) || (supportError != null))
- ) {
+ ) {
// we don't write actually into a file, only to RAM, and schedule writing the file.
-// int l = res.getResponseHeader().size();
- final int l = responseHeader.size();
+ // int l = res.getResponseHeader().size();
+ final int l = responseHeader.size();
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
-// FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
+ // FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
client.writeTo(toClientAndMemory);
// cached bytes
byte[] cacheArray;
@@ -627,7 +629,7 @@ public final class HTTPDProxyHandler {
final ResponseHeader cachedResponseHeader,
final byte[] cacheEntry,
OutputStream respond
- ) throws IOException {
+ ) throws IOException {
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
final String clienthttpVer; // the http version of the client connection
@@ -721,12 +723,12 @@ public final class HTTPDProxyHandler {
requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader));
}
- // only gzip-encoding is supported, remove other encodings (e. g. deflate)
+ // only gzip-encoding is supported, remove other encodings (e. g. deflate)
if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) {
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
- } else {
+ } else {
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, "");
- }
+ }
addXForwardedForHeader(conProp, requestHeader);
}
@@ -782,7 +784,7 @@ public final class HTTPDProxyHandler {
if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {
forceConnectionClose(conProp);
} else {
- chunkedOut = new ChunkedOutputStream(respond);
+ chunkedOut = new ChunkedOutputStream(respond);
}
responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
}
@@ -933,12 +935,12 @@ public final class HTTPDProxyHandler {
errorMessage = "IP address of the destination host could not be determined";
}
} else if ((exceptionMsg != null) &&
- (
- (exceptionMsg.indexOf("socket write error",0)>=0) ||
- (exceptionMsg.indexOf("Read timed out",0) >= 0) ||
- (exceptionMsg.indexOf("Broken pipe",0) >= 0) ||
- (exceptionMsg.indexOf("server has closed connection",0) >= 0)
- )) {
+ (
+ (exceptionMsg.indexOf("socket write error",0)>=0) ||
+ (exceptionMsg.indexOf("Read timed out",0) >= 0) ||
+ (exceptionMsg.indexOf("Broken pipe",0) >= 0) ||
+ (exceptionMsg.indexOf("server has closed connection",0) >= 0)
+ )) {
errorMessage = exceptionMsg;
ConcurrentLog.logException(e);
} else {
@@ -983,7 +985,7 @@ public final class HTTPDProxyHandler {
final serverObjects detailedErrorMsgMap = new serverObjects();
// generic toplevel domains
- final HashSet topLevelDomains = new HashSet(Arrays.asList(new String[]{
+ final HashSet topLevelDomains = new HashSet<>(Arrays.asList(new String[]{
"aero", // Fluggesellschaften/Luftfahrt
"arpa", // Einrichtung des ARPANet
"biz", // Business
@@ -1012,8 +1014,8 @@ public final class HTTPDProxyHandler {
}));
// getting some connection properties
- DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
- int orgHostPort = orgurl.getPort();
+ final DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
+ final int orgHostPort = orgurl.getPort();
String orgHostName = orgurl.getHost();
if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase(Locale.ROOT);
@@ -1025,7 +1027,7 @@ public final class HTTPDProxyHandler {
detailedErrorMsgMap.put("hostName", orgHostName);
// guessing hostnames
- final HashSet testHostNames = new HashSet();
+ final HashSet testHostNames = new HashSet<>();
String testHostName = null;
if (!orgHostName.startsWith("www.")) {
testHostName = "www." + orgHostName;
@@ -1042,7 +1044,7 @@ public final class HTTPDProxyHandler {
if (addr != null) testHostNames.add(testHostName);
}
- int pos = orgHostName.lastIndexOf('.');
+ final int pos = orgHostName.lastIndexOf('.');
if (pos != -1) {
final Iterator iter = topLevelDomains.iterator();
while (iter.hasNext()) {
@@ -1142,7 +1144,7 @@ public final class HTTPDProxyHandler {
logMessage.append(' ');
// Method
- HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
+ final HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
final String requestMethod = origrequest.getMethod();
logMessage.append(requestMethod);
logMessage.append(' ');
diff --git a/source/net/yacy/visualization/DemoApplet.java b/source/net/yacy/visualization/DemoApplet.java
deleted file mode 100644
index 55ceb5430..000000000
--- a/source/net/yacy/visualization/DemoApplet.java
+++ /dev/null
@@ -1,75 +0,0 @@
-package net.yacy.visualization;
-
-import java.applet.Applet;
-import java.awt.Dimension;
-import java.awt.Graphics;
-
-
-
-@SuppressWarnings("removal")
-public class DemoApplet extends Applet implements Runnable {
- // can be run in eclipse with
- // Run -> Run As -> Java Applet
-
- // see http://www.javaworld.com/javaworld/jw-03-1996/jw-03-animation.html?page=3
-
- private static final long serialVersionUID = -8230253094143014406L;
-
- private int delay;
- private Thread animator;
- private RasterPlotter offGraphics;
-
- @Override
- public void init() {
- final String str = getParameter("fps");
- final int fps = (str != null) ? Integer.parseInt(str) : 10;
- delay = (fps > 0) ? (1000 / fps) : 100;
- }
-
- @Override
- public void start() {
- animator = new Thread(this);
- animator.start();
- }
-
- @Override
- public void run() {
- while (Thread.currentThread() == animator) {
- final long time = System.currentTimeMillis();
- repaint();
- try {
- Thread.sleep(delay - System.currentTimeMillis() + time);
- } catch (final InterruptedException e) {
- break;
- }
- }
- }
-
- @Override
- public void stop() {
- animator = null;
- }
-
- @Override
- public void update(final Graphics g) {
- final Dimension d = getSize();
- offGraphics = new RasterPlotter(d.width, d.height, RasterPlotter.DrawMode.MODE_REPLACE, "FFFFFF");
- paintFrame(offGraphics);
- g.drawImage(offGraphics.getImage(), 0, 0, null);
- }
-
- @Override
- public void paint(final Graphics g) {
- if (offGraphics != null) {
- g.drawImage(offGraphics.getImage(), 0, 0, null);
- }
- }
-
- public void paintFrame(final RasterPlotter m) {
- RasterPlotter.demoPaint(m);
- final int y = (int) (System.currentTimeMillis() / 10 % 300);
- m.setColor(RasterPlotter.GREY);
- PrintTool.print(m, 0, y, 0, "Hello World", -1, 100);
- }
-
-}