From 1893661ee4aa8dd4428355188b104db59aded2bb Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 5 Oct 2022 14:38:59 +0200 Subject: [PATCH] removed/suppressed more warnings --- source/net/yacy/cora/util/Html2Image.java | 640 +++++++++--------- source/net/yacy/cora/util/Memory.java | 19 +- .../yacy/data/list/XMLBlacklistImporter.java | 29 +- source/net/yacy/gui/framework/Browser.java | 46 +- source/net/yacy/kelondro/util/FileUtils.java | 201 +++--- source/net/yacy/kelondro/util/OS.java | 272 ++++---- .../yacy/server/http/HTTPDProxyHandler.java | 230 +++---- source/net/yacy/visualization/DemoApplet.java | 75 -- 8 files changed, 723 insertions(+), 789 deletions(-) delete mode 100644 source/net/yacy/visualization/DemoApplet.java diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.java index baf8d76be..8e437a2fb 100644 --- a/source/net/yacy/cora/util/Html2Image.java +++ b/source/net/yacy/cora/util/Html2Image.java @@ -43,6 +43,10 @@ import javax.swing.text.html.HTMLDocument; import javax.swing.text.html.HTMLEditorKit; import javax.swing.text.html.ImageView; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.rendering.ImageType; +import org.apache.pdfbox.rendering.PDFRenderer; + import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; @@ -51,57 +55,53 @@ import net.yacy.document.ImageParser; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.OS; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.rendering.ImageType; -import org.apache.pdfbox.rendering.PDFRenderer; - /** * Convert html to an copy on disk-image in a other file format * currently (pdf and/or jpg) */ public class Html2Image { - + // Mac - /** - * Path to wkhtmltopdf executable on Mac OS when installed using - * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html. - * This can also be a path on Debian or another Gnu/Linux distribution. - */ - private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf"); - + /** + * Path to wkhtmltopdf executable on Mac OS when installed using + * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html. + * This can also be a path on Debian or another Gnu/Linux distribution. + */ + private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf"); + // to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip // the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/ - + private final static File convertMac1 = new File("/opt/local/bin/convert"); private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert"); - + /* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript The imagemagick policy at /etc should also be checked : if it contains a line such as it must be edited with rights="read" at minimum - */ + */ private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images private final static File convertDebian = new File("/usr/bin/convert"); - - /** - * Path to wkhtmltopdf executable on Windows, when installed with default - * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from - * https://wkhtmltopdf.org/downloads.html - */ - private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"); - - /** - * Path to wkhtmltopdf executable on Windows, when installed with default - * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from - * https://wkhtmltopdf.org/downloads.html - */ - private static final File WKHTMLTOPDF_WINDOWS_X86 = new File( - "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"); - - /** Command to use when wkhtmltopdf is included in the system Path */ - private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf"; - - /** Command to use when imagemagick convert is included in the system Path */ - private static final String CONVERT_COMMAND = "convert"; + + /** + * Path to wkhtmltopdf executable on Windows, when installed with default + * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from + * https://wkhtmltopdf.org/downloads.html + */ + private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"); + + /** + * Path to wkhtmltopdf executable on Windows, when installed with default + * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from + * https://wkhtmltopdf.org/downloads.html + */ + private static final File WKHTMLTOPDF_WINDOWS_X86 = new File( + "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"); + + /** Command to use when wkhtmltopdf is included in the system Path */ + private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf"; + + /** Command to use when imagemagick convert is included in the system Path */ + private static final String CONVERT_COMMAND = "convert"; private static boolean usexvfb = false; @@ -109,109 +109,111 @@ public class Html2Image { * @return when the wkhtmltopdf command is detected as available in the system */ public static boolean wkhtmltopdfAvailable() { - /* Check wkhtmltopdf common installation paths and system Path */ - return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath(); + /* Check wkhtmltopdf common installation paths and system Path */ + return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath(); } - + /** * @return a wkhtmltopdf executable file when one can be found, null otherwise */ private static File wkhtmltopdfExecutable() { - File executable = null; - if(OS.isWindows) { - if(WKHTMLTOPDF_WINDOWS.exists()) { - executable = WKHTMLTOPDF_WINDOWS; - } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) { - executable = WKHTMLTOPDF_WINDOWS_X86; - } - } else { - if(wkhtmltopdfMac.exists()) { - executable = wkhtmltopdfMac; - } else if(wkhtmltopdfDebian.exists()) { - executable = wkhtmltopdfDebian; - } - } - return executable; + File executable = null; + if(OS.isWindows) { + if(WKHTMLTOPDF_WINDOWS.exists()) { + executable = WKHTMLTOPDF_WINDOWS; + } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) { + executable = WKHTMLTOPDF_WINDOWS_X86; + } + } else { + if(wkhtmltopdfMac.exists()) { + executable = wkhtmltopdfMac; + } else if(wkhtmltopdfDebian.exists()) { + executable = wkhtmltopdfDebian; + } + } + return executable; } /** * @return true when wkhtmltopdf is available in system path */ - private static boolean wkhtmltopdfAvailableInPath() { - boolean available = false; - try { - final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V"); - available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; - } catch (final IOException e) { - ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path."); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); // preserve thread interrupted state - } - return available; - } - + private static boolean wkhtmltopdfAvailableInPath() { + boolean available = false; + try { + @SuppressWarnings("deprecation") + final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V"); + available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; + } catch (final IOException e) { + ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path."); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // preserve thread interrupted state + } + return available; + } + /** * @return a imagemagick convert executable file when one can be found, null otherwise */ private static File convertExecutable() { - File executable = null; - if(!OS.isWindows) { - if(convertMac1.exists()) { - executable = convertMac1; - } else if(convertMac2.exists()) { - executable = convertMac2; - } else if(convertDebian.exists()) { - executable = convertDebian; - } - } - return executable; + File executable = null; + if(!OS.isWindows) { + if(convertMac1.exists()) { + executable = convertMac1; + } else if(convertMac2.exists()) { + executable = convertMac2; + } else if(convertDebian.exists()) { + executable = convertDebian; + } + } + return executable; } - + /** * @return when the imagemagick convert command is detected as available in the system */ public static boolean convertAvailable() { - /* Check convert common installation paths and system Path */ + /* Check convert common installation paths and system Path */ return convertExecutable() != null || convertAvailableInPath(); } /** * @return when imagemagick convert is available in system path */ - private static boolean convertAvailableInPath() { - boolean available = false; - if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS - try { - final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version"); - available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; - } catch (final IOException e) { - ConcurrentLog.fine("Html2Image", "convert is not included in system path."); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); // preserve thread interrupted state - } - } - return available; - } - - /** - * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource. - * wKhtmltopdf may be called multiple times with various parameters flavors in - * case of failure. - * - * @param url the URL of a web resource to fetch, render and convert to - * a pdf file. Must not be null. - * @param proxy the eventual proxy address to use. Can be null. Must be of - * the form http://host:port; use YaCy here as proxy which is - * mostly http://localhost:8090 - * @param destination the destination PDF file that should be written. Must not - * be null. - * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf - * call termination. Beyond this limit the process is killed. - * @return true when the destination file was successfully written - */ + private static boolean convertAvailableInPath() { + boolean available = false; + if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS + try { + @SuppressWarnings("deprecation") + final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version"); + available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; + } catch (final IOException e) { + ConcurrentLog.fine("Html2Image", "convert is not included in system path."); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // preserve thread interrupted state + } + } + return available; + } + + /** + * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource. + * wKhtmltopdf may be called multiple times with various parameters flavors in + * case of failure. + * + * @param url the URL of a web resource to fetch, render and convert to + * a pdf file. Must not be null. + * @param proxy the eventual proxy address to use. Can be null. Must be of + * the form http://host:port; use YaCy here as proxy which is + * mostly http://localhost:8090 + * @param destination the destination PDF file that should be written. Must not + * be null. + * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf + * call termination. Beyond this limit the process is killed. + * @return true when the destination file was successfully written + */ public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) { boolean success = false; - for (boolean ignoreErrors: new boolean[]{false, true}) { + for (final boolean ignoreErrors: new boolean[]{false, true}) { success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds); if (success) break; if (!success && proxy != null) { @@ -227,49 +229,49 @@ public class Html2Image { } return success; } - - /** - * Run wkhtmltopdf in a separate process to fetch and render to PDF a web - * resource. - * - * @param url the URL of a web resource to fetch, render and convert to - * a pdf file. Must not be null. - * @param proxy the eventual proxy address to use. Can be null. - * @param destination the destination PDF file that should be written. Must not - * be null. - * @param userAgent TODO: implement - * @param acceptLanguage TODO: implement - * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors - * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf - * dedicated process termination. Beyond this limit the - * process is killed. - * @return true when the destination file was successfully written - */ - private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination, - final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) { - final String wkhtmltopdfCmd; - final File wkhtmltopdf = wkhtmltopdfExecutable(); - if(wkhtmltopdf != null) { - wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath(); - } else if(wkhtmltopdfAvailableInPath()) { - wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND; - } else { - ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!"); - return false; - } + + /** + * Run wkhtmltopdf in a separate process to fetch and render to PDF a web + * resource. + * + * @param url the URL of a web resource to fetch, render and convert to + * a pdf file. Must not be null. + * @param proxy the eventual proxy address to use. Can be null. + * @param destination the destination PDF file that should be written. Must not + * be null. + * @param userAgent TODO: implement + * @param acceptLanguage TODO: implement + * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors + * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf + * dedicated process termination. Beyond this limit the + * process is killed. + * @return true when the destination file was successfully written + */ + private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination, + final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) { + final String wkhtmltopdfCmd; + final File wkhtmltopdf = wkhtmltopdfExecutable(); + if(wkhtmltopdf != null) { + wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath(); + } else if(wkhtmltopdfAvailableInPath()) { + wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND; + } else { + ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!"); + return false; + } String commandline = - wkhtmltopdfCmd + " -q --title '" + url + "' " + - //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " + - //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") + - (proxy == null ? "" : "--proxy " + proxy + " ") + - (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it... - //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " + - "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " + - url + " " + destination.getAbsolutePath(); + wkhtmltopdfCmd + " -q --title '" + url + "' " + + //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " + + //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") + + (proxy == null ? "" : "--proxy " + proxy + " ") + + (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it... + //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " + + "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " + + url + " " + destination.getAbsolutePath(); try { - ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline); + ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline); if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) { - return true; + return true; } // if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers commandline = "xvfb-run -a " + commandline; @@ -280,46 +282,46 @@ public class Html2Image { } } - /** - * Run a wkhtmltopdf commandline in a separate process. - * - * @param proxy the eventual proxy address to use. Can be null. - * @param destination the destination PDF file that should be written. Must not - * be null. - * @param commandline the wkhtmltopdf command line to execute. Must not be null. - * @param maxSeconds the maximum time in seconds to wait for the process - * termination. Beyond this limit the process is killed. - * @return true when the destination file was successfully written - * @throws IOException when an unexpected error occurred - */ - private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) - throws IOException { - final Process p = Runtime.getRuntime().exec(commandline); - - try { - p.waitFor(maxSeconds, TimeUnit.SECONDS); - } catch (final InterruptedException e) { - p.destroyForcibly(); - ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline); - Thread.currentThread().interrupt(); // Keep the thread interrupted state - return false; - } - if(p.isAlive()) { - ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline); - p.destroyForcibly(); - return false; - } - if (p.exitValue() == 0 && destination.exists()) { - return true; - } - final List messages = OS.readStreams(p); - ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline); - for (final String message : messages) { - ConcurrentLog.warn("Html2Image", ">> " + message); - } - return false; - } - + /** + * Run a wkhtmltopdf commandline in a separate process. + * + * @param proxy the eventual proxy address to use. Can be null. + * @param destination the destination PDF file that should be written. Must not + * be null. + * @param commandline the wkhtmltopdf command line to execute. Must not be null. + * @param maxSeconds the maximum time in seconds to wait for the process + * termination. Beyond this limit the process is killed. + * @return true when the destination file was successfully written + * @throws IOException when an unexpected error occurred + */ + private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException { + @SuppressWarnings("deprecation") + final Process p = Runtime.getRuntime().exec(commandline); + + try { + p.waitFor(maxSeconds, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + p.destroyForcibly(); + ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline); + Thread.currentThread().interrupt(); // Keep the thread interrupted state + return false; + } + if(p.isAlive()) { + ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline); + p.destroyForcibly(); + return false; + } + if (p.exitValue() == 0 && destination.exists()) { + return true; + } + final List messages = OS.readStreams(p); + ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline); + for (final String message : messages) { + ConcurrentLog.warn("Html2Image", ">> " + message); + } + return false; + } + /** * Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75 * using internal pdf library or external command line tool on linux or mac @@ -332,34 +334,34 @@ public class Html2Image { * @return true when the ouput image file was successfully written. */ public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) { - /* Deduce the ouput image format from the file extension */ - String imageFormat = MultiProtocolURL.getFileExtension(image.getName()); - if(imageFormat.isEmpty()) { - /* Use JPEG as a default fallback */ - imageFormat = "jpg"; - } - String convertCmd = null; - final File convert = convertExecutable(); - if(convert != null) { - convertCmd = convert.getAbsolutePath(); - } else if(convertAvailableInPath()) { - convertCmd = CONVERT_COMMAND; - } else { - ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!"); - } + /* Deduce the ouput image format from the file extension */ + String imageFormat = MultiProtocolURL.getFileExtension(image.getName()); + if(imageFormat.isEmpty()) { + /* Use JPEG as a default fallback */ + imageFormat = "jpg"; + } + String convertCmd = null; + final File convert = convertExecutable(); + if(convert != null) { + convertCmd = convert.getAbsolutePath(); + } else if(convertAvailableInPath()) { + convertCmd = CONVERT_COMMAND; + } else { + ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!"); + } // convert pdf to jpg using internal pdfbox capability if (convertCmd == null) { try (final PDDocument pdoc = PDDocument.load(pdf);) { - - BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB); + + final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB); return ImageIO.write(bi, imageFormat, image); } catch (final IOException ex) { - ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox" - + (ex.getMessage() != null ? " : " + ex.getMessage() : "")); - return false; + ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox" + + (ex.getMessage() != null ? " : " + ex.getMessage() : "")); + return false; } } @@ -367,18 +369,18 @@ public class Html2Image { try { // i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg // note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf - String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath(); + final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath(); List message = OS.execSynchronous(command); if (image.exists()) return true; ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command); - for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m); - + for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m); + // another try for mac: use Image Events using AppleScript in osacript commands... // the following command overwrites a pdf with an png, so we must make a copy first if (!OS.isMacArchitecture) return false; - File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf"); + final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf"); org.apache.commons.io.FileUtils.copyFile(pdf, pngFile); - String[] commandx = {"osascript", + final String[] commandx = {"osascript", "-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"", "-e", "tell application \"Image Events\"", "-e", "set Img to open file ImgFile", @@ -386,15 +388,15 @@ public class Html2Image { "-e", "end tell"}; //ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx); message = OS.execSynchronous(commandx); - for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m); + for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m); // now we must read and convert this file to the target format with the target size 1024x1024 try { - File newPngFile = new File(pngFile.getAbsolutePath() + ".png"); + final File newPngFile = new File(pngFile.getAbsolutePath() + ".png"); pngFile.renameTo(newPngFile); final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile)); if(img == null) { - /* Should not happen. If so, ImageParser.parse() should already have logged about the error */ - return false; + /* Should not happen. If so, ImageParser.parse() should already have logged about the error */ + return false; } final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING); final MediaTracker mediaTracker = new MediaTracker(new Container()); @@ -406,24 +408,24 @@ public class Html2Image { ImageIO.write(bi, imageFormat, image); newPngFile.delete(); return image.exists(); - } catch (IOException e) { + } catch (final IOException e) { ConcurrentLog.logException(e); return false; } - } catch (IOException e) { + } catch (final IOException e) { e.printStackTrace(); return false; } } - + /** * render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported! * @param url * @param size - * @throws IOException + * @throws IOException */ public static void writeSwingImage(String url, Dimension size, File destination) throws IOException { - + // set up a pane for rendering final JEditorPane htmlPane = new JEditorPane(); htmlPane.setSize(size); @@ -434,7 +436,7 @@ public class Html2Image { @Override public Document createDefaultDocument() { - HTMLDocument doc = (HTMLDocument) super.createDefaultDocument(); + final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument(); doc.setAsynchronousLoadPriority(-1); return doc; } @@ -444,7 +446,7 @@ public class Html2Image { return new HTMLFactory() { @Override public View create(Element elem) { - View view = super.create(elem); + final View view = super.create(elem); if (view instanceof ImageView) { ((ImageView) view).setLoadsSynchronously(true); } @@ -460,23 +462,23 @@ public class Html2Image { public void propertyChange(PropertyChangeEvent evt) { } }); - + // load the page try { htmlPane.setPage(url); - } catch (IOException e) { + } catch (final IOException e) { e.printStackTrace(); } - + // render the page - Dimension prefSize = htmlPane.getPreferredSize(); - BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB); - Graphics graphics = img.getGraphics(); + final Dimension prefSize = htmlPane.getPreferredSize(); + final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB); + final Graphics graphics = img.getGraphics(); htmlPane.setSize(prefSize); htmlPane.paint(graphics); ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination); } - + /** * Test PDF or image snapshot generation for a given URL. * @param args main arguments list: @@ -491,88 +493,88 @@ public class Html2Image { * * */ - public static void main(String[] args) { - final String usageMessage = "Usage : java " + Html2Image.class.getName() - + " [wkhtmltopdf|swing]"; - int exitStatus = 0; - try { - if (args.length < 2) { - System.out.println("Missing required parameter(s)."); - System.out.println(usageMessage); - exitStatus = 1; - return; - } - final String targetPath = args[1]; - if (args.length < 3 || "wkhtmltopdf".equals(args[2])) { - if(Html2Image.wkhtmltopdfAvailable()) { - final File targetPdfFile; - if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) { - targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf"); - } else if(targetPath.endsWith(".pdf")) { - targetPdfFile = new File(targetPath); - } else { - System.out.println("Unsupported output format"); - System.out.println(usageMessage); - exitStatus = 1; - return; - } - if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent, - "en-us,en;q=0.5", targetPdfFile, 30)) { - if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) { - if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) { - ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile); - } else { - exitStatus = 1; - return; - } - } - } else { - exitStatus = 1; - return; - } - } else { - System.out.println("Unable to locate wkhtmltopdf executable on this system!"); - exitStatus = 1; - return; - } - } else if ("swing".equals(args[2])) { - if(targetPath.endsWith(".pdf")) { - System.out.println("Pdf output format is not supported with swing method."); - exitStatus = 1; - return; - } - if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) { - System.out.println("Unsupported output format"); - System.out.println(usageMessage); - exitStatus = 1; - return; - } - - try { - Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath)); - } catch (final IOException e) { - e.printStackTrace(); - exitStatus = 1; - return; - } - } else { - System.out.println("Unknown method : please specify either wkhtmltopdf or swing."); - exitStatus = 1; - return; - } - } finally { - /* Shutdown running threads */ - Domains.close(); - try { - HTTPClient.closeConnectionManager(); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); // restore interrupted state - } - ConcurrentLog.shutdown(); - if(exitStatus != 0) { - System.exit(exitStatus); - } - } - } - + public static void main(String[] args) { + final String usageMessage = "Usage : java " + Html2Image.class.getName() + + " [wkhtmltopdf|swing]"; + int exitStatus = 0; + try { + if (args.length < 2) { + System.out.println("Missing required parameter(s)."); + System.out.println(usageMessage); + exitStatus = 1; + return; + } + final String targetPath = args[1]; + if (args.length < 3 || "wkhtmltopdf".equals(args[2])) { + if(Html2Image.wkhtmltopdfAvailable()) { + final File targetPdfFile; + if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) { + targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf"); + } else if(targetPath.endsWith(".pdf")) { + targetPdfFile = new File(targetPath); + } else { + System.out.println("Unsupported output format"); + System.out.println(usageMessage); + exitStatus = 1; + return; + } + if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent, + "en-us,en;q=0.5", targetPdfFile, 30)) { + if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) { + if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) { + ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile); + } else { + exitStatus = 1; + return; + } + } + } else { + exitStatus = 1; + return; + } + } else { + System.out.println("Unable to locate wkhtmltopdf executable on this system!"); + exitStatus = 1; + return; + } + } else if ("swing".equals(args[2])) { + if(targetPath.endsWith(".pdf")) { + System.out.println("Pdf output format is not supported with swing method."); + exitStatus = 1; + return; + } + if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) { + System.out.println("Unsupported output format"); + System.out.println(usageMessage); + exitStatus = 1; + return; + } + + try { + Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath)); + } catch (final IOException e) { + e.printStackTrace(); + exitStatus = 1; + return; + } + } else { + System.out.println("Unknown method : please specify either wkhtmltopdf or swing."); + exitStatus = 1; + return; + } + } finally { + /* Shutdown running threads */ + Domains.close(); + try { + HTTPClient.closeConnectionManager(); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); // restore interrupted state + } + ConcurrentLog.shutdown(); + if(exitStatus != 0) { + System.exit(exitStatus); + } + } + } + } diff --git a/source/net/yacy/cora/util/Memory.java b/source/net/yacy/cora/util/Memory.java index 3a1334d91..eb47f0908 100644 --- a/source/net/yacy/cora/util/Memory.java +++ b/source/net/yacy/cora/util/Memory.java @@ -115,8 +115,9 @@ public class Memory { * @return the "recent cpu usage" for the whole operating environment; * a negative value if not available. */ + @SuppressWarnings("deprecation") public static double getSystemCpuLoad() { - com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); return operatingSystemMXBean.getSystemCpuLoad(); } @@ -136,13 +137,13 @@ public class Memory { * a negative value if not available. */ public static double getProcessCpuLoad() { - com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); return operatingSystemMXBean.getProcessCpuLoad(); } public static Map status() { - Runtime runtime = Runtime.getRuntime(); - Map status = new LinkedHashMap<>(); + final Runtime runtime = Runtime.getRuntime(); + final Map status = new LinkedHashMap<>(); status.put("service", "Peer"); status.put("assigned_memory", runtime.maxMemory()); status.put("used_memory", runtime.totalMemory() - runtime.freeMemory()); @@ -153,7 +154,7 @@ public class Memory { status.put("load_system_load_average", Memory.getSystemLoadAverage()); status.put("load_system_cpu_load", Memory.getSystemCpuLoad()); status.put("load_process_cpu_load", Memory.getProcessCpuLoad()); - YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer(); + final YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer(); status.put("server_threads", server == null ? 0 : server.getServerThreads()); return status; } @@ -163,7 +164,7 @@ public class Memory { * @return the number of deadlocked threads */ public static long deadlocks() { - long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); + final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); if (deadlockIDs == null) return 0; return deadlockIDs.length; } @@ -172,10 +173,10 @@ public class Memory { * write deadlocked threads as to the log as warning */ public static void logDeadlocks() { - long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); + final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); if (deadlockIDs == null) return; - ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true); - for (ThreadInfo ti : infos) { + final ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true); + for (final ThreadInfo ti : infos) { ConcurrentLog.warn("DEADLOCKREPORT", ti.toString()); } } diff --git a/source/net/yacy/data/list/XMLBlacklistImporter.java b/source/net/yacy/data/list/XMLBlacklistImporter.java index 6f72f091d..20155df3c 100644 --- a/source/net/yacy/data/list/XMLBlacklistImporter.java +++ b/source/net/yacy/data/list/XMLBlacklistImporter.java @@ -33,7 +33,6 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.XMLReader; import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; /** * This class provides methods to import blacklists from an XML file (see @@ -54,13 +53,15 @@ public class XMLBlacklistImporter extends DefaultHandler { * @throws java.io.IOException if input can't be read * @throws org.xml.sax.SAXException if XML can't be parsed */ + @SuppressWarnings("deprecation") public synchronized ListAccumulator parse(InputSource input) throws IOException, SAXException { - XMLReader reader = XMLReaderFactory.createXMLReader(); + @SuppressWarnings("deprecation") + final XMLReader reader = org.xml.sax.helpers.XMLReaderFactory.createXMLReader(); reader.setContentHandler(this); reader.parse(input); - return ba; + return this.ba; } /** @@ -104,7 +105,7 @@ public class XMLBlacklistImporter extends DefaultHandler { */ @Override public void startDocument() { - ba = new ListAccumulator(); + this.ba = new ListAccumulator(); } /** @@ -127,24 +128,24 @@ public class XMLBlacklistImporter extends DefaultHandler { public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) { if (qName.equalsIgnoreCase("list")) { - currentListName = attributes.getValue("name"); - ba.addList(currentListName); - + this.currentListName = attributes.getValue("name"); + this.ba.addList(this.currentListName); + int attributesLength = 0; if ((attributesLength = attributes.getLength()) > 1) { for (int i = 0; i < attributesLength; i++) { if (!attributes.getQName(i).equals("name")) { - ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i)); + this.ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i)); } } } } - + if (qName.equalsIgnoreCase("item")) { - lastText = new StringBuilder(); + this.lastText = new StringBuilder(); } - + } /** @@ -162,7 +163,7 @@ public class XMLBlacklistImporter extends DefaultHandler { @Override public void endElement(final String uri, final String localName, final String qName) throws SAXException { if (qName.equalsIgnoreCase("item")) { - ba.addEntryToCurrent(lastText.toString()); + this.ba.addEntryToCurrent(this.lastText.toString()); } } @@ -175,8 +176,8 @@ public class XMLBlacklistImporter extends DefaultHandler { */ @Override public void characters(char[] ch, int start, int length) throws SAXException { - if (lastText == null) lastText = new StringBuilder(); - lastText.append(ch, start, length); + if (this.lastText == null) this.lastText = new StringBuilder(); + this.lastText.append(ch, start, length); } } diff --git a/source/net/yacy/gui/framework/Browser.java b/source/net/yacy/gui/framework/Browser.java index 0b967c6ac..741a30047 100644 --- a/source/net/yacy/gui/framework/Browser.java +++ b/source/net/yacy/gui/framework/Browser.java @@ -37,7 +37,7 @@ import net.yacy.cora.util.ConcurrentLog; public class Browser { - // constants for system identification + // constants for system identification public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.* public static final int systemMacOSX = 1; // all Mac OS X public static final int systemUnix = 2; // all Unix/Linux type systems @@ -100,7 +100,7 @@ public class Browser { openBrowserClassic(url); } } - + public static void openBrowserClassic(final String url) { try { if (systemOS == systemMacOSX) { @@ -113,12 +113,12 @@ public class Browser { throw new RuntimeException("System unknown"); } } catch (final Throwable e) { - ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString()); + ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString()); } } private static void openBrowserMac(final String url) throws Exception { - Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""}); + final Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""}); p.waitFor(); if (p.exitValue() != 0) { throw new RuntimeException("Mac Exec Error: " + errorResponse(p)); @@ -131,12 +131,13 @@ public class Browser { * @throws Exception when an error occured */ private static void openDefaultUnixBrowser(final String url) throws Exception { - /* Use the freedesktop xdg-open to open url with the default browser. - * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/) - * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it - * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */ - String cmd = "xdg-open " + url; - Process p = Runtime.getRuntime().exec(cmd); + /* Use the freedesktop xdg-open to open url with the default browser. + * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/) + * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it + * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */ + final String cmd = "xdg-open " + url; + @SuppressWarnings("deprecation") + final Process p = Runtime.getRuntime().exec(cmd); p.waitFor(); if (p.exitValue() != 0) { throw new RuntimeException("Unix Exec Error/xdg-open: " + errorResponse(p)); @@ -152,7 +153,8 @@ public class Browser { cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\""; } //cmd = "cmd.exe /c start javascript:document.location='" + url + "'"; - Process p = Runtime.getRuntime().exec(cmd); + @SuppressWarnings("deprecation") + final Process p = Runtime.getRuntime().exec(cmd); p.waitFor(); if (p.exitValue() != 0) { throw new RuntimeException("EXEC ERROR: " + errorResponse(p)); @@ -183,15 +185,15 @@ public class Browser { * @param args */ public static void main(final String[] args) { - try { - if (args.length > 0 && "-u".equals(args[0])) { - openBrowser(args[1]); - } else { - System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]"); - } - } finally { - ConcurrentLog.shutdown(); - } - System.out.println("The End!"); - } + try { + if (args.length > 0 && "-u".equals(args[0])) { + openBrowser(args[1]); + } else { + System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]"); + } + } finally { + ConcurrentLog.shutdown(); + } + System.out.println("The End!"); + } } diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 711d0c71a..408ee459d 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -56,15 +56,14 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.storage.Files; -import net.yacy.cora.util.ConcurrentLog; - import org.apache.commons.lang.StringUtils; - import org.mozilla.intl.chardet.nsDetector; import org.mozilla.intl.chardet.nsPSMDetector; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.storage.Files; +import net.yacy.cora.util.ConcurrentLog; + public final class FileUtils { private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size @@ -98,7 +97,7 @@ public final class FileUtils { * @see #copy(File source, File dest) */ public static long copy(final InputStream source, final OutputStream dest, final long count) - throws IOException { + throws IOException { assert count < 0 || count > 0 : "precondition violated: count == " + count + " (nothing to copy)"; if ( count == 0 ) { // no bytes to copy @@ -106,15 +105,15 @@ public final class FileUtils { } final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; - int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE); + final int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE); int c; long total = 0; long remaining; if(count > 0) { - remaining = count; + remaining = count; } else { - remaining = Long.MAX_VALUE; + remaining = Long.MAX_VALUE; } while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) { dest.write(buffer, 0, c); @@ -158,7 +157,7 @@ public final class FileUtils { * @throws NullPointerException when a parameter is null */ public static int copy(final InputStream source, final Writer dest, final Charset inputCharset) - throws IOException { + throws IOException { final InputStreamReader reader = new InputStreamReader(source, inputCharset); return copy(reader, dest); } @@ -212,8 +211,8 @@ public final class FileUtils { // an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here //Log.logException(e); throw new IOException( - e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), - e); + e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), + e); } return count; } @@ -260,8 +259,8 @@ public final class FileUtils { fos.close(); } catch (final Exception e ) { ConcurrentLog.warn( - "FileUtils", - "cannot close FileOutputStream for " + dest + "! " + e.getMessage()); + "FileUtils", + "cannot close FileOutputStream for " + dest + "! " + e.getMessage()); } } } @@ -282,17 +281,17 @@ public final class FileUtils { * @see #copy(File source, File dest) */ public static void copyRange(final File source, final OutputStream dest, final int start) - throws IOException { + throws IOException { InputStream fis = null; try { fis = new FileInputStream(source); final long skipped = fis.skip(start); if ( skipped != start ) { throw new IllegalStateException("Unable to skip '" - + start - + "' bytes. Only '" - + skipped - + "' bytes skipped."); + + start + + "' bytes. Only '" + + skipped + + "' bytes skipped."); } copy(fis, dest, -1); } finally { @@ -300,7 +299,7 @@ public final class FileUtils { try { fis.close(); } catch (final Exception e ) { - ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); + ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); } } } @@ -328,7 +327,7 @@ public final class FileUtils { try { fis.close(); } catch (final Exception e ) { - ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); + ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); } } } @@ -368,17 +367,17 @@ public final class FileUtils { * @throws NullPointerException when source parameter is null */ public static byte[] read(final InputStream source) throws IOException { - byte[] content; - try { - content = read(source, -1); - } finally { - /* source input stream must be closed here in all cases */ - try { - source.close(); - } catch(IOException ignoredException) { - } - } - return content; + byte[] content; + try { + content = read(source, -1); + } finally { + /* source input stream must be closed here in all cases */ + try { + source.close(); + } catch(final IOException ignoredException) { + } + } + return content; } /** @@ -392,7 +391,7 @@ public final class FileUtils { */ public static byte[] read(final InputStream source, final int count) throws IOException { if(count == 0) { - return new byte[0]; + return new byte[0]; } final ByteArrayOutputStream baos = new ByteArrayOutputStream(512); copy(source, baos, count); @@ -414,7 +413,7 @@ public final class FileUtils { try { fis.close(); } catch (final Exception e ) { - ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); + ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); } } fis = null; @@ -470,17 +469,17 @@ public final class FileUtils { return source; } - /** - * Generate a set of strings matching each line of the given file. Lines are - * lower cased and any eventual surrounding space characters are removed. Empty - * lines and lines starting with the '#' character are ignored. - * - * @param file - * a file to load - * @return a set of strings eventually empty - */ + /** + * Generate a set of strings matching each line of the given file. Lines are + * lower cased and any eventual surrounding space characters are removed. Empty + * lines and lines starting with the '#' character are ignored. + * + * @param file + * a file to load + * @return a set of strings eventually empty + */ public static HashSet loadList(final File file) { - final HashSet set = new HashSet(); + final HashSet set = new HashSet<>(); BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream(file))); @@ -497,7 +496,7 @@ public final class FileUtils { try { br.close(); } catch (final Exception e ) { - ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file); + ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file); } } } @@ -516,10 +515,10 @@ public final class FileUtils { } public static ConcurrentHashMap loadMapB(final File f) { - ConcurrentHashMap m = loadMap(f); + final ConcurrentHashMap m = loadMap(f); if (m == null) return null; - ConcurrentHashMap mb = new ConcurrentHashMap(); - for (Map.Entry e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue())); + final ConcurrentHashMap mb = new ConcurrentHashMap<>(); + for (final Map.Entry e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue())); return mb; } @@ -529,7 +528,7 @@ public final class FileUtils { private final static String[] unescaped_strings_out = {"\\", "\n", "="}; public static void saveMap(final File file, final Map props, final String comment) { - boolean err = false; + boolean err = false; PrintWriter pw = null; final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000)); try { @@ -550,10 +549,7 @@ public final class FileUtils { pw.println(key + "=" + value); } pw.println("# EOF"); - } catch (final FileNotFoundException e ) { - ConcurrentLog.warn("FileUtils", e.getMessage(), e); - err = true; - } catch (final UnsupportedEncodingException e ) { + } catch (final FileNotFoundException | UnsupportedEncodingException e ) { ConcurrentLog.warn("FileUtils", e.getMessage(), e); err = true; } finally { @@ -568,10 +564,10 @@ public final class FileUtils { // ignore } } - + public static void saveMapB(final File file, final Map props, final String comment) { - HashMap m = new HashMap(); - for (Map.Entry e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue())); + final HashMap m = new HashMap<>(); + for (final Map.Entry e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue())); saveMap(file, m, comment); } @@ -582,7 +578,7 @@ public final class FileUtils { public static ConcurrentHashMap table(final Iterator li) { String line; - final ConcurrentHashMap props = new ConcurrentHashMap(); + final ConcurrentHashMap props = new ConcurrentHashMap<>(); while ( li.hasNext() ) { int pos = 0; line = li.next().trim(); @@ -594,8 +590,8 @@ public final class FileUtils { pos = line.indexOf('=', pos + 1); } while ( pos > 0 && line.charAt(pos - 1) == '\\' ); if ( pos > 0 ) try { - String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out); - String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out); + final String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out); + final String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out); //System.out.println("key = " + key + ", value = " + value); props.put(key, value); } catch (final IndexOutOfBoundsException e) { @@ -606,7 +602,7 @@ public final class FileUtils { } public static Map table(final byte[] a) { - if (a == null) return new ConcurrentHashMap(); + if (a == null) return new ConcurrentHashMap<>(); //System.out.println("***TABLE: a.size = " + a.length); return table(strings(a)); } @@ -627,7 +623,7 @@ public final class FileUtils { */ public static ArrayList getListArray(final File listFile) { String line; - final ArrayList list = new ArrayList(); + final ArrayList list = new ArrayList<>(); BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile), StandardCharsets.UTF_8)); @@ -642,7 +638,7 @@ public final class FileUtils { try { br.close(); } catch (final Exception e ) { - ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile); + ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile); } } } @@ -711,7 +707,7 @@ public final class FileUtils { try { br.close(); } catch (final Exception e ) { - ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile); + ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile); } } } @@ -760,7 +756,7 @@ public final class FileUtils { * @return array of file names */ public static List getDirListing(final File dir, final String filter) { - final List ret = new LinkedList(); + final List ret = new LinkedList<>(); File[] fileList; if ( dir != null ) { if ( !dir.exists() ) { @@ -781,39 +777,39 @@ public final class FileUtils { public static ArrayList getDirsRecursive(final File dir, final String notdir) { return getDirsRecursive(dir, notdir, true); } - + /** * @param sourceDir source directory. Must be not null. * @param notdir name of dir to exlcude. Can be null * @param fileNameFilter filter to apply on file names. Can be null. - * @return list of all files passing fileFilter under sourceDir including sub directories + * @return list of all files passing fileFilter under sourceDir including sub directories */ public static List getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) { - List dirList = getDirsRecursive(sourceDir, - notdir); - dirList.add(sourceDir); - List files = new ArrayList<>(); - for (final File dir : dirList) { - Collections.addAll(files, dir.listFiles(fileNameFilter)); - } - return files; + final List dirList = getDirsRecursive(sourceDir, + notdir); + dirList.add(sourceDir); + final List files = new ArrayList<>(); + for (final File dir : dirList) { + Collections.addAll(files, dir.listFiles(fileNameFilter)); + } + return files; } /** * Returns a List of all dirs and subdirs as File Objects Warning: untested */ private static ArrayList getDirsRecursive( - final File dir, - final String notdir, - final boolean excludeDotfiles) { + final File dir, + final String notdir, + final boolean excludeDotfiles) { final File[] dirList = dir.listFiles(); - final ArrayList resultList = new ArrayList(); + final ArrayList resultList = new ArrayList<>(); ArrayList recursive; Iterator iter; for ( int i = 0; i < dirList.length; i++ ) { if ( dirList[i].isDirectory() - && (!excludeDotfiles || !dirList[i].getName().startsWith(".")) - && !dirList[i].getName().equals(notdir) ) { + && (!excludeDotfiles || !dirList[i].getName().startsWith(".")) + && !dirList[i].getName().equals(notdir) ) { resultList.add(dirList[i]); recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles); iter = recursive.iterator(); @@ -927,12 +923,12 @@ public final class FileUtils { // create the temp file final File tempFile = - File.createTempFile( - parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName), - (!fileExt.isEmpty()) ? "." + fileExt : fileExt); + File.createTempFile( + parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName), + (!fileExt.isEmpty()) ? "." + fileExt : fileExt); return tempFile; } - + /** * delete files and directories if a directory is not empty, delete also everything inside because * deletion sometimes fails on windows, there is also a windows exec included @@ -973,7 +969,7 @@ public final class FileUtils { break; } } - */ + */ if ( path.exists() ) { path.deleteOnExit(); String p = ""; @@ -986,6 +982,7 @@ public final class FileUtils { // deleting files on windows sometimes does not work with java try { final String command = "cmd /C del /F /Q \"" + p + "\""; + @SuppressWarnings("deprecation") final Process r = Runtime.getRuntime().exec(command); if ( r == null ) { ConcurrentLog.severe("FileUtils", "cannot execute command: " + command); @@ -1002,7 +999,7 @@ public final class FileUtils { } } } - + /** * Checks if a certain file is in a given directory. * @param file the file to check @@ -1010,9 +1007,9 @@ public final class FileUtils { * @return true if file is contained in directory */ public static boolean isInDirectory(final File file, final File directory) { - + boolean inDirectory; - + try { inDirectory = ( directory != null @@ -1024,10 +1021,10 @@ public final class FileUtils { } catch (final IOException e) { inDirectory = false; } - + return inDirectory; } - + /** * Auto-detect the charset of content in a stream. * Used code from http://jchardet.sourceforge.net/. @@ -1040,28 +1037,28 @@ public final class FileUtils { public static List detectCharset(final InputStream inStream) throws IOException { // auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.html List result; - nsDetector det = new nsDetector(nsPSMDetector.ALL); - byte[] buf = new byte[1024] ; + final nsDetector det = new nsDetector(nsPSMDetector.ALL); + final byte[] buf = new byte[1024] ; int len; boolean done = false ; boolean isAscii = true ; while ((len = inStream.read(buf,0,buf.length)) != -1) { if (isAscii) { - isAscii = det.isAscii(buf,len); + isAscii = det.isAscii(buf,len); } if (!isAscii && !done) { - done = det.DoIt(buf,len, false); + done = det.DoIt(buf,len, false); } } det.DataEnd(); result = new ArrayList<>(); if (isAscii) { result.add(StandardCharsets.US_ASCII.name()); } else { - for (String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch" + for (final String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch" } return result; } - + /** * Because the checking of very large files for their charset may take some time, we do this concurrently in this method * This method does not return anything but it logs an info line if the charset is a good choice @@ -1071,19 +1068,19 @@ public final class FileUtils { * @param concurrent if this shall run concurrently */ public static void checkCharset(final File file, final String givenCharset, final boolean concurrent) { - Thread t = new Thread("FileUtils.checkCharset") { + final Thread t = new Thread("FileUtils.checkCharset") { @Override public void run() { - try (final FileInputStream fileStream = new FileInputStream(file); - final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources - List charsets = FileUtils.detectCharset(imp); + try (final FileInputStream fileStream = new FileInputStream(file); + final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources + final List charsets = FileUtils.detectCharset(imp); if (charsets.contains(givenCharset)) { ConcurrentLog.info("checkCharset", "appropriate charset '" + givenCharset + "' for import of " + file + ", is part one detected " + charsets); } else { ConcurrentLog.warn("checkCharset", "possibly wrong charset '" + givenCharset + "' for import of " + file + ", use one of " + charsets); } - } catch (IOException e) {} - + } catch (final IOException e) {} + } }; if (concurrent) t.start(); else t.run(); diff --git a/source/net/yacy/kelondro/util/OS.java b/source/net/yacy/kelondro/util/OS.java index beca081a1..abf7d1695 100644 --- a/source/net/yacy/kelondro/util/OS.java +++ b/source/net/yacy/kelondro/util/OS.java @@ -41,7 +41,7 @@ import net.yacy.server.serverCore; public final class OS { - // constants for system identification + // constants for system identification public enum System { MacOSC, // 'classic' Mac OS 7.6.1/8.*/9.* MacOSX, // all Mac OS X @@ -50,125 +50,129 @@ public final class OS { Unknown; // any other system } - // constants for file type identification (Mac only) - public static final String blankTypeString = "____"; - - // system-identification statics - private static final System systemOS; - public static final boolean isMacArchitecture; - private static final boolean isUnixFS; - public static final boolean canExecUnix; - public static final boolean isWindows; - public static final boolean isWin32; - - // calculated system constants - public static int maxPathLength = 65535; - - // Macintosh-specific statics - public static final Map macFSTypeCache = new HashMap(); - public static final Map macFSCreatorCache = new HashMap(); - - // static initialization - static { - // check operation system type - final Properties sysprop = java.lang.System.getProperties(); - final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT); - if (sysname.startsWith("mac os x")) systemOS = System.MacOSX; - else if (sysname.startsWith("mac os")) systemOS = System.MacOSC; - else if (sysname.startsWith("windows")) systemOS = System.Windows; - else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix; - else systemOS = System.Unknown; - - isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX)); - isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix)); - canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows)))); - isWindows = (systemOS == System.Windows); - isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86")); - - // set up maximum path length according to system - if (isWindows) maxPathLength = 255; else maxPathLength = 65535; - } - - public static String infoString() { - String s = "System="; - if (systemOS == System.Unknown) s += "unknown"; - else if (systemOS == System.MacOSC) s += "Mac OS Classic"; - else if (systemOS == System.MacOSX) s += "Mac OS X"; - else if (systemOS == System.Unix) s += "Unix/Linux"; - else if (systemOS == System.Windows) s += "Windows"; - else s += "unknown"; - if (isMacArchitecture) s += ", Mac System Architecture"; - if (isUnixFS) s += ", has Unix-like File System"; - if (canExecUnix) s += ", can execute Unix-Shell Commands"; - return s; - } - - /** generates a 2-character string containing information about the OS-type*/ - public static String infoKey() { - String s = ""; - if (systemOS == System.Unknown) s += "o"; - else if (systemOS == System.MacOSC) s += "c"; - else if (systemOS == System.MacOSX) s += "x"; - else if (systemOS == System.Unix) s += "u"; - else if (systemOS == System.Windows) s += "w"; - else s += "o"; - if (isMacArchitecture) s += "m"; - if (isUnixFS) s += "f"; - if (canExecUnix) s += "e"; - return s; - } - - public static void deployScript(final File scriptFile, final String theScript) throws IOException { - FileUtils.copy(UTF8.getBytes(theScript), scriptFile); - if(!isWindows){ // set executable - try { - Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor(); - } catch (final InterruptedException e) { - ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e); - throw new IOException(e.getMessage()); - } - } - } + // constants for file type identification (Mac only) + public static final String blankTypeString = "____"; + + // system-identification statics + private static final System systemOS; + public static final boolean isMacArchitecture; + private static final boolean isUnixFS; + public static final boolean canExecUnix; + public static final boolean isWindows; + public static final boolean isWin32; + + // calculated system constants + public static int maxPathLength = 65535; + + // Macintosh-specific statics + public static final Map macFSTypeCache = new HashMap<>(); + public static final Map macFSCreatorCache = new HashMap<>(); + + // static initialization + static { + // check operation system type + final Properties sysprop = java.lang.System.getProperties(); + final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT); + if (sysname.startsWith("mac os x")) systemOS = System.MacOSX; + else if (sysname.startsWith("mac os")) systemOS = System.MacOSC; + else if (sysname.startsWith("windows")) systemOS = System.Windows; + else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix; + else systemOS = System.Unknown; + + isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX)); + isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix)); + canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows)))); + isWindows = (systemOS == System.Windows); + isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86")); + + // set up maximum path length according to system + if (isWindows) maxPathLength = 255; else maxPathLength = 65535; + } + + public static String infoString() { + String s = "System="; + if (systemOS == System.Unknown) s += "unknown"; + else if (systemOS == System.MacOSC) s += "Mac OS Classic"; + else if (systemOS == System.MacOSX) s += "Mac OS X"; + else if (systemOS == System.Unix) s += "Unix/Linux"; + else if (systemOS == System.Windows) s += "Windows"; + else s += "unknown"; + if (isMacArchitecture) s += ", Mac System Architecture"; + if (isUnixFS) s += ", has Unix-like File System"; + if (canExecUnix) s += ", can execute Unix-Shell Commands"; + return s; + } + + /** generates a 2-character string containing information about the OS-type*/ + public static String infoKey() { + String s = ""; + if (systemOS == System.Unknown) s += "o"; + else if (systemOS == System.MacOSC) s += "c"; + else if (systemOS == System.MacOSX) s += "x"; + else if (systemOS == System.Unix) s += "u"; + else if (systemOS == System.Windows) s += "w"; + else s += "o"; + if (isMacArchitecture) s += "m"; + if (isUnixFS) s += "f"; + if (canExecUnix) s += "e"; + return s; + } - /** - * use a hack to get the current process PID - * @return the PID of the current java process or -1 if the PID cannot be obtained - */ - public static int getPID() { + @SuppressWarnings("deprecation") + public static void deployScript(final File scriptFile, final String theScript) throws IOException { + FileUtils.copy(UTF8.getBytes(theScript), scriptFile); + if(!isWindows){ // set executable + try { + Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor(); + } catch (final InterruptedException e) { + ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e); + throw new IOException(e.getMessage()); + } + } + } + + /** + * use a hack to get the current process PID + * @return the PID of the current java process or -1 if the PID cannot be obtained + */ + public static int getPID() { final String pids = ManagementFactory.getRuntimeMXBean().getName(); final int p = pids.indexOf('@'); return p >= 0 ? NumberTools.parseIntDecSubstring(pids, 0, p) : -1; - } - - public static void execAsynchronous(final File scriptFile) throws IOException { - // runs a script as separate thread - String starterFileExtension = null; - String script = null; - if(isWindows){ - starterFileExtension = ".starter.bat"; - // use /K to debug, /C for release - script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\""; - } else { // unix/linux - starterFileExtension = ".starter.sh"; - script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING; - } - final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension); - deployScript(starterFile, script); - try { - Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor(); - } catch (final InterruptedException e) { - throw new IOException(e.getMessage()); - } - FileUtils.deletedelete(starterFile); - } + } + @SuppressWarnings("deprecation") + public static void execAsynchronous(final File scriptFile) throws IOException { + // runs a script as separate thread + String starterFileExtension = null; + String script = null; + if(isWindows){ + starterFileExtension = ".starter.bat"; + // use /K to debug, /C for release + script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\""; + } else { // unix/linux + starterFileExtension = ".starter.sh"; + script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING; + } + final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension); + deployScript(starterFile, script); + try { + Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor(); + } catch (final InterruptedException e) { + throw new IOException(e.getMessage()); + } + FileUtils.deletedelete(starterFile); + } + + @SuppressWarnings("deprecation") public static List execSynchronous(final String command) throws IOException { // runs a unix/linux command and returns output as Vector of Strings // this method blocks until the command is executed final Process p = Runtime.getRuntime().exec(command); return readStreams(p); } - + + @SuppressWarnings("deprecation") public static List execSynchronous(final String[] command) throws IOException { // runs a unix/linux command and returns output as Vector of Strings // this method blocks until the command is executed @@ -184,40 +188,40 @@ public final class OS { */ public static List readStreams(final Process p) throws IOException { String line; - final List output = new ArrayList<>(); - - if(p == null) { - return output; - } - - try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream()); - final BufferedReader in = new BufferedReader(streamReader);) { - while ((line = in.readLine()) != null) { - output.add(line); - } - } - - try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream()); - final BufferedReader in = new BufferedReader(streamReader);) { - while ((line = in.readLine()) != null) { - output.add(line); - } - } + final List output = new ArrayList<>(); + + if(p == null) { + return output; + } + + try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream()); + final BufferedReader in = new BufferedReader(streamReader);) { + while ((line = in.readLine()) != null) { + output.add(line); + } + } + + try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream()); + final BufferedReader in = new BufferedReader(streamReader);) { + while ((line = in.readLine()) != null) { + output.add(line); + } + } return output; } - public static void main(final String[] args) { + public static void main(final String[] args) { try { - List v = execSynchronous("/usr/local/bin/wkhtmltoimage"); - for (String r: v) java.lang.System.out.println(r); - } catch (IOException e) { + final List v = execSynchronous("/usr/local/bin/wkhtmltoimage"); + for (final String r: v) java.lang.System.out.println(r); + } catch (final IOException e) { } /* if (args[0].equals("-m")) { java.lang.System.out.println("Maximum possible memory: " + Integer.toString(getWin32MaxHeap()) + "m"); } - */ - } + */ + } } diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java index 744c4515c..e300ec9e2 100644 --- a/source/net/yacy/server/http/HTTPDProxyHandler.java +++ b/source/net/yacy/server/http/HTTPDProxyHandler.java @@ -37,7 +37,7 @@ support gzip-ed encoding. We also do not support unrealistic 'expires' values that would force a cache to be flushed immediately pragma non-cache attributes are supported -*/ + */ package net.yacy.server.http; @@ -67,6 +67,7 @@ import java.util.logging.FileHandler; import java.util.logging.Level; import java.util.logging.LogManager; import java.util.logging.Logger; + import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -89,6 +90,7 @@ import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; +@SuppressWarnings("deprecation") public final class HTTPDProxyHandler { @@ -108,7 +110,7 @@ public final class HTTPDProxyHandler { // creating a logger private static final ConcurrentLog log = new ConcurrentLog("PROXY"); - /** + /** * Do logging configuration for special proxy access log file */ static { @@ -117,77 +119,77 @@ public final class HTTPDProxyHandler { sb = Switchboard.getSwitchboard(); if (sb != null) { - // set timeout - timeout = sb.getConfigInt("proxy.clientTimeout", 60000); + // set timeout + timeout = sb.getConfigInt("proxy.clientTimeout", 60000); - // do logger initialization - try { - log.info("Configuring proxy access logging ..."); - - // getting the logging manager - final LogManager manager = LogManager.getLogManager(); - final String className = HTTPDProxyHandler.class.getName(); - - // determining if proxy access logging is enabled - final String enabled = manager.getProperty(className + ".logging.enabled"); - if ("true".equalsIgnoreCase(enabled)) { - - // reading out some needed configuration properties - int limit = 1024*1024, count = 20; - String pattern = manager.getProperty(className + ".logging.FileHandler.pattern"); - if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log"; - // make pattern absolute - if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath(); - - final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit"); - if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {} - - final String countStr = manager.getProperty(className + ".logging.FileHandler.count"); - if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {} - - // creating the proxy access logger - final Logger proxyLogger = Logger.getLogger("PROXY.access"); - proxyLogger.setUseParentHandlers(false); - proxyLogger.setLevel(Level.FINEST); - - final FileHandler txtLog = new FileHandler(pattern, limit, count, true); - txtLog.setFormatter(new ProxyLogFormatter()); - txtLog.setLevel(Level.FINEST); - proxyLogger.addHandler(txtLog); - - log.info("Proxy access logging configuration done." + - "\n\tFilename: " + pattern + - "\n\tLimit: " + limitStr + - "\n\tCount: " + countStr); - } else { - log.info("Proxy access logging is deactivated."); + // do logger initialization + try { + log.info("Configuring proxy access logging ..."); + + // getting the logging manager + final LogManager manager = LogManager.getLogManager(); + final String className = HTTPDProxyHandler.class.getName(); + + // determining if proxy access logging is enabled + final String enabled = manager.getProperty(className + ".logging.enabled"); + if ("true".equalsIgnoreCase(enabled)) { + + // reading out some needed configuration properties + int limit = 1024*1024, count = 20; + String pattern = manager.getProperty(className + ".logging.FileHandler.pattern"); + if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log"; + // make pattern absolute + if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath(); + + final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit"); + if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {} + + final String countStr = manager.getProperty(className + ".logging.FileHandler.count"); + if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {} + + // creating the proxy access logger + final Logger proxyLogger = Logger.getLogger("PROXY.access"); + proxyLogger.setUseParentHandlers(false); + proxyLogger.setLevel(Level.FINEST); + + final FileHandler txtLog = new FileHandler(pattern, limit, count, true); + txtLog.setFormatter(new ProxyLogFormatter()); + txtLog.setLevel(Level.FINEST); + proxyLogger.addHandler(txtLog); + + log.info("Proxy access logging configuration done." + + "\n\tFilename: " + pattern + + "\n\tLimit: " + limitStr + + "\n\tCount: " + countStr); + } else { + log.info("Proxy access logging is deactivated."); + } + } catch (final Exception e) { + log.severe("Unable to configure proxy access logging.",e); } - } catch (final Exception e) { - log.severe("Unable to configure proxy access logging.",e); - } - // load the yellow-list - final String f = sb.getConfig("proxyYellowList", null); - if (f != null) { - yellowList = FileUtils.loadList(new File(f)); - log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries"); - } else { - yellowList = null; - } + // load the yellow-list + final String f = sb.getConfig("proxyYellowList", null); + if (f != null) { + yellowList = FileUtils.loadList(new File(f)); + log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries"); + } else { + yellowList = null; + } - final String redirectorPath = sb.getConfig("externalRedirector", ""); - if (redirectorPath.length() > 0 && !redirectorEnabled) { - try { - redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null); - redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream()); - redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream())); - redirectorEnabled=true; - } catch (final IOException e) { - System.out.println("redirector not Found"); + final String redirectorPath = sb.getConfig("externalRedirector", ""); + if (redirectorPath.length() > 0 && !redirectorEnabled) { + try { + redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null); + redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream()); + redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream())); + redirectorEnabled=true; + } catch (final IOException e) { + System.out.println("redirector not Found"); + } } - } } else { - yellowList = null; + yellowList = null; } } @@ -342,25 +344,25 @@ public final class HTTPDProxyHandler { // case 1 and case 3 if (cachedResponseHeader == null) { if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web"); - fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent); + fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent); } else { - final Request request = new Request( - null, + final Request request = new Request( + null, url, requestHeader.referer() == null ? null : requestHeader.referer().hash(), - "", - cachedResponseHeader.lastModified(), - sb.crawler.defaultProxyProfile.handle(), - 0, - sb.crawler.defaultProxyProfile.timezoneOffset()); + "", + cachedResponseHeader.lastModified(), + sb.crawler.defaultProxyProfile.handle(), + 0, + sb.crawler.defaultProxyProfile.timezoneOffset()); final Response response = new Response( - request, + request, requestHeader, cachedResponseHeader, sb.crawler.defaultProxyProfile, true, null - ); + ); final byte[] cacheContent = Cache.getContent(url.hash()); if (cacheContent != null && response.isFreshForProxy()) { if (log.isFinest()) log.finest(reqID + " fulfill request from cache"); @@ -433,9 +435,9 @@ public final class HTTPDProxyHandler { // remove yacy-subdomain-path, when accessing /env if ((yAddress != null) - && (remotePath.startsWith("/env")) - && (yAddress.indexOf('/') != -1) - ) yAddress = yAddress.substring(0, yAddress.indexOf('/')); + && (remotePath.startsWith("/env")) + && (yAddress.indexOf('/') != -1) + ) yAddress = yAddress.substring(0, yAddress.indexOf('/')); modifyProxyHeaders(requestHeader, httpVer); @@ -448,17 +450,17 @@ public final class HTTPDProxyHandler { try (final HTTPClient client = new HTTPClient(agent, timeout)) { client.setHeader(requestHeader.entrySet()); client.setRedirecting(false); - client.GET(getUrl, false); + client.GET(getUrl, false); if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine()); - int statusCode = client.getHttpResponse().getStatusLine().getStatusCode(); + final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode(); final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders()); // determine if it's an internal error of the httpc if (responseHeader.isEmpty()) { - throw new Exception(client.getHttpResponse().getStatusLine().toString()); + throw new Exception(client.getHttpResponse().getStatusLine().toString()); } - ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond); + final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond); // the cache does either not exist or is (supposed to be) stale long sizeBeforeDelete = -1; @@ -475,19 +477,19 @@ public final class HTTPDProxyHandler { // reserver cache entry final Request request = new Request( - null, + null, url, requestHeader.referer() == null ? null : requestHeader.referer().hash(), - "", - responseHeader.lastModified(), - sb.crawler.defaultProxyProfile.handle(), - 0, - sb.crawler.defaultProxyProfile.timezoneOffset()); + "", + responseHeader.lastModified(), + sb.crawler.defaultProxyProfile.handle(), + 0, + sb.crawler.defaultProxyProfile.timezoneOffset()); // handle incoming cookies handleIncomingCookies(responseHeader, host, ip); -// prepareResponseHeader(responseHeader, res.getHttpVer()); + // prepareResponseHeader(responseHeader, res.getHttpVer()); prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString()); // sending the respond header back to the client @@ -506,7 +508,7 @@ public final class HTTPDProxyHandler { if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) { - OutputStream outStream = chunkedOut != null ? chunkedOut : respond; + final OutputStream outStream = chunkedOut != null ? chunkedOut : respond; final Response response = new Response( request, requestHeader, @@ -514,7 +516,7 @@ public final class HTTPDProxyHandler { sb.crawler.defaultProxyProfile, true, null - ); + ); final String storeError = response.shallStoreCacheForProxy(); final boolean storeHTCache = response.profile().storeHTCache(); final String supportError = TextParser.supports(response.url(), response.getMimeType()); @@ -530,14 +532,14 @@ public final class HTTPDProxyHandler { * c) the content should be indexed */ ((storeHTCache) || (supportError != null)) - ) { + ) { // we don't write actually into a file, only to RAM, and schedule writing the file. -// int l = res.getResponseHeader().size(); - final int l = responseHeader.size(); + // int l = res.getResponseHeader().size(); + final int l = responseHeader.size(); final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l); final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream}); -// FileUtils.copy(res.getDataAsStream(), toClientAndMemory); + // FileUtils.copy(res.getDataAsStream(), toClientAndMemory); client.writeTo(toClientAndMemory); // cached bytes byte[] cacheArray; @@ -627,7 +629,7 @@ public final class HTTPDProxyHandler { final ResponseHeader cachedResponseHeader, final byte[] cacheEntry, OutputStream respond - ) throws IOException { + ) throws IOException { final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); final String clienthttpVer; // the http version of the client connection @@ -721,12 +723,12 @@ public final class HTTPDProxyHandler { requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader)); } - // only gzip-encoding is supported, remove other encodings (e. g. deflate) + // only gzip-encoding is supported, remove other encodings (e. g. deflate) if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) { requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP); - } else { + } else { requestHeader.put(HeaderFramework.ACCEPT_ENCODING, ""); - } + } addXForwardedForHeader(conProp, requestHeader); } @@ -782,7 +784,7 @@ public final class HTTPDProxyHandler { if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) { forceConnectionClose(conProp); } else { - chunkedOut = new ChunkedOutputStream(respond); + chunkedOut = new ChunkedOutputStream(respond); } responseHeader.remove(HeaderFramework.CONTENT_LENGTH); } @@ -933,12 +935,12 @@ public final class HTTPDProxyHandler { errorMessage = "IP address of the destination host could not be determined"; } } else if ((exceptionMsg != null) && - ( - (exceptionMsg.indexOf("socket write error",0)>=0) || - (exceptionMsg.indexOf("Read timed out",0) >= 0) || - (exceptionMsg.indexOf("Broken pipe",0) >= 0) || - (exceptionMsg.indexOf("server has closed connection",0) >= 0) - )) { + ( + (exceptionMsg.indexOf("socket write error",0)>=0) || + (exceptionMsg.indexOf("Read timed out",0) >= 0) || + (exceptionMsg.indexOf("Broken pipe",0) >= 0) || + (exceptionMsg.indexOf("server has closed connection",0) >= 0) + )) { errorMessage = exceptionMsg; ConcurrentLog.logException(e); } else { @@ -983,7 +985,7 @@ public final class HTTPDProxyHandler { final serverObjects detailedErrorMsgMap = new serverObjects(); // generic toplevel domains - final HashSet topLevelDomains = new HashSet(Arrays.asList(new String[]{ + final HashSet topLevelDomains = new HashSet<>(Arrays.asList(new String[]{ "aero", // Fluggesellschaften/Luftfahrt "arpa", // Einrichtung des ARPANet "biz", // Business @@ -1012,8 +1014,8 @@ public final class HTTPDProxyHandler { })); // getting some connection properties - DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL); - int orgHostPort = orgurl.getPort(); + final DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL); + final int orgHostPort = orgurl.getPort(); String orgHostName = orgurl.getHost(); if (orgHostName == null) orgHostName = "unknown"; orgHostName = orgHostName.toLowerCase(Locale.ROOT); @@ -1025,7 +1027,7 @@ public final class HTTPDProxyHandler { detailedErrorMsgMap.put("hostName", orgHostName); // guessing hostnames - final HashSet testHostNames = new HashSet(); + final HashSet testHostNames = new HashSet<>(); String testHostName = null; if (!orgHostName.startsWith("www.")) { testHostName = "www." + orgHostName; @@ -1042,7 +1044,7 @@ public final class HTTPDProxyHandler { if (addr != null) testHostNames.add(testHostName); } - int pos = orgHostName.lastIndexOf('.'); + final int pos = orgHostName.lastIndexOf('.'); if (pos != -1) { final Iterator iter = topLevelDomains.iterator(); while (iter.hasNext()) { @@ -1142,7 +1144,7 @@ public final class HTTPDProxyHandler { logMessage.append(' '); // Method - HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST); + final HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST); final String requestMethod = origrequest.getMethod(); logMessage.append(requestMethod); logMessage.append(' '); diff --git a/source/net/yacy/visualization/DemoApplet.java b/source/net/yacy/visualization/DemoApplet.java deleted file mode 100644 index 55ceb5430..000000000 --- a/source/net/yacy/visualization/DemoApplet.java +++ /dev/null @@ -1,75 +0,0 @@ -package net.yacy.visualization; - -import java.applet.Applet; -import java.awt.Dimension; -import java.awt.Graphics; - - - -@SuppressWarnings("removal") -public class DemoApplet extends Applet implements Runnable { - // can be run in eclipse with - // Run -> Run As -> Java Applet - - // see http://www.javaworld.com/javaworld/jw-03-1996/jw-03-animation.html?page=3 - - private static final long serialVersionUID = -8230253094143014406L; - - private int delay; - private Thread animator; - private RasterPlotter offGraphics; - - @Override - public void init() { - final String str = getParameter("fps"); - final int fps = (str != null) ? Integer.parseInt(str) : 10; - delay = (fps > 0) ? (1000 / fps) : 100; - } - - @Override - public void start() { - animator = new Thread(this); - animator.start(); - } - - @Override - public void run() { - while (Thread.currentThread() == animator) { - final long time = System.currentTimeMillis(); - repaint(); - try { - Thread.sleep(delay - System.currentTimeMillis() + time); - } catch (final InterruptedException e) { - break; - } - } - } - - @Override - public void stop() { - animator = null; - } - - @Override - public void update(final Graphics g) { - final Dimension d = getSize(); - offGraphics = new RasterPlotter(d.width, d.height, RasterPlotter.DrawMode.MODE_REPLACE, "FFFFFF"); - paintFrame(offGraphics); - g.drawImage(offGraphics.getImage(), 0, 0, null); - } - - @Override - public void paint(final Graphics g) { - if (offGraphics != null) { - g.drawImage(offGraphics.getImage(), 0, 0, null); - } - } - - public void paintFrame(final RasterPlotter m) { - RasterPlotter.demoPaint(m); - final int y = (int) (System.currentTimeMillis() / 10 % 300); - m.setColor(RasterPlotter.GREY); - PrintTool.print(m, 0, y, 0, "Hello World", -1, 100); - } - -}