boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed/suppressed more warnings

Browse Source
pull/533/head
Michael Peter Christen 2 years ago
parent 51cf17d252
commit 1893661ee4
8 changed files with 723 additions and 789 deletions
Show Stats Download Patch File Download Diff File

640
source/net/yacy/cora/util/Html2Image.java
Unescape View File

@ -43,6 +43,10 @@ import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit; import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.ImageView; import javax.swing.text.html.ImageView;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
@ -51,57 +55,53 @@ import net.yacy.document.ImageParser;
import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.OS; import net.yacy.kelondro.util.OS;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
/** /**
* Convert html to an copy on disk-image in a other file format * Convert html to an copy on disk-image in a other file format
* currently (pdf and/or jpg) * currently (pdf and/or jpg)
*/ */
public class Html2Image { public class Html2Image {
// Mac // Mac
/** /**
* Path to wkhtmltopdf executable on Mac OS when installed using * Path to wkhtmltopdf executable on Mac OS when installed using
* wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html. * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
* This can also be a path on Debian or another Gnu/Linux distribution. * This can also be a path on Debian or another Gnu/Linux distribution.
*/ */
private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf"); private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
// to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip // to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip
// the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/ // the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/
private final static File convertMac1 = new File("/opt/local/bin/convert"); private final static File convertMac1 = new File("/opt/local/bin/convert");
private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert"); private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert");
/* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript /* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript
The imagemagick policy at /etc should also be checked : The imagemagick policy at /etc should also be checked :
if it contains a line such as <policy domain="coder" rights="none" pattern="PDF" /> it must be edited with rights="read" at minimum if it contains a line such as <policy domain="coder" rights="none" pattern="PDF" /> it must be edited with rights="read" at minimum
*/ */
private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images
private final static File convertDebian = new File("/usr/bin/convert"); private final static File convertDebian = new File("/usr/bin/convert");
/** /**
* Path to wkhtmltopdf executable on Windows, when installed with default * Path to wkhtmltopdf executable on Windows, when installed with default
* settings using wkhtmltox-n.n.n.msvc2015-win64.exe from * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
* https://wkhtmltopdf.org/downloads.html * https://wkhtmltopdf.org/downloads.html
*/ */
private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"); private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
/** /**
* Path to wkhtmltopdf executable on Windows, when installed with default * Path to wkhtmltopdf executable on Windows, when installed with default
* settings using wkhtmltox-n.n.n.msvc2015-win32.exe from * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
* https://wkhtmltopdf.org/downloads.html * https://wkhtmltopdf.org/downloads.html
*/ */
private static final File WKHTMLTOPDF_WINDOWS_X86 = new File( private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
"C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe"); "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
/** Command to use when wkhtmltopdf is included in the system Path */ /** Command to use when wkhtmltopdf is included in the system Path */
private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf"; private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
/** Command to use when imagemagick convert is included in the system Path */ /** Command to use when imagemagick convert is included in the system Path */
private static final String CONVERT_COMMAND = "convert"; private static final String CONVERT_COMMAND = "convert";
private static boolean usexvfb = false; private static boolean usexvfb = false;
@ -109,109 +109,111 @@ public class Html2Image {
* @return when the wkhtmltopdf command is detected as available in the system * @return when the wkhtmltopdf command is detected as available in the system
*/ */
public static boolean wkhtmltopdfAvailable() { public static boolean wkhtmltopdfAvailable() {
/* Check wkhtmltopdf common installation paths and system Path */ /* Check wkhtmltopdf common installation paths and system Path */
return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath(); return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
} }
/** /**
* @return a wkhtmltopdf executable file when one can be found, null otherwise * @return a wkhtmltopdf executable file when one can be found, null otherwise
*/ */
private static File wkhtmltopdfExecutable() { private static File wkhtmltopdfExecutable() {
File executable = null; File executable = null;
if(OS.isWindows) { if(OS.isWindows) {
if(WKHTMLTOPDF_WINDOWS.exists()) { if(WKHTMLTOPDF_WINDOWS.exists()) {
executable = WKHTMLTOPDF_WINDOWS; executable = WKHTMLTOPDF_WINDOWS;
} else if(WKHTMLTOPDF_WINDOWS_X86.exists()) { } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
executable = WKHTMLTOPDF_WINDOWS_X86; executable = WKHTMLTOPDF_WINDOWS_X86;
} }
} else { } else {
if(wkhtmltopdfMac.exists()) { if(wkhtmltopdfMac.exists()) {
executable = wkhtmltopdfMac; executable = wkhtmltopdfMac;
} else if(wkhtmltopdfDebian.exists()) { } else if(wkhtmltopdfDebian.exists()) {
executable = wkhtmltopdfDebian; executable = wkhtmltopdfDebian;
} }
} }
return executable; return executable;
} }
/** /**
* @return true when wkhtmltopdf is available in system path * @return true when wkhtmltopdf is available in system path
*/ */
private static boolean wkhtmltopdfAvailableInPath() { private static boolean wkhtmltopdfAvailableInPath() {
boolean available = false; boolean available = false;
try { try {
final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V"); @SuppressWarnings("deprecation")
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
} catch (final IOException e) { available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path."); } catch (final IOException e) {
} catch (final InterruptedException e) { ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
Thread.currentThread().interrupt(); // preserve thread interrupted state } catch (final InterruptedException e) {
} Thread.currentThread().interrupt(); // preserve thread interrupted state
return available; }
} return available;
}
/** /**
* @return a imagemagick convert executable file when one can be found, null otherwise * @return a imagemagick convert executable file when one can be found, null otherwise
*/ */
private static File convertExecutable() { private static File convertExecutable() {
File executable = null; File executable = null;
if(!OS.isWindows) { if(!OS.isWindows) {
if(convertMac1.exists()) { if(convertMac1.exists()) {
executable = convertMac1; executable = convertMac1;
} else if(convertMac2.exists()) { } else if(convertMac2.exists()) {
executable = convertMac2; executable = convertMac2;
} else if(convertDebian.exists()) { } else if(convertDebian.exists()) {
executable = convertDebian; executable = convertDebian;
} }
} }
return executable; return executable;
} }
/** /**
* @return when the imagemagick convert command is detected as available in the system * @return when the imagemagick convert command is detected as available in the system
*/ */
public static boolean convertAvailable() { public static boolean convertAvailable() {
/* Check convert common installation paths and system Path */ /* Check convert common installation paths and system Path */
return convertExecutable() != null || convertAvailableInPath(); return convertExecutable() != null || convertAvailableInPath();
} }
/** /**
* @return when imagemagick convert is available in system path * @return when imagemagick convert is available in system path
*/ */
private static boolean convertAvailableInPath() { private static boolean convertAvailableInPath() {
boolean available = false; boolean available = false;
if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
try { try {
final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version"); @SuppressWarnings("deprecation")
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
} catch (final IOException e) { available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
ConcurrentLog.fine("Html2Image", "convert is not included in system path."); } catch (final IOException e) {
} catch (final InterruptedException e) { ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
Thread.currentThread().interrupt(); // preserve thread interrupted state } catch (final InterruptedException e) {
} Thread.currentThread().interrupt(); // preserve thread interrupted state
} }
return available; }
} return available;
}
/**
* Run the wkhtmltopdf external tool to fetch and render to PDF a web resource. /**
* wKhtmltopdf may be called multiple times with various parameters flavors in * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
* case of failure. * wKhtmltopdf may be called multiple times with various parameters flavors in
* * case of failure.
* @param url the URL of a web resource to fetch, render and convert to *
* a pdf file. Must not be null. * @param url the URL of a web resource to fetch, render and convert to
* @param proxy the eventual proxy address to use. Can be null. Must be of * a pdf file. Must not be null.
* the form http://host:port; use YaCy here as proxy which is * @param proxy the eventual proxy address to use. Can be null. Must be of
* mostly http://localhost:8090 * the form http://host:port; use YaCy here as proxy which is
* @param destination the destination PDF file that should be written. Must not * mostly http://localhost:8090
* be null. * @param destination the destination PDF file that should be written. Must not
* @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf * be null.
* call termination. Beyond this limit the process is killed. * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf
* @return true when the destination file was successfully written * call termination. Beyond this limit the process is killed.
*/ * @return true when the destination file was successfully written
*/
public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) { public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {
boolean success = false; boolean success = false;
for (boolean ignoreErrors: new boolean[]{false, true}) { for (final boolean ignoreErrors: new boolean[]{false, true}) {
success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds); success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
if (success) break; if (success) break;
if (!success && proxy != null) { if (!success && proxy != null) {
@ -227,49 +229,49 @@ public class Html2Image {
} }
return success; return success;
} }
/** /**
* Run wkhtmltopdf in a separate process to fetch and render to PDF a web * Run wkhtmltopdf in a separate process to fetch and render to PDF a web
* resource. * resource.
* *
* @param url the URL of a web resource to fetch, render and convert to * @param url the URL of a web resource to fetch, render and convert to
* a pdf file. Must not be null. * a pdf file. Must not be null.
* @param proxy the eventual proxy address to use. Can be null. * @param proxy the eventual proxy address to use. Can be null.
* @param destination the destination PDF file that should be written. Must not * @param destination the destination PDF file that should be written. Must not
* be null. * be null.
* @param userAgent TODO: implement * @param userAgent TODO: implement
* @param acceptLanguage TODO: implement * @param acceptLanguage TODO: implement
* @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
* @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf
* dedicated process termination. Beyond this limit the * dedicated process termination. Beyond this limit the
* process is killed. * process is killed.
* @return true when the destination file was successfully written * @return true when the destination file was successfully written
*/ */
private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination, private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) { final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
final String wkhtmltopdfCmd; final String wkhtmltopdfCmd;
final File wkhtmltopdf = wkhtmltopdfExecutable(); final File wkhtmltopdf = wkhtmltopdfExecutable();
if(wkhtmltopdf != null) { if(wkhtmltopdf != null) {
wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath(); wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
} else if(wkhtmltopdfAvailableInPath()) { } else if(wkhtmltopdfAvailableInPath()) {
wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND; wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
} else { } else {
ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!"); ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
return false; return false;
} }
String commandline = String commandline =
wkhtmltopdfCmd + " -q --title '" + url + "' " + wkhtmltopdfCmd + " -q --title '" + url + "' " +
//acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " + //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
//(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") + //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
(proxy == null ? "" : "--proxy " + proxy + " ") + (proxy == null ? "" : "--proxy " + proxy + " ") +
(ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it... (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
//"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " + //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
"--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " + "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
url + " " + destination.getAbsolutePath(); url + " " + destination.getAbsolutePath();
try { try {
ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline); ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) { if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) {
return true; return true;
} }
// if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers // if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers
commandline = "xvfb-run -a " + commandline; commandline = "xvfb-run -a " + commandline;
@ -280,46 +282,46 @@ public class Html2Image {
} }
} }
/** /**
* Run a wkhtmltopdf commandline in a separate process. * Run a wkhtmltopdf commandline in a separate process.
* *
* @param proxy the eventual proxy address to use. Can be null. * @param proxy the eventual proxy address to use. Can be null.
* @param destination the destination PDF file that should be written. Must not * @param destination the destination PDF file that should be written. Must not
* be null. * be null.
* @param commandline the wkhtmltopdf command line to execute. Must not be null. * @param commandline the wkhtmltopdf command line to execute. Must not be null.
* @param maxSeconds the maximum time in seconds to wait for the process * @param maxSeconds the maximum time in seconds to wait for the process
* termination. Beyond this limit the process is killed. * termination. Beyond this limit the process is killed.
* @return true when the destination file was successfully written * @return true when the destination file was successfully written
* @throws IOException when an unexpected error occurred * @throws IOException when an unexpected error occurred
*/ */
private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException {
throws IOException { @SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(commandline); final Process p = Runtime.getRuntime().exec(commandline);
try { try {
p.waitFor(maxSeconds, TimeUnit.SECONDS); p.waitFor(maxSeconds, TimeUnit.SECONDS);
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
p.destroyForcibly(); p.destroyForcibly();
ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline); ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
Thread.currentThread().interrupt(); // Keep the thread interrupted state Thread.currentThread().interrupt(); // Keep the thread interrupted state
return false; return false;
} }
if(p.isAlive()) { if(p.isAlive()) {
ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline); ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
p.destroyForcibly(); p.destroyForcibly();
return false; return false;
} }
if (p.exitValue() == 0 && destination.exists()) { if (p.exitValue() == 0 && destination.exists()) {
return true; return true;
} }
final List<String> messages = OS.readStreams(p); final List<String> messages = OS.readStreams(p);
ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline); ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
for (final String message : messages) { for (final String message : messages) {
ConcurrentLog.warn("Html2Image", ">> " + message); ConcurrentLog.warn("Html2Image", ">> " + message);
} }
return false; return false;
} }
/** /**
* Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75 * Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75
* using internal pdf library or external command line tool on linux or mac * using internal pdf library or external command line tool on linux or mac
@ -332,34 +334,34 @@ public class Html2Image {
* @return true when the ouput image file was successfully written. * @return true when the ouput image file was successfully written.
*/ */
public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) { public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) {
/* Deduce the ouput image format from the file extension */ /* Deduce the ouput image format from the file extension */
String imageFormat = MultiProtocolURL.getFileExtension(image.getName()); String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
if(imageFormat.isEmpty()) { if(imageFormat.isEmpty()) {
/* Use JPEG as a default fallback */ /* Use JPEG as a default fallback */
imageFormat = "jpg"; imageFormat = "jpg";
} }
String convertCmd = null; String convertCmd = null;
final File convert = convertExecutable(); final File convert = convertExecutable();
if(convert != null) { if(convert != null) {
convertCmd = convert.getAbsolutePath(); convertCmd = convert.getAbsolutePath();
} else if(convertAvailableInPath()) { } else if(convertAvailableInPath()) {
convertCmd = CONVERT_COMMAND; convertCmd = CONVERT_COMMAND;
} else { } else {
ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!"); ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
} }
// convert pdf to jpg using internal pdfbox capability // convert pdf to jpg using internal pdfbox capability
if (convertCmd == null) { if (convertCmd == null) {
try (final PDDocument pdoc = PDDocument.load(pdf);) { try (final PDDocument pdoc = PDDocument.load(pdf);) {
BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB); final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
return ImageIO.write(bi, imageFormat, image); return ImageIO.write(bi, imageFormat, image);
} catch (final IOException ex) { } catch (final IOException ex) {
ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox" ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
+ (ex.getMessage() != null ? " : " + ex.getMessage() : "")); + (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
return false; return false;
} }
} }
@ -367,18 +369,18 @@ public class Html2Image {
try { try {
// i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg // i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg
// note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf // note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf
String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath(); final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
List<String> message = OS.execSynchronous(command); List<String> message = OS.execSynchronous(command);
if (image.exists()) return true; if (image.exists()) return true;
ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command); ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command);
for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m); for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
// another try for mac: use Image Events using AppleScript in osacript commands... // another try for mac: use Image Events using AppleScript in osacript commands...
// the following command overwrites a pdf with an png, so we must make a copy first // the following command overwrites a pdf with an png, so we must make a copy first
if (!OS.isMacArchitecture) return false; if (!OS.isMacArchitecture) return false;
File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf"); final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
org.apache.commons.io.FileUtils.copyFile(pdf, pngFile); org.apache.commons.io.FileUtils.copyFile(pdf, pngFile);
String[] commandx = {"osascript", final String[] commandx = {"osascript",
"-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"", "-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"",
"-e", "tell application \"Image Events\"", "-e", "tell application \"Image Events\"",
"-e", "set Img to open file ImgFile", "-e", "set Img to open file ImgFile",
@ -386,15 +388,15 @@ public class Html2Image {
"-e", "end tell"}; "-e", "end tell"};
//ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx); //ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx);
message = OS.execSynchronous(commandx); message = OS.execSynchronous(commandx);
for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m); for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
// now we must read and convert this file to the target format with the target size 1024x1024 // now we must read and convert this file to the target format with the target size 1024x1024
try { try {
File newPngFile = new File(pngFile.getAbsolutePath() + ".png"); final File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
pngFile.renameTo(newPngFile); pngFile.renameTo(newPngFile);
final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile)); final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile));
if(img == null) { if(img == null) {
/* Should not happen. If so, ImageParser.parse() should already have logged about the error */ /* Should not happen. If so, ImageParser.parse() should already have logged about the error */
return false; return false;
} }
final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING); final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
final MediaTracker mediaTracker = new MediaTracker(new Container()); final MediaTracker mediaTracker = new MediaTracker(new Container());
@ -406,24 +408,24 @@ public class Html2Image {
ImageIO.write(bi, imageFormat, image); ImageIO.write(bi, imageFormat, image);
newPngFile.delete(); newPngFile.delete();
return image.exists(); return image.exists();
} catch (IOException e) { } catch (final IOException e) {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
return false; return false;
} }
} catch (IOException e) { } catch (final IOException e) {
e.printStackTrace(); e.printStackTrace();
return false; return false;
} }
} }
/** /**
* render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported! * render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported!
* @param url * @param url
* @param size * @param size
* @throws IOException * @throws IOException
*/ */
public static void writeSwingImage(String url, Dimension size, File destination) throws IOException { public static void writeSwingImage(String url, Dimension size, File destination) throws IOException {
// set up a pane for rendering // set up a pane for rendering
final JEditorPane htmlPane = new JEditorPane(); final JEditorPane htmlPane = new JEditorPane();
htmlPane.setSize(size); htmlPane.setSize(size);
@ -434,7 +436,7 @@ public class Html2Image {
@Override @Override
public Document createDefaultDocument() { public Document createDefaultDocument() {
HTMLDocument doc = (HTMLDocument) super.createDefaultDocument(); final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
doc.setAsynchronousLoadPriority(-1); doc.setAsynchronousLoadPriority(-1);
return doc; return doc;
} }
@ -444,7 +446,7 @@ public class Html2Image {
return new HTMLFactory() { return new HTMLFactory() {
@Override @Override
public View create(Element elem) { public View create(Element elem) {
View view = super.create(elem); final View view = super.create(elem);
if (view instanceof ImageView) { if (view instanceof ImageView) {
((ImageView) view).setLoadsSynchronously(true); ((ImageView) view).setLoadsSynchronously(true);
} }
@ -460,23 +462,23 @@ public class Html2Image {
public void propertyChange(PropertyChangeEvent evt) { public void propertyChange(PropertyChangeEvent evt) {
} }
}); });
// load the page // load the page
try { try {
htmlPane.setPage(url); htmlPane.setPage(url);
} catch (IOException e) { } catch (final IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
// render the page // render the page
Dimension prefSize = htmlPane.getPreferredSize(); final Dimension prefSize = htmlPane.getPreferredSize();
BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB); final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
Graphics graphics = img.getGraphics(); final Graphics graphics = img.getGraphics();
htmlPane.setSize(prefSize); htmlPane.setSize(prefSize);
htmlPane.paint(graphics); htmlPane.paint(graphics);
ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination); ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination);
} }
/** /**
* Test PDF or image snapshot generation for a given URL. * Test PDF or image snapshot generation for a given URL.
* @param args main arguments list: * @param args main arguments list:
@ -491,88 +493,88 @@ public class Html2Image {
* </li> * </li>
* </ol> * </ol>
*/ */
public static void main(String[] args) { public static void main(String[] args) {
final String usageMessage = "Usage : java " + Html2Image.class.getName() final String usageMessage = "Usage : java " + Html2Image.class.getName()
+ " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]"; + " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";
int exitStatus = 0; int exitStatus = 0;
try { try {
if (args.length < 2) { if (args.length < 2) {
System.out.println("Missing required parameter(s)."); System.out.println("Missing required parameter(s).");
System.out.println(usageMessage); System.out.println(usageMessage);
exitStatus = 1; exitStatus = 1;
return; return;
} }
final String targetPath = args[1]; final String targetPath = args[1];
if (args.length < 3 || "wkhtmltopdf".equals(args[2])) { if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
if(Html2Image.wkhtmltopdfAvailable()) { if(Html2Image.wkhtmltopdfAvailable()) {
final File targetPdfFile; final File targetPdfFile;
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) { if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf"); targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
} else if(targetPath.endsWith(".pdf")) { } else if(targetPath.endsWith(".pdf")) {
targetPdfFile = new File(targetPath); targetPdfFile = new File(targetPath);
} else { } else {
System.out.println("Unsupported output format"); System.out.println("Unsupported output format");
System.out.println(usageMessage); System.out.println(usageMessage);
exitStatus = 1; exitStatus = 1;
return; return;
} }
if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent, if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,
"en-us,en;q=0.5", targetPdfFile, 30)) { "en-us,en;q=0.5", targetPdfFile, 30)) {
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) { if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) { if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile); ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
} else { } else {
exitStatus = 1; exitStatus = 1;
return; return;
} }
} }
} else { } else {
exitStatus = 1; exitStatus = 1;
return; return;
} }
} else { } else {
System.out.println("Unable to locate wkhtmltopdf executable on this system!"); System.out.println("Unable to locate wkhtmltopdf executable on this system!");
exitStatus = 1; exitStatus = 1;
return; return;
} }
} else if ("swing".equals(args[2])) { } else if ("swing".equals(args[2])) {
if(targetPath.endsWith(".pdf")) { if(targetPath.endsWith(".pdf")) {
System.out.println("Pdf output format is not supported with swing method."); System.out.println("Pdf output format is not supported with swing method.");
exitStatus = 1; exitStatus = 1;
return; return;
} }
if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) { if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
System.out.println("Unsupported output format"); System.out.println("Unsupported output format");
System.out.println(usageMessage); System.out.println(usageMessage);
exitStatus = 1; exitStatus = 1;
return; return;
} }
try { try {
Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath)); Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
} catch (final IOException e) { } catch (final IOException e) {
e.printStackTrace(); e.printStackTrace();
exitStatus = 1; exitStatus = 1;
return; return;
} }
} else { } else {
System.out.println("Unknown method : please specify either wkhtmltopdf or swing."); System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
exitStatus = 1; exitStatus = 1;
return; return;
} }
} finally { } finally {
/* Shutdown running threads */ /* Shutdown running threads */
Domains.close(); Domains.close();
try { try {
HTTPClient.closeConnectionManager(); HTTPClient.closeConnectionManager();
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // restore interrupted state Thread.currentThread().interrupt(); // restore interrupted state
} }
ConcurrentLog.shutdown(); ConcurrentLog.shutdown();
if(exitStatus != 0) { if(exitStatus != 0) {
System.exit(exitStatus); System.exit(exitStatus);
} }
} }
} }
} }

19
source/net/yacy/cora/util/Memory.java
Unescape View File

@ -115,8 +115,9 @@ public class Memory {
* @return the "recent cpu usage" for the whole operating environment; * @return the "recent cpu usage" for the whole operating environment;
* a negative value if not available. * a negative value if not available.
*/ */
@SuppressWarnings("deprecation")
public static double getSystemCpuLoad() { public static double getSystemCpuLoad() {
com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
return operatingSystemMXBean.getSystemCpuLoad(); return operatingSystemMXBean.getSystemCpuLoad();
} }
@ -136,13 +137,13 @@ public class Memory {
* a negative value if not available. * a negative value if not available.
*/ */
public static double getProcessCpuLoad() { public static double getProcessCpuLoad() {
com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
return operatingSystemMXBean.getProcessCpuLoad(); return operatingSystemMXBean.getProcessCpuLoad();
} }
public static Map<String, Object> status() { public static Map<String, Object> status() {
Runtime runtime = Runtime.getRuntime(); final Runtime runtime = Runtime.getRuntime();
Map<String, Object> status = new LinkedHashMap<>(); final Map<String, Object> status = new LinkedHashMap<>();
status.put("service", "Peer"); status.put("service", "Peer");
status.put("assigned_memory", runtime.maxMemory()); status.put("assigned_memory", runtime.maxMemory());
status.put("used_memory", runtime.totalMemory() - runtime.freeMemory()); status.put("used_memory", runtime.totalMemory() - runtime.freeMemory());
@ -153,7 +154,7 @@ public class Memory {
status.put("load_system_load_average", Memory.getSystemLoadAverage()); status.put("load_system_load_average", Memory.getSystemLoadAverage());
status.put("load_system_cpu_load", Memory.getSystemCpuLoad()); status.put("load_system_cpu_load", Memory.getSystemCpuLoad());
status.put("load_process_cpu_load", Memory.getProcessCpuLoad()); status.put("load_process_cpu_load", Memory.getProcessCpuLoad());
YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer(); final YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();
status.put("server_threads", server == null ? 0 : server.getServerThreads()); status.put("server_threads", server == null ? 0 : server.getServerThreads());
return status; return status;
} }
@ -163,7 +164,7 @@ public class Memory {
* @return the number of deadlocked threads * @return the number of deadlocked threads
*/ */
public static long deadlocks() { public static long deadlocks() {
long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
if (deadlockIDs == null) return 0; if (deadlockIDs == null) return 0;
return deadlockIDs.length; return deadlockIDs.length;
} }
@ -172,10 +173,10 @@ public class Memory {
* write deadlocked threads as to the log as warning * write deadlocked threads as to the log as warning
*/ */
public static void logDeadlocks() { public static void logDeadlocks() {
long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads(); final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
if (deadlockIDs == null) return; if (deadlockIDs == null) return;
ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true); final ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);
for (ThreadInfo ti : infos) { for (final ThreadInfo ti : infos) {
ConcurrentLog.warn("DEADLOCKREPORT", ti.toString()); ConcurrentLog.warn("DEADLOCKREPORT", ti.toString());
} }
} }

29
source/net/yacy/data/list/XMLBlacklistImporter.java
Unescape View File

@ -33,7 +33,6 @@ import org.xml.sax.InputSource;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import org.xml.sax.XMLReader; import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler; import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;
/** /**
* This class provides methods to import blacklists from an XML file (see * This class provides methods to import blacklists from an XML file (see
@ -54,13 +53,15 @@ public class XMLBlacklistImporter extends DefaultHandler {
* @throws java.io.IOException if input can't be read * @throws java.io.IOException if input can't be read
* @throws org.xml.sax.SAXException if XML can't be parsed * @throws org.xml.sax.SAXException if XML can't be parsed
*/ */
@SuppressWarnings("deprecation")
public synchronized ListAccumulator parse(InputSource input) throws IOException, SAXException { public synchronized ListAccumulator parse(InputSource input) throws IOException, SAXException {
XMLReader reader = XMLReaderFactory.createXMLReader(); @SuppressWarnings("deprecation")
final XMLReader reader = org.xml.sax.helpers.XMLReaderFactory.createXMLReader();
reader.setContentHandler(this); reader.setContentHandler(this);
reader.parse(input); reader.parse(input);
return ba; return this.ba;
} }
/** /**
@ -104,7 +105,7 @@ public class XMLBlacklistImporter extends DefaultHandler {
*/ */
@Override @Override
public void startDocument() { public void startDocument() {
ba = new ListAccumulator(); this.ba = new ListAccumulator();
} }
/** /**
@ -127,24 +128,24 @@ public class XMLBlacklistImporter extends DefaultHandler {
public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) { public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) {
if (qName.equalsIgnoreCase("list")) { if (qName.equalsIgnoreCase("list")) {
currentListName = attributes.getValue("name"); this.currentListName = attributes.getValue("name");
ba.addList(currentListName); this.ba.addList(this.currentListName);
int attributesLength = 0; int attributesLength = 0;
if ((attributesLength = attributes.getLength()) > 1) { if ((attributesLength = attributes.getLength()) > 1) {
for (int i = 0; i < attributesLength; i++) { for (int i = 0; i < attributesLength; i++) {
if (!attributes.getQName(i).equals("name")) { if (!attributes.getQName(i).equals("name")) {
ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i)); this.ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));
} }
} }
} }
} }
if (qName.equalsIgnoreCase("item")) { if (qName.equalsIgnoreCase("item")) {
lastText = new StringBuilder(); this.lastText = new StringBuilder();
} }
} }
/** /**
@ -162,7 +163,7 @@ public class XMLBlacklistImporter extends DefaultHandler {
@Override @Override
public void endElement(final String uri, final String localName, final String qName) throws SAXException { public void endElement(final String uri, final String localName, final String qName) throws SAXException {
if (qName.equalsIgnoreCase("item")) { if (qName.equalsIgnoreCase("item")) {
ba.addEntryToCurrent(lastText.toString()); this.ba.addEntryToCurrent(this.lastText.toString());
} }
} }
@ -175,8 +176,8 @@ public class XMLBlacklistImporter extends DefaultHandler {
*/ */
@Override @Override
public void characters(char[] ch, int start, int length) throws SAXException { public void characters(char[] ch, int start, int length) throws SAXException {
if (lastText == null) lastText = new StringBuilder(); if (this.lastText == null) this.lastText = new StringBuilder();
lastText.append(ch, start, length); this.lastText.append(ch, start, length);
} }
} }

46
source/net/yacy/gui/framework/Browser.java
Unescape View File

@ -37,7 +37,7 @@ import net.yacy.cora.util.ConcurrentLog;
public class Browser { public class Browser {
// constants for system identification // constants for system identification
public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.* public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.*
public static final int systemMacOSX = 1; // all Mac OS X public static final int systemMacOSX = 1; // all Mac OS X
public static final int systemUnix = 2; // all Unix/Linux type systems public static final int systemUnix = 2; // all Unix/Linux type systems
@ -100,7 +100,7 @@ public class Browser {
openBrowserClassic(url); openBrowserClassic(url);
} }
} }
public static void openBrowserClassic(final String url) { public static void openBrowserClassic(final String url) {
try { try {
if (systemOS == systemMacOSX) { if (systemOS == systemMacOSX) {
@ -113,12 +113,12 @@ public class Browser {
throw new RuntimeException("System unknown"); throw new RuntimeException("System unknown");
} }
} catch (final Throwable e) { } catch (final Throwable e) {
ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString()); ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());
} }
} }
private static void openBrowserMac(final String url) throws Exception { private static void openBrowserMac(final String url) throws Exception {
Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""}); final Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});
p.waitFor(); p.waitFor();
if (p.exitValue() != 0) { if (p.exitValue() != 0) {
throw new RuntimeException("Mac Exec Error: " + errorResponse(p)); throw new RuntimeException("Mac Exec Error: " + errorResponse(p));
@ -131,12 +131,13 @@ public class Browser {
* @throws Exception when an error occured * @throws Exception when an error occured
*/ */
private static void openDefaultUnixBrowser(final String url) throws Exception { private static void openDefaultUnixBrowser(final String url) throws Exception {
/* Use the freedesktop xdg-open to open url with the default browser. /* Use the freedesktop xdg-open to open url with the default browser.
* xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/) * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)
* It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it
* (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */ * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */
String cmd = "xdg-open " + url; final String cmd = "xdg-open " + url;
Process p = Runtime.getRuntime().exec(cmd); @SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(cmd);
p.waitFor(); p.waitFor();
if (p.exitValue() != 0) { if (p.exitValue() != 0) {
throw new RuntimeException("Unix Exec Error/xdg-open: " + errorResponse(p)); throw new RuntimeException("Unix Exec Error/xdg-open: " + errorResponse(p));
@ -152,7 +153,8 @@ public class Browser {
cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\""; cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\"";
} }
//cmd = "cmd.exe /c start javascript:document.location='" + url + "'"; //cmd = "cmd.exe /c start javascript:document.location='" + url + "'";
Process p = Runtime.getRuntime().exec(cmd); @SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(cmd);
p.waitFor(); p.waitFor();
if (p.exitValue() != 0) { if (p.exitValue() != 0) {
throw new RuntimeException("EXEC ERROR: " + errorResponse(p)); throw new RuntimeException("EXEC ERROR: " + errorResponse(p));
@ -183,15 +185,15 @@ public class Browser {
* @param args * @param args
*/ */
public static void main(final String[] args) { public static void main(final String[] args) {
try { try {
if (args.length > 0 && "-u".equals(args[0])) { if (args.length > 0 && "-u".equals(args[0])) {
openBrowser(args[1]); openBrowser(args[1]);
} else { } else {
System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]"); System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");
} }
} finally { } finally {
ConcurrentLog.shutdown(); ConcurrentLog.shutdown();
} }
System.out.println("The End!"); System.out.println("The End!");
} }
} }

201
source/net/yacy/kelondro/util/FileUtils.java
Unescape View File

@ -56,15 +56,14 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.storage.Files;
import net.yacy.cora.util.ConcurrentLog;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.mozilla.intl.chardet.nsDetector; import org.mozilla.intl.chardet.nsDetector;
import org.mozilla.intl.chardet.nsPSMDetector; import org.mozilla.intl.chardet.nsPSMDetector;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.storage.Files;
import net.yacy.cora.util.ConcurrentLog;
public final class FileUtils { public final class FileUtils {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
@ -98,7 +97,7 @@ public final class FileUtils {
* @see #copy(File source, File dest) * @see #copy(File source, File dest)
*/ */
public static long copy(final InputStream source, final OutputStream dest, final long count) public static long copy(final InputStream source, final OutputStream dest, final long count)
throws IOException { throws IOException {
assert count < 0 || count > 0 : "precondition violated: count == " + count + " (nothing to copy)"; assert count < 0 || count > 0 : "precondition violated: count == " + count + " (nothing to copy)";
if ( count == 0 ) { if ( count == 0 ) {
// no bytes to copy // no bytes to copy
@ -106,15 +105,15 @@ public final class FileUtils {
} }
final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE); final int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
int c; int c;
long total = 0; long total = 0;
long remaining; long remaining;
if(count > 0) { if(count > 0) {
remaining = count; remaining = count;
} else { } else {
remaining = Long.MAX_VALUE; remaining = Long.MAX_VALUE;
} }
while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) { while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) {
dest.write(buffer, 0, c); dest.write(buffer, 0, c);
@ -158,7 +157,7 @@ public final class FileUtils {
* @throws NullPointerException when a parameter is null * @throws NullPointerException when a parameter is null
*/ */
public static int copy(final InputStream source, final Writer dest, final Charset inputCharset) public static int copy(final InputStream source, final Writer dest, final Charset inputCharset)
throws IOException { throws IOException {
final InputStreamReader reader = new InputStreamReader(source, inputCharset); final InputStreamReader reader = new InputStreamReader(source, inputCharset);
return copy(reader, dest); return copy(reader, dest);
} }
@ -212,8 +211,8 @@ public final class FileUtils {
// an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here // an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here
//Log.logException(e); //Log.logException(e);
throw new IOException( throw new IOException(
e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(), e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),
e); e);
} }
return count; return count;
} }
@ -260,8 +259,8 @@ public final class FileUtils {
fos.close(); fos.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn( ConcurrentLog.warn(
"FileUtils", "FileUtils",
"cannot close FileOutputStream for " + dest + "! " + e.getMessage()); "cannot close FileOutputStream for " + dest + "! " + e.getMessage());
} }
} }
} }
@ -282,17 +281,17 @@ public final class FileUtils {
* @see #copy(File source, File dest) * @see #copy(File source, File dest)
*/ */
public static void copyRange(final File source, final OutputStream dest, final int start) public static void copyRange(final File source, final OutputStream dest, final int start)
throws IOException { throws IOException {
InputStream fis = null; InputStream fis = null;
try { try {
fis = new FileInputStream(source); fis = new FileInputStream(source);
final long skipped = fis.skip(start); final long skipped = fis.skip(start);
if ( skipped != start ) { if ( skipped != start ) {
throw new IllegalStateException("Unable to skip '" throw new IllegalStateException("Unable to skip '"
+ start + start
+ "' bytes. Only '" + "' bytes. Only '"
+ skipped + skipped
+ "' bytes skipped."); + "' bytes skipped.");
} }
copy(fis, dest, -1); copy(fis, dest, -1);
} finally { } finally {
@ -300,7 +299,7 @@ public final class FileUtils {
try { try {
fis.close(); fis.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
} }
} }
} }
@ -328,7 +327,7 @@ public final class FileUtils {
try { try {
fis.close(); fis.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
} }
} }
} }
@ -368,17 +367,17 @@ public final class FileUtils {
* @throws NullPointerException when source parameter is null * @throws NullPointerException when source parameter is null
*/ */
public static byte[] read(final InputStream source) throws IOException { public static byte[] read(final InputStream source) throws IOException {
byte[] content; byte[] content;
try { try {
content = read(source, -1); content = read(source, -1);
} finally { } finally {
/* source input stream must be closed here in all cases */ /* source input stream must be closed here in all cases */
try { try {
source.close(); source.close();
} catch(IOException ignoredException) { } catch(final IOException ignoredException) {
} }
} }
return content; return content;
} }
/** /**
@ -392,7 +391,7 @@ public final class FileUtils {
*/ */
public static byte[] read(final InputStream source, final int count) throws IOException { public static byte[] read(final InputStream source, final int count) throws IOException {
if(count == 0) { if(count == 0) {
return new byte[0]; return new byte[0];
} }
final ByteArrayOutputStream baos = new ByteArrayOutputStream(512); final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
copy(source, baos, count); copy(source, baos, count);
@ -414,7 +413,7 @@ public final class FileUtils {
try { try {
fis.close(); fis.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source); ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
} }
} }
fis = null; fis = null;
@ -470,17 +469,17 @@ public final class FileUtils {
return source; return source;
} }
/** /**
* Generate a set of strings matching each line of the given file. Lines are * Generate a set of strings matching each line of the given file. Lines are
* lower cased and any eventual surrounding space characters are removed. Empty * lower cased and any eventual surrounding space characters are removed. Empty
* lines and lines starting with the '#' character are ignored. * lines and lines starting with the '#' character are ignored.
* *
* @param file * @param file
* a file to load * a file to load
* @return a set of strings eventually empty * @return a set of strings eventually empty
*/ */
public static HashSet<String> loadList(final File file) { public static HashSet<String> loadList(final File file) {
final HashSet<String> set = new HashSet<String>(); final HashSet<String> set = new HashSet<>();
BufferedReader br = null; BufferedReader br = null;
try { try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file))); br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
@ -497,7 +496,7 @@ public final class FileUtils {
try { try {
br.close(); br.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file); ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
} }
} }
} }
@ -516,10 +515,10 @@ public final class FileUtils {
} }
public static ConcurrentHashMap<String, byte[]> loadMapB(final File f) { public static ConcurrentHashMap<String, byte[]> loadMapB(final File f) {
ConcurrentHashMap<String, String> m = loadMap(f); final ConcurrentHashMap<String, String> m = loadMap(f);
if (m == null) return null; if (m == null) return null;
ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<String, byte[]>(); final ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<>();
for (Map.Entry<String, String> e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue())); for (final Map.Entry<String, String> e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));
return mb; return mb;
} }
@ -529,7 +528,7 @@ public final class FileUtils {
private final static String[] unescaped_strings_out = {"\\", "\n", "="}; private final static String[] unescaped_strings_out = {"\\", "\n", "="};
public static void saveMap(final File file, final Map<String, String> props, final String comment) { public static void saveMap(final File file, final Map<String, String> props, final String comment) {
boolean err = false; boolean err = false;
PrintWriter pw = null; PrintWriter pw = null;
final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000)); final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000));
try { try {
@ -550,10 +549,7 @@ public final class FileUtils {
pw.println(key + "=" + value); pw.println(key + "=" + value);
} }
pw.println("# EOF"); pw.println("# EOF");
} catch (final FileNotFoundException e ) { } catch (final FileNotFoundException | UnsupportedEncodingException e ) {
ConcurrentLog.warn("FileUtils", e.getMessage(), e);
err = true;
} catch (final UnsupportedEncodingException e ) {
ConcurrentLog.warn("FileUtils", e.getMessage(), e); ConcurrentLog.warn("FileUtils", e.getMessage(), e);
err = true; err = true;
} finally { } finally {
@ -568,10 +564,10 @@ public final class FileUtils {
// ignore // ignore
} }
} }
public static void saveMapB(final File file, final Map<String, byte[]> props, final String comment) { public static void saveMapB(final File file, final Map<String, byte[]> props, final String comment) {
HashMap<String, String> m = new HashMap<String, String>(); final HashMap<String, String> m = new HashMap<>();
for (Map.Entry<String, byte[]> e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue())); for (final Map.Entry<String, byte[]> e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));
saveMap(file, m, comment); saveMap(file, m, comment);
} }
@ -582,7 +578,7 @@ public final class FileUtils {
public static ConcurrentHashMap<String, String> table(final Iterator<String> li) { public static ConcurrentHashMap<String, String> table(final Iterator<String> li) {
String line; String line;
final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<String, String>(); final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<>();
while ( li.hasNext() ) { while ( li.hasNext() ) {
int pos = 0; int pos = 0;
line = li.next().trim(); line = li.next().trim();
@ -594,8 +590,8 @@ public final class FileUtils {
pos = line.indexOf('=', pos + 1); pos = line.indexOf('=', pos + 1);
} while ( pos > 0 && line.charAt(pos - 1) == '\\' ); } while ( pos > 0 && line.charAt(pos - 1) == '\\' );
if ( pos > 0 ) try { if ( pos > 0 ) try {
String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out); final String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);
String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out); final String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);
//System.out.println("key = " + key + ", value = " + value); //System.out.println("key = " + key + ", value = " + value);
props.put(key, value); props.put(key, value);
} catch (final IndexOutOfBoundsException e) { } catch (final IndexOutOfBoundsException e) {
@ -606,7 +602,7 @@ public final class FileUtils {
} }
public static Map<String, String> table(final byte[] a) { public static Map<String, String> table(final byte[] a) {
if (a == null) return new ConcurrentHashMap<String, String>(); if (a == null) return new ConcurrentHashMap<>();
//System.out.println("***TABLE: a.size = " + a.length); //System.out.println("***TABLE: a.size = " + a.length);
return table(strings(a)); return table(strings(a));
} }
@ -627,7 +623,7 @@ public final class FileUtils {
*/ */
public static ArrayList<String> getListArray(final File listFile) { public static ArrayList<String> getListArray(final File listFile) {
String line; String line;
final ArrayList<String> list = new ArrayList<String>(); final ArrayList<String> list = new ArrayList<>();
BufferedReader br = null; BufferedReader br = null;
try { try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile), StandardCharsets.UTF_8)); br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile), StandardCharsets.UTF_8));
@ -642,7 +638,7 @@ public final class FileUtils {
try { try {
br.close(); br.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile); ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
} }
} }
} }
@ -711,7 +707,7 @@ public final class FileUtils {
try { try {
br.close(); br.close();
} catch (final Exception e ) { } catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile); ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
} }
} }
} }
@ -760,7 +756,7 @@ public final class FileUtils {
* @return array of file names * @return array of file names
*/ */
public static List<String> getDirListing(final File dir, final String filter) { public static List<String> getDirListing(final File dir, final String filter) {
final List<String> ret = new LinkedList<String>(); final List<String> ret = new LinkedList<>();
File[] fileList; File[] fileList;
if ( dir != null ) { if ( dir != null ) {
if ( !dir.exists() ) { if ( !dir.exists() ) {
@ -781,39 +777,39 @@ public final class FileUtils {
public static ArrayList<File> getDirsRecursive(final File dir, final String notdir) { public static ArrayList<File> getDirsRecursive(final File dir, final String notdir) {
return getDirsRecursive(dir, notdir, true); return getDirsRecursive(dir, notdir, true);
} }
/** /**
* @param sourceDir source directory. Must be not null. * @param sourceDir source directory. Must be not null.
* @param notdir name of dir to exlcude. Can be null * @param notdir name of dir to exlcude. Can be null
* @param fileNameFilter filter to apply on file names. Can be null. * @param fileNameFilter filter to apply on file names. Can be null.
* @return list of all files passing fileFilter under sourceDir including sub directories * @return list of all files passing fileFilter under sourceDir including sub directories
*/ */
public static List<File> getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) { public static List<File> getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) {
List<File> dirList = getDirsRecursive(sourceDir, final List<File> dirList = getDirsRecursive(sourceDir,
notdir); notdir);
dirList.add(sourceDir); dirList.add(sourceDir);
List<File> files = new ArrayList<>(); final List<File> files = new ArrayList<>();
for (final File dir : dirList) { for (final File dir : dirList) {
Collections.addAll(files, dir.listFiles(fileNameFilter)); Collections.addAll(files, dir.listFiles(fileNameFilter));
} }
return files; return files;
} }
/** /**
* Returns a List of all dirs and subdirs as File Objects Warning: untested * Returns a List of all dirs and subdirs as File Objects Warning: untested
*/ */
private static ArrayList<File> getDirsRecursive( private static ArrayList<File> getDirsRecursive(
final File dir, final File dir,
final String notdir, final String notdir,
final boolean excludeDotfiles) { final boolean excludeDotfiles) {
final File[] dirList = dir.listFiles(); final File[] dirList = dir.listFiles();
final ArrayList<File> resultList = new ArrayList<File>(); final ArrayList<File> resultList = new ArrayList<>();
ArrayList<File> recursive; ArrayList<File> recursive;
Iterator<File> iter; Iterator<File> iter;
for ( int i = 0; i < dirList.length; i++ ) { for ( int i = 0; i < dirList.length; i++ ) {
if ( dirList[i].isDirectory() if ( dirList[i].isDirectory()
&& (!excludeDotfiles || !dirList[i].getName().startsWith(".")) && (!excludeDotfiles || !dirList[i].getName().startsWith("."))
&& !dirList[i].getName().equals(notdir) ) { && !dirList[i].getName().equals(notdir) ) {
resultList.add(dirList[i]); resultList.add(dirList[i]);
recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles); recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles);
iter = recursive.iterator(); iter = recursive.iterator();
@ -927,12 +923,12 @@ public final class FileUtils {
// create the temp file // create the temp file
final File tempFile = final File tempFile =
File.createTempFile( File.createTempFile(
parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName), parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),
(!fileExt.isEmpty()) ? "." + fileExt : fileExt); (!fileExt.isEmpty()) ? "." + fileExt : fileExt);
return tempFile; return tempFile;
} }
/** /**
* delete files and directories if a directory is not empty, delete also everything inside because * delete files and directories if a directory is not empty, delete also everything inside because
* deletion sometimes fails on windows, there is also a windows exec included * deletion sometimes fails on windows, there is also a windows exec included
@ -973,7 +969,7 @@ public final class FileUtils {
break; break;
} }
} }
*/ */
if ( path.exists() ) { if ( path.exists() ) {
path.deleteOnExit(); path.deleteOnExit();
String p = ""; String p = "";
@ -986,6 +982,7 @@ public final class FileUtils {
// deleting files on windows sometimes does not work with java // deleting files on windows sometimes does not work with java
try { try {
final String command = "cmd /C del /F /Q \"" + p + "\""; final String command = "cmd /C del /F /Q \"" + p + "\"";
@SuppressWarnings("deprecation")
final Process r = Runtime.getRuntime().exec(command); final Process r = Runtime.getRuntime().exec(command);
if ( r == null ) { if ( r == null ) {
ConcurrentLog.severe("FileUtils", "cannot execute command: " + command); ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);
@ -1002,7 +999,7 @@ public final class FileUtils {
} }
} }
} }
/** /**
* Checks if a certain file is in a given directory. * Checks if a certain file is in a given directory.
* @param file the file to check * @param file the file to check
@ -1010,9 +1007,9 @@ public final class FileUtils {
* @return true if file is contained in directory * @return true if file is contained in directory
*/ */
public static boolean isInDirectory(final File file, final File directory) { public static boolean isInDirectory(final File file, final File directory) {
boolean inDirectory; boolean inDirectory;
try { try {
inDirectory = ( inDirectory = (
directory != null directory != null
@ -1024,10 +1021,10 @@ public final class FileUtils {
} catch (final IOException e) { } catch (final IOException e) {
inDirectory = false; inDirectory = false;
} }
return inDirectory; return inDirectory;
} }
/** /**
* Auto-detect the charset of content in a stream. * Auto-detect the charset of content in a stream.
* Used code from http://jchardet.sourceforge.net/. * Used code from http://jchardet.sourceforge.net/.
@ -1040,28 +1037,28 @@ public final class FileUtils {
public static List<String> detectCharset(final InputStream inStream) throws IOException { public static List<String> detectCharset(final InputStream inStream) throws IOException {
// auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.html // auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.html
List<String> result; List<String> result;
nsDetector det = new nsDetector(nsPSMDetector.ALL); final nsDetector det = new nsDetector(nsPSMDetector.ALL);
byte[] buf = new byte[1024] ; final byte[] buf = new byte[1024] ;
int len; int len;
boolean done = false ; boolean done = false ;
boolean isAscii = true ; boolean isAscii = true ;
while ((len = inStream.read(buf,0,buf.length)) != -1) { while ((len = inStream.read(buf,0,buf.length)) != -1) {
if (isAscii) { if (isAscii) {
isAscii = det.isAscii(buf,len); isAscii = det.isAscii(buf,len);
} }
if (!isAscii && !done) { if (!isAscii && !done) {
done = det.DoIt(buf,len, false); done = det.DoIt(buf,len, false);
} }
} det.DataEnd(); } det.DataEnd();
result = new ArrayList<>(); result = new ArrayList<>();
if (isAscii) { if (isAscii) {
result.add(StandardCharsets.US_ASCII.name()); result.add(StandardCharsets.US_ASCII.name());
} else { } else {
for (String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch" for (final String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"
} }
return result; return result;
} }
/** /**
* Because the checking of very large files for their charset may take some time, we do this concurrently in this method * Because the checking of very large files for their charset may take some time, we do this concurrently in this method
* This method does not return anything but it logs an info line if the charset is a good choice * This method does not return anything but it logs an info line if the charset is a good choice
@ -1071,19 +1068,19 @@ public final class FileUtils {
* @param concurrent if this shall run concurrently * @param concurrent if this shall run concurrently
*/ */
public static void checkCharset(final File file, final String givenCharset, final boolean concurrent) { public static void checkCharset(final File file, final String givenCharset, final boolean concurrent) {
Thread t = new Thread("FileUtils.checkCharset") { final Thread t = new Thread("FileUtils.checkCharset") {
@Override @Override
public void run() { public void run() {
try (final FileInputStream fileStream = new FileInputStream(file); try (final FileInputStream fileStream = new FileInputStream(file);
final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources
List<String> charsets = FileUtils.detectCharset(imp); final List<String> charsets = FileUtils.detectCharset(imp);
if (charsets.contains(givenCharset)) { if (charsets.contains(givenCharset)) {
ConcurrentLog.info("checkCharset", "appropriate charset '" + givenCharset + "' for import of " + file + ", is part one detected " + charsets); ConcurrentLog.info("checkCharset", "appropriate charset '" + givenCharset + "' for import of " + file + ", is part one detected " + charsets);
} else { } else {
ConcurrentLog.warn("checkCharset", "possibly wrong charset '" + givenCharset + "' for import of " + file + ", use one of " + charsets); ConcurrentLog.warn("checkCharset", "possibly wrong charset '" + givenCharset + "' for import of " + file + ", use one of " + charsets);
} }
} catch (IOException e) {} } catch (final IOException e) {}
} }
}; };
if (concurrent) t.start(); else t.run(); if (concurrent) t.start(); else t.run();

272
source/net/yacy/kelondro/util/OS.java
Unescape View File

@ -41,7 +41,7 @@ import net.yacy.server.serverCore;
public final class OS { public final class OS {
// constants for system identification // constants for system identification
public enum System { public enum System {
MacOSC, // 'classic' Mac OS 7.6.1/8.*/9.* MacOSC, // 'classic' Mac OS 7.6.1/8.*/9.*
MacOSX, // all Mac OS X MacOSX, // all Mac OS X
@ -50,125 +50,129 @@ public final class OS {
Unknown; // any other system Unknown; // any other system
} }
// constants for file type identification (Mac only) // constants for file type identification (Mac only)
public static final String blankTypeString = "____"; public static final String blankTypeString = "____";
// system-identification statics // system-identification statics
private static final System systemOS; private static final System systemOS;
public static final boolean isMacArchitecture; public static final boolean isMacArchitecture;
private static final boolean isUnixFS; private static final boolean isUnixFS;
public static final boolean canExecUnix; public static final boolean canExecUnix;
public static final boolean isWindows; public static final boolean isWindows;
public static final boolean isWin32; public static final boolean isWin32;
// calculated system constants // calculated system constants
public static int maxPathLength = 65535; public static int maxPathLength = 65535;
// Macintosh-specific statics // Macintosh-specific statics
public static final Map<String, String> macFSTypeCache = new HashMap<String, String>(); public static final Map<String, String> macFSTypeCache = new HashMap<>();
public static final Map<String, String> macFSCreatorCache = new HashMap<String, String>(); public static final Map<String, String> macFSCreatorCache = new HashMap<>();
// static initialization // static initialization
static { static {
// check operation system type // check operation system type
final Properties sysprop = java.lang.System.getProperties(); final Properties sysprop = java.lang.System.getProperties();
final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT); final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
if (sysname.startsWith("mac os x")) systemOS = System.MacOSX; if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;
else if (sysname.startsWith("mac os")) systemOS = System.MacOSC; else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;
else if (sysname.startsWith("windows")) systemOS = System.Windows; else if (sysname.startsWith("windows")) systemOS = System.Windows;
else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix; else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;
else systemOS = System.Unknown; else systemOS = System.Unknown;
isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX)); isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));
isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix)); isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));
canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows)))); canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));
isWindows = (systemOS == System.Windows); isWindows = (systemOS == System.Windows);
isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86")); isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));
// set up maximum path length according to system // set up maximum path length according to system
if (isWindows) maxPathLength = 255; else maxPathLength = 65535; if (isWindows) maxPathLength = 255; else maxPathLength = 65535;
} }
public static String infoString() { public static String infoString() {
String s = "System="; String s = "System=";
if (systemOS == System.Unknown) s += "unknown"; if (systemOS == System.Unknown) s += "unknown";
else if (systemOS == System.MacOSC) s += "Mac OS Classic"; else if (systemOS == System.MacOSC) s += "Mac OS Classic";
else if (systemOS == System.MacOSX) s += "Mac OS X"; else if (systemOS == System.MacOSX) s += "Mac OS X";
else if (systemOS == System.Unix) s += "Unix/Linux"; else if (systemOS == System.Unix) s += "Unix/Linux";
else if (systemOS == System.Windows) s += "Windows"; else if (systemOS == System.Windows) s += "Windows";
else s += "unknown"; else s += "unknown";
if (isMacArchitecture) s += ", Mac System Architecture"; if (isMacArchitecture) s += ", Mac System Architecture";
if (isUnixFS) s += ", has Unix-like File System"; if (isUnixFS) s += ", has Unix-like File System";
if (canExecUnix) s += ", can execute Unix-Shell Commands"; if (canExecUnix) s += ", can execute Unix-Shell Commands";
return s; return s;
} }
/** generates a 2-character string containing information about the OS-type*/ /** generates a 2-character string containing information about the OS-type*/
public static String infoKey() { public static String infoKey() {
String s = ""; String s = "";
if (systemOS == System.Unknown) s += "o"; if (systemOS == System.Unknown) s += "o";
else if (systemOS == System.MacOSC) s += "c"; else if (systemOS == System.MacOSC) s += "c";
else if (systemOS == System.MacOSX) s += "x"; else if (systemOS == System.MacOSX) s += "x";
else if (systemOS == System.Unix) s += "u"; else if (systemOS == System.Unix) s += "u";
else if (systemOS == System.Windows) s += "w"; else if (systemOS == System.Windows) s += "w";
else s += "o"; else s += "o";
if (isMacArchitecture) s += "m"; if (isMacArchitecture) s += "m";
if (isUnixFS) s += "f"; if (isUnixFS) s += "f";
if (canExecUnix) s += "e"; if (canExecUnix) s += "e";
return s; return s;
} }
public static void deployScript(final File scriptFile, final String theScript) throws IOException {
FileUtils.copy(UTF8.getBytes(theScript), scriptFile);
if(!isWindows){ // set executable
try {
Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
throw new IOException(e.getMessage());
}
}
}
/** @SuppressWarnings("deprecation")
* use a hack to get the current process PID public static void deployScript(final File scriptFile, final String theScript) throws IOException {
* @return the PID of the current java process or -1 if the PID cannot be obtained FileUtils.copy(UTF8.getBytes(theScript), scriptFile);
*/ if(!isWindows){ // set executable
public static int getPID() { try {
Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
throw new IOException(e.getMessage());
}
}
}
/**
* use a hack to get the current process PID
* @return the PID of the current java process or -1 if the PID cannot be obtained
*/
public static int getPID() {
final String pids = ManagementFactory.getRuntimeMXBean().getName(); final String pids = ManagementFactory.getRuntimeMXBean().getName();
final int p = pids.indexOf('@'); final int p = pids.indexOf('@');
return p >= 0 ? NumberTools.parseIntDecSubstring(pids, 0, p) : -1; return p >= 0 ? NumberTools.parseIntDecSubstring(pids, 0, p) : -1;
} }
public static void execAsynchronous(final File scriptFile) throws IOException {
// runs a script as separate thread
String starterFileExtension = null;
String script = null;
if(isWindows){
starterFileExtension = ".starter.bat";
// use /K to debug, /C for release
script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";
} else { // unix/linux
starterFileExtension = ".starter.sh";
script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;
}
final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);
deployScript(starterFile, script);
try {
Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
throw new IOException(e.getMessage());
}
FileUtils.deletedelete(starterFile);
}
@SuppressWarnings("deprecation")
public static void execAsynchronous(final File scriptFile) throws IOException {
// runs a script as separate thread
String starterFileExtension = null;
String script = null;
if(isWindows){
starterFileExtension = ".starter.bat";
// use /K to debug, /C for release
script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";
} else { // unix/linux
starterFileExtension = ".starter.sh";
script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;
}
final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);
deployScript(starterFile, script);
try {
Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
throw new IOException(e.getMessage());
}
FileUtils.deletedelete(starterFile);
}
@SuppressWarnings("deprecation")
public static List<String> execSynchronous(final String command) throws IOException { public static List<String> execSynchronous(final String command) throws IOException {
// runs a unix/linux command and returns output as Vector of Strings // runs a unix/linux command and returns output as Vector of Strings
// this method blocks until the command is executed // this method blocks until the command is executed
final Process p = Runtime.getRuntime().exec(command); final Process p = Runtime.getRuntime().exec(command);
return readStreams(p); return readStreams(p);
} }
@SuppressWarnings("deprecation")
public static List<String> execSynchronous(final String[] command) throws IOException { public static List<String> execSynchronous(final String[] command) throws IOException {
// runs a unix/linux command and returns output as Vector of Strings // runs a unix/linux command and returns output as Vector of Strings
// this method blocks until the command is executed // this method blocks until the command is executed
@ -184,40 +188,40 @@ public final class OS {
*/ */
public static List<String> readStreams(final Process p) throws IOException { public static List<String> readStreams(final Process p) throws IOException {
String line; String line;
final List<String> output = new ArrayList<>(); final List<String> output = new ArrayList<>();
if(p == null) { if(p == null) {
return output; return output;
} }
try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream()); try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());
final BufferedReader in = new BufferedReader(streamReader);) { final BufferedReader in = new BufferedReader(streamReader);) {
while ((line = in.readLine()) != null) { while ((line = in.readLine()) != null) {
output.add(line); output.add(line);
} }
} }
try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream()); try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());
final BufferedReader in = new BufferedReader(streamReader);) { final BufferedReader in = new BufferedReader(streamReader);) {
while ((line = in.readLine()) != null) { while ((line = in.readLine()) != null) {
output.add(line); output.add(line);
} }
} }
return output; return output;
} }
public static void main(final String[] args) { public static void main(final String[] args) {
try { try {
List<String> v = execSynchronous("/usr/local/bin/wkhtmltoimage"); final List<String> v = execSynchronous("/usr/local/bin/wkhtmltoimage");
for (String r: v) java.lang.System.out.println(r); for (final String r: v) java.lang.System.out.println(r);
} catch (IOException e) { } catch (final IOException e) {
} }
/* /*
if (args[0].equals("-m")) { if (args[0].equals("-m")) {
java.lang.System.out.println("Maximum possible memory: " + Integer.toString(getWin32MaxHeap()) + "m"); java.lang.System.out.println("Maximum possible memory: " + Integer.toString(getWin32MaxHeap()) + "m");
} }
*/ */
} }
} }

230
source/net/yacy/server/http/HTTPDProxyHandler.java
Unescape View File

@ -37,7 +37,7 @@
support gzip-ed encoding. We also do not support unrealistic support gzip-ed encoding. We also do not support unrealistic
'expires' values that would force a cache to be flushed immediately 'expires' values that would force a cache to be flushed immediately
pragma non-cache attributes are supported pragma non-cache attributes are supported
*/ */
package net.yacy.server.http; package net.yacy.server.http;
@ -67,6 +67,7 @@ import java.util.logging.FileHandler;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.LogManager; import java.util.logging.LogManager;
import java.util.logging.Logger; import java.util.logging.Logger;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
@ -89,6 +90,7 @@ import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
@SuppressWarnings("deprecation")
public final class HTTPDProxyHandler { public final class HTTPDProxyHandler {
@ -108,7 +110,7 @@ public final class HTTPDProxyHandler {
// creating a logger // creating a logger
private static final ConcurrentLog log = new ConcurrentLog("PROXY"); private static final ConcurrentLog log = new ConcurrentLog("PROXY");
/** /**
* Do logging configuration for special proxy access log file * Do logging configuration for special proxy access log file
*/ */
static { static {
@ -117,77 +119,77 @@ public final class HTTPDProxyHandler {
sb = Switchboard.getSwitchboard(); sb = Switchboard.getSwitchboard();
if (sb != null) { if (sb != null) {
// set timeout // set timeout
timeout = sb.getConfigInt("proxy.clientTimeout", 60000); timeout = sb.getConfigInt("proxy.clientTimeout", 60000);
// do logger initialization // do logger initialization
try { try {
log.info("Configuring proxy access logging ..."); log.info("Configuring proxy access logging ...");
// getting the logging manager // getting the logging manager
final LogManager manager = LogManager.getLogManager(); final LogManager manager = LogManager.getLogManager();
final String className = HTTPDProxyHandler.class.getName(); final String className = HTTPDProxyHandler.class.getName();
// determining if proxy access logging is enabled // determining if proxy access logging is enabled
final String enabled = manager.getProperty(className + ".logging.enabled"); final String enabled = manager.getProperty(className + ".logging.enabled");
if ("true".equalsIgnoreCase(enabled)) { if ("true".equalsIgnoreCase(enabled)) {
// reading out some needed configuration properties // reading out some needed configuration properties
int limit = 1024*1024, count = 20; int limit = 1024*1024, count = 20;
String pattern = manager.getProperty(className + ".logging.FileHandler.pattern"); String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");
if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log"; if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";
// make pattern absolute // make pattern absolute
if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath(); if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();
final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit"); final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");
if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {} if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}
final String countStr = manager.getProperty(className + ".logging.FileHandler.count"); final String countStr = manager.getProperty(className + ".logging.FileHandler.count");
if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {} if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}
// creating the proxy access logger // creating the proxy access logger
final Logger proxyLogger = Logger.getLogger("PROXY.access"); final Logger proxyLogger = Logger.getLogger("PROXY.access");
proxyLogger.setUseParentHandlers(false); proxyLogger.setUseParentHandlers(false);
proxyLogger.setLevel(Level.FINEST); proxyLogger.setLevel(Level.FINEST);
final FileHandler txtLog = new FileHandler(pattern, limit, count, true); final FileHandler txtLog = new FileHandler(pattern, limit, count, true);
txtLog.setFormatter(new ProxyLogFormatter()); txtLog.setFormatter(new ProxyLogFormatter());
txtLog.setLevel(Level.FINEST); txtLog.setLevel(Level.FINEST);
proxyLogger.addHandler(txtLog); proxyLogger.addHandler(txtLog);
log.info("Proxy access logging configuration done." + log.info("Proxy access logging configuration done." +
"\n\tFilename: " + pattern + "\n\tFilename: " + pattern +
"\n\tLimit: " + limitStr + "\n\tLimit: " + limitStr +
"\n\tCount: " + countStr); "\n\tCount: " + countStr);
} else { } else {
log.info("Proxy access logging is deactivated."); log.info("Proxy access logging is deactivated.");
}
} catch (final Exception e) {
log.severe("Unable to configure proxy access logging.",e);
} }
} catch (final Exception e) {
log.severe("Unable to configure proxy access logging.",e);
}
// load the yellow-list // load the yellow-list
final String f = sb.getConfig("proxyYellowList", null); final String f = sb.getConfig("proxyYellowList", null);
if (f != null) { if (f != null) {
yellowList = FileUtils.loadList(new File(f)); yellowList = FileUtils.loadList(new File(f));
log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries"); log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
} else { } else {
yellowList = null; yellowList = null;
} }
final String redirectorPath = sb.getConfig("externalRedirector", ""); final String redirectorPath = sb.getConfig("externalRedirector", "");
if (redirectorPath.length() > 0 && !redirectorEnabled) { if (redirectorPath.length() > 0 && !redirectorEnabled) {
try { try {
redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null); redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);
redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream()); redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream())); redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));
redirectorEnabled=true; redirectorEnabled=true;
} catch (final IOException e) { } catch (final IOException e) {
System.out.println("redirector not Found"); System.out.println("redirector not Found");
}
} }
}
} else { } else {
yellowList = null; yellowList = null;
} }
} }
@ -342,25 +344,25 @@ public final class HTTPDProxyHandler {
// case 1 and case 3 // case 1 and case 3
if (cachedResponseHeader == null) { if (cachedResponseHeader == null) {
if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web"); if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web");
fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent); fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
} else { } else {
final Request request = new Request( final Request request = new Request(
null, null,
url, url,
requestHeader.referer() == null ? null : requestHeader.referer().hash(), requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"", "",
cachedResponseHeader.lastModified(), cachedResponseHeader.lastModified(),
sb.crawler.defaultProxyProfile.handle(), sb.crawler.defaultProxyProfile.handle(),
0, 0,
sb.crawler.defaultProxyProfile.timezoneOffset()); sb.crawler.defaultProxyProfile.timezoneOffset());
final Response response = new Response( final Response response = new Response(
request, request,
requestHeader, requestHeader,
cachedResponseHeader, cachedResponseHeader,
sb.crawler.defaultProxyProfile, sb.crawler.defaultProxyProfile,
true, true,
null null
); );
final byte[] cacheContent = Cache.getContent(url.hash()); final byte[] cacheContent = Cache.getContent(url.hash());
if (cacheContent != null && response.isFreshForProxy()) { if (cacheContent != null && response.isFreshForProxy()) {
if (log.isFinest()) log.finest(reqID + " fulfill request from cache"); if (log.isFinest()) log.finest(reqID + " fulfill request from cache");
@ -433,9 +435,9 @@ public final class HTTPDProxyHandler {
// remove yacy-subdomain-path, when accessing /env // remove yacy-subdomain-path, when accessing /env
if ((yAddress != null) if ((yAddress != null)
&& (remotePath.startsWith("/env")) && (remotePath.startsWith("/env"))
&& (yAddress.indexOf('/') != -1) && (yAddress.indexOf('/') != -1)
) yAddress = yAddress.substring(0, yAddress.indexOf('/')); ) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
modifyProxyHeaders(requestHeader, httpVer); modifyProxyHeaders(requestHeader, httpVer);
@ -448,17 +450,17 @@ public final class HTTPDProxyHandler {
try (final HTTPClient client = new HTTPClient(agent, timeout)) { try (final HTTPClient client = new HTTPClient(agent, timeout)) {
client.setHeader(requestHeader.entrySet()); client.setHeader(requestHeader.entrySet());
client.setRedirecting(false); client.setRedirecting(false);
client.GET(getUrl, false); client.GET(getUrl, false);
if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine()); if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
int statusCode = client.getHttpResponse().getStatusLine().getStatusCode(); final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders()); final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
// determine if it's an internal error of the httpc // determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) { if (responseHeader.isEmpty()) {
throw new Exception(client.getHttpResponse().getStatusLine().toString()); throw new Exception(client.getHttpResponse().getStatusLine().toString());
} }
ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond); final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);
// the cache does either not exist or is (supposed to be) stale // the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1; long sizeBeforeDelete = -1;
@ -475,19 +477,19 @@ public final class HTTPDProxyHandler {
// reserver cache entry // reserver cache entry
final Request request = new Request( final Request request = new Request(
null, null,
url, url,
requestHeader.referer() == null ? null : requestHeader.referer().hash(), requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"", "",
responseHeader.lastModified(), responseHeader.lastModified(),
sb.crawler.defaultProxyProfile.handle(), sb.crawler.defaultProxyProfile.handle(),
0, 0,
sb.crawler.defaultProxyProfile.timezoneOffset()); sb.crawler.defaultProxyProfile.timezoneOffset());
// handle incoming cookies // handle incoming cookies
handleIncomingCookies(responseHeader, host, ip); handleIncomingCookies(responseHeader, host, ip);
// prepareResponseHeader(responseHeader, res.getHttpVer()); // prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString()); prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client // sending the respond header back to the client
@ -506,7 +508,7 @@ public final class HTTPDProxyHandler {
if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) { if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
OutputStream outStream = chunkedOut != null ? chunkedOut : respond; final OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
final Response response = new Response( final Response response = new Response(
request, request,
requestHeader, requestHeader,
@ -514,7 +516,7 @@ public final class HTTPDProxyHandler {
sb.crawler.defaultProxyProfile, sb.crawler.defaultProxyProfile,
true, true,
null null
); );
final String storeError = response.shallStoreCacheForProxy(); final String storeError = response.shallStoreCacheForProxy();
final boolean storeHTCache = response.profile().storeHTCache(); final boolean storeHTCache = response.profile().storeHTCache();
final String supportError = TextParser.supports(response.url(), response.getMimeType()); final String supportError = TextParser.supports(response.url(), response.getMimeType());
@ -530,14 +532,14 @@ public final class HTTPDProxyHandler {
* c) the content should be indexed * c) the content should be indexed
*/ */
((storeHTCache) || (supportError != null)) ((storeHTCache) || (supportError != null))
) { ) {
// we don't write actually into a file, only to RAM, and schedule writing the file. // we don't write actually into a file, only to RAM, and schedule writing the file.
// int l = res.getResponseHeader().size(); // int l = res.getResponseHeader().size();
final int l = responseHeader.size(); final int l = responseHeader.size();
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l); final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream}); final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
// FileUtils.copy(res.getDataAsStream(), toClientAndMemory); // FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
client.writeTo(toClientAndMemory); client.writeTo(toClientAndMemory);
// cached bytes // cached bytes
byte[] cacheArray; byte[] cacheArray;
@ -627,7 +629,7 @@ public final class HTTPDProxyHandler {
final ResponseHeader cachedResponseHeader, final ResponseHeader cachedResponseHeader,
final byte[] cacheEntry, final byte[] cacheEntry,
OutputStream respond OutputStream respond
) throws IOException { ) throws IOException {
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER); final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
final String clienthttpVer; // the http version of the client connection final String clienthttpVer; // the http version of the client connection
@ -721,12 +723,12 @@ public final class HTTPDProxyHandler {
requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader)); requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader));
} }
// only gzip-encoding is supported, remove other encodings (e. g. deflate) // only gzip-encoding is supported, remove other encodings (e. g. deflate)
if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) { if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) {
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP); requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
} else { } else {
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, ""); requestHeader.put(HeaderFramework.ACCEPT_ENCODING, "");
} }
addXForwardedForHeader(conProp, requestHeader); addXForwardedForHeader(conProp, requestHeader);
} }
@ -782,7 +784,7 @@ public final class HTTPDProxyHandler {
if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) { if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {
forceConnectionClose(conProp); forceConnectionClose(conProp);
} else { } else {
chunkedOut = new ChunkedOutputStream(respond); chunkedOut = new ChunkedOutputStream(respond);
} }
responseHeader.remove(HeaderFramework.CONTENT_LENGTH); responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
} }
@ -933,12 +935,12 @@ public final class HTTPDProxyHandler {
errorMessage = "IP address of the destination host could not be determined"; errorMessage = "IP address of the destination host could not be determined";
} }
} else if ((exceptionMsg != null) && } else if ((exceptionMsg != null) &&
( (
(exceptionMsg.indexOf("socket write error",0)>=0) || (exceptionMsg.indexOf("socket write error",0)>=0) ||
(exceptionMsg.indexOf("Read timed out",0) >= 0) || (exceptionMsg.indexOf("Read timed out",0) >= 0) ||
(exceptionMsg.indexOf("Broken pipe",0) >= 0) || (exceptionMsg.indexOf("Broken pipe",0) >= 0) ||
(exceptionMsg.indexOf("server has closed connection",0) >= 0) (exceptionMsg.indexOf("server has closed connection",0) >= 0)
)) { )) {
errorMessage = exceptionMsg; errorMessage = exceptionMsg;
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} else { } else {
@ -983,7 +985,7 @@ public final class HTTPDProxyHandler {
final serverObjects detailedErrorMsgMap = new serverObjects(); final serverObjects detailedErrorMsgMap = new serverObjects();
// generic toplevel domains // generic toplevel domains
final HashSet<String> topLevelDomains = new HashSet<String>(Arrays.asList(new String[]{ final HashSet<String> topLevelDomains = new HashSet<>(Arrays.asList(new String[]{
"aero", // Fluggesellschaften/Luftfahrt "aero", // Fluggesellschaften/Luftfahrt
"arpa", // Einrichtung des ARPANet "arpa", // Einrichtung des ARPANet
"biz", // Business "biz", // Business
@ -1012,8 +1014,8 @@ public final class HTTPDProxyHandler {
})); }));
// getting some connection properties // getting some connection properties
DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL); final DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
int orgHostPort = orgurl.getPort(); final int orgHostPort = orgurl.getPort();
String orgHostName = orgurl.getHost(); String orgHostName = orgurl.getHost();
if (orgHostName == null) orgHostName = "unknown"; if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase(Locale.ROOT); orgHostName = orgHostName.toLowerCase(Locale.ROOT);
@ -1025,7 +1027,7 @@ public final class HTTPDProxyHandler {
detailedErrorMsgMap.put("hostName", orgHostName); detailedErrorMsgMap.put("hostName", orgHostName);
// guessing hostnames // guessing hostnames
final HashSet<String> testHostNames = new HashSet<String>(); final HashSet<String> testHostNames = new HashSet<>();
String testHostName = null; String testHostName = null;
if (!orgHostName.startsWith("www.")) { if (!orgHostName.startsWith("www.")) {
testHostName = "www." + orgHostName; testHostName = "www." + orgHostName;
@ -1042,7 +1044,7 @@ public final class HTTPDProxyHandler {
if (addr != null) testHostNames.add(testHostName); if (addr != null) testHostNames.add(testHostName);
} }
int pos = orgHostName.lastIndexOf('.'); final int pos = orgHostName.lastIndexOf('.');
if (pos != -1) { if (pos != -1) {
final Iterator<String> iter = topLevelDomains.iterator(); final Iterator<String> iter = topLevelDomains.iterator();
while (iter.hasNext()) { while (iter.hasNext()) {
@ -1142,7 +1144,7 @@ public final class HTTPDProxyHandler {
logMessage.append(' '); logMessage.append(' ');
// Method // Method
HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST); final HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
final String requestMethod = origrequest.getMethod(); final String requestMethod = origrequest.getMethod();
logMessage.append(requestMethod); logMessage.append(requestMethod);
logMessage.append(' '); logMessage.append(' ');

75
source/net/yacy/visualization/DemoApplet.java
Unescape View File

@ -1,75 +0,0 @@
package net.yacy.visualization;
import java.applet.Applet;
import java.awt.Dimension;
import java.awt.Graphics;
@SuppressWarnings("removal")
public class DemoApplet extends Applet implements Runnable {
// can be run in eclipse with
// Run -> Run As -> Java Applet
// see http://www.javaworld.com/javaworld/jw-03-1996/jw-03-animation.html?page=3
private static final long serialVersionUID = -8230253094143014406L;
private int delay;
private Thread animator;
private RasterPlotter offGraphics;
@Override
public void init() {
final String str = getParameter("fps");
final int fps = (str != null) ? Integer.parseInt(str) : 10;
delay = (fps > 0) ? (1000 / fps) : 100;
}
@Override
public void start() {
animator = new Thread(this);
animator.start();
}
@Override
public void run() {
while (Thread.currentThread() == animator) {
final long time = System.currentTimeMillis();
repaint();
try {
Thread.sleep(delay - System.currentTimeMillis() + time);
} catch (final InterruptedException e) {
break;
}
}
}
@Override
public void stop() {
animator = null;
}
@Override
public void update(final Graphics g) {
final Dimension d = getSize();
offGraphics = new RasterPlotter(d.width, d.height, RasterPlotter.DrawMode.MODE_REPLACE, "FFFFFF");
paintFrame(offGraphics);
g.drawImage(offGraphics.getImage(), 0, 0, null);
}
@Override
public void paint(final Graphics g) {
if (offGraphics != null) {
g.drawImage(offGraphics.getImage(), 0, 0, null);
}
}
public void paintFrame(final RasterPlotter m) {
RasterPlotter.demoPaint(m);
final int y = (int) (System.currentTimeMillis() / 10 % 300);
m.setColor(RasterPlotter.GREY);
PrintTool.print(m, 0, y, 0, "Hello World", -1, 100);
}
}
Write Preview
Loading…
Cancel
Save