boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed/suppressed more warnings

Browse Source
pull/533/head
Michael Peter Christen 2 years ago
parent 51cf17d252
commit 1893661ee4
8 changed files with 723 additions and 789 deletions
Show Stats Download Patch File Download Diff File

640
source/net/yacy/cora/util/Html2Image.java
Unescape View File

@ -43,6 +43,10 @@ import javax.swing.text.html.HTMLDocument;
import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.ImageView;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
@ -51,57 +55,53 @@ import net.yacy.document.ImageParser;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.OS;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
/**
* Convert html to an copy on disk-image in a other file format
* currently (pdf and/or jpg)
*/
public class Html2Image {
// Mac
/**
* Path to wkhtmltopdf executable on Mac OS when installed using
* wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
* This can also be a path on Debian or another Gnu/Linux distribution.
*/
private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
/**
* Path to wkhtmltopdf executable on Mac OS when installed using
* wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.
* This can also be a path on Debian or another Gnu/Linux distribution.
*/
private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");
// to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip
// the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/
private final static File convertMac1 = new File("/opt/local/bin/convert");
private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert");
/* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscript
The imagemagick policy at /etc should also be checked :
if it contains a line such as <policy domain="coder" rights="none" pattern="PDF" /> it must be edited with rights="read" at minimum
*/
*/
private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create images
private final static File convertDebian = new File("/usr/bin/convert");
/**
* Path to wkhtmltopdf executable on Windows, when installed with default
* settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
* https://wkhtmltopdf.org/downloads.html
*/
private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
/**
* Path to wkhtmltopdf executable on Windows, when installed with default
* settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
* https://wkhtmltopdf.org/downloads.html
*/
private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
"C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
/** Command to use when wkhtmltopdf is included in the system Path */
private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
/** Command to use when imagemagick convert is included in the system Path */
private static final String CONVERT_COMMAND = "convert";
/**
* Path to wkhtmltopdf executable on Windows, when installed with default
* settings using wkhtmltox-n.n.n.msvc2015-win64.exe from
* https://wkhtmltopdf.org/downloads.html
*/
private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
/**
* Path to wkhtmltopdf executable on Windows, when installed with default
* settings using wkhtmltox-n.n.n.msvc2015-win32.exe from
* https://wkhtmltopdf.org/downloads.html
*/
private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(
"C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");
/** Command to use when wkhtmltopdf is included in the system Path */
private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";
/** Command to use when imagemagick convert is included in the system Path */
private static final String CONVERT_COMMAND = "convert";
private static boolean usexvfb = false;
@ -109,109 +109,111 @@ public class Html2Image {
* @return when the wkhtmltopdf command is detected as available in the system
*/
public static boolean wkhtmltopdfAvailable() {
/* Check wkhtmltopdf common installation paths and system Path */
return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
/* Check wkhtmltopdf common installation paths and system Path */
return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();
}
/**
* @return a wkhtmltopdf executable file when one can be found, null otherwise
*/
private static File wkhtmltopdfExecutable() {
File executable = null;
if(OS.isWindows) {
if(WKHTMLTOPDF_WINDOWS.exists()) {
executable = WKHTMLTOPDF_WINDOWS;
} else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
executable = WKHTMLTOPDF_WINDOWS_X86;
}
} else {
if(wkhtmltopdfMac.exists()) {
executable = wkhtmltopdfMac;
} else if(wkhtmltopdfDebian.exists()) {
executable = wkhtmltopdfDebian;
}
}
return executable;
File executable = null;
if(OS.isWindows) {
if(WKHTMLTOPDF_WINDOWS.exists()) {
executable = WKHTMLTOPDF_WINDOWS;
} else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {
executable = WKHTMLTOPDF_WINDOWS_X86;
}
} else {
if(wkhtmltopdfMac.exists()) {
executable = wkhtmltopdfMac;
} else if(wkhtmltopdfDebian.exists()) {
executable = wkhtmltopdfDebian;
}
}
return executable;
}
/**
* @return true when wkhtmltopdf is available in system path
*/
private static boolean wkhtmltopdfAvailableInPath() {
boolean available = false;
try {
final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
} catch (final IOException e) {
ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
} catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // preserve thread interrupted state
}
return available;
}
private static boolean wkhtmltopdfAvailableInPath() {
boolean available = false;
try {
@SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
} catch (final IOException e) {
ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");
} catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // preserve thread interrupted state
}
return available;
}
/**
* @return a imagemagick convert executable file when one can be found, null otherwise
*/
private static File convertExecutable() {
File executable = null;
if(!OS.isWindows) {
if(convertMac1.exists()) {
executable = convertMac1;
} else if(convertMac2.exists()) {
executable = convertMac2;
} else if(convertDebian.exists()) {
executable = convertDebian;
}
}
return executable;
File executable = null;
if(!OS.isWindows) {
if(convertMac1.exists()) {
executable = convertMac1;
} else if(convertMac2.exists()) {
executable = convertMac2;
} else if(convertDebian.exists()) {
executable = convertDebian;
}
}
return executable;
}
/**
* @return when the imagemagick convert command is detected as available in the system
*/
public static boolean convertAvailable() {
/* Check convert common installation paths and system Path */
/* Check convert common installation paths and system Path */
return convertExecutable() != null || convertAvailableInPath();
}
/**
* @return when imagemagick convert is available in system path
*/
private static boolean convertAvailableInPath() {
boolean available = false;
if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
try {
final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
} catch (final IOException e) {
ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
} catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // preserve thread interrupted state
}
}
return available;
}
/**
* Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
* wKhtmltopdf may be called multiple times with various parameters flavors in
* case of failure.
*
* @param url the URL of a web resource to fetch, render and convert to
* a pdf file. Must not be null.
* @param proxy the eventual proxy address to use. Can be null. Must be of
* the form http://host:port; use YaCy here as proxy which is
* mostly http://localhost:8090
* @param destination the destination PDF file that should be written. Must not
* be null.
* @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf
* call termination. Beyond this limit the process is killed.
* @return true when the destination file was successfully written
*/
private static boolean convertAvailableInPath() {
boolean available = false;
if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS
try {
@SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");
available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;
} catch (final IOException e) {
ConcurrentLog.fine("Html2Image", "convert is not included in system path.");
} catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // preserve thread interrupted state
}
}
return available;
}
/**
* Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.
* wKhtmltopdf may be called multiple times with various parameters flavors in
* case of failure.
*
* @param url the URL of a web resource to fetch, render and convert to
* a pdf file. Must not be null.
* @param proxy the eventual proxy address to use. Can be null. Must be of
* the form http://host:port; use YaCy here as proxy which is
* mostly http://localhost:8090
* @param destination the destination PDF file that should be written. Must not
* be null.
* @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf
* call termination. Beyond this limit the process is killed.
* @return true when the destination file was successfully written
*/
public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {
boolean success = false;
for (boolean ignoreErrors: new boolean[]{false, true}) {
for (final boolean ignoreErrors: new boolean[]{false, true}) {
success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);
if (success) break;
if (!success && proxy != null) {
@ -227,49 +229,49 @@ public class Html2Image {
}
return success;
}
/**
* Run wkhtmltopdf in a separate process to fetch and render to PDF a web
* resource.
*
* @param url the URL of a web resource to fetch, render and convert to
* a pdf file. Must not be null.
* @param proxy the eventual proxy address to use. Can be null.
* @param destination the destination PDF file that should be written. Must not
* be null.
* @param userAgent TODO: implement
* @param acceptLanguage TODO: implement
* @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
* @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf
* dedicated process termination. Beyond this limit the
* process is killed.
* @return true when the destination file was successfully written
*/
private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
final String wkhtmltopdfCmd;
final File wkhtmltopdf = wkhtmltopdfExecutable();
if(wkhtmltopdf != null) {
wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
} else if(wkhtmltopdfAvailableInPath()) {
wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
} else {
ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
return false;
}
/**
* Run wkhtmltopdf in a separate process to fetch and render to PDF a web
* resource.
*
* @param url the URL of a web resource to fetch, render and convert to
* a pdf file. Must not be null.
* @param proxy the eventual proxy address to use. Can be null.
* @param destination the destination PDF file that should be written. Must not
* be null.
* @param userAgent TODO: implement
* @param acceptLanguage TODO: implement
* @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors
* @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf
* dedicated process termination. Beyond this limit the
* process is killed.
* @return true when the destination file was successfully written
*/
private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,
final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {
final String wkhtmltopdfCmd;
final File wkhtmltopdf = wkhtmltopdfExecutable();
if(wkhtmltopdf != null) {
wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();
} else if(wkhtmltopdfAvailableInPath()) {
wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;
} else {
ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");
return false;
}
String commandline =
wkhtmltopdfCmd + " -q --title '" + url + "' " +
//acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
//(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
(proxy == null ? "" : "--proxy " + proxy + " ") +
(ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
//"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
"--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
url + " " + destination.getAbsolutePath();
wkhtmltopdfCmd + " -q --title '" + url + "' " +
//acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +
//(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +
(proxy == null ? "" : "--proxy " + proxy + " ") +
(ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...
//"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +
"--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +
url + " " + destination.getAbsolutePath();
try {
ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);
if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) {
return true;
return true;
}
// if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless servers
commandline = "xvfb-run -a " + commandline;
@ -280,46 +282,46 @@ public class Html2Image {
}
}
/**
* Run a wkhtmltopdf commandline in a separate process.
*
* @param proxy the eventual proxy address to use. Can be null.
* @param destination the destination PDF file that should be written. Must not
* be null.
* @param commandline the wkhtmltopdf command line to execute. Must not be null.
* @param maxSeconds the maximum time in seconds to wait for the process
* termination. Beyond this limit the process is killed.
* @return true when the destination file was successfully written
* @throws IOException when an unexpected error occurred
*/
private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds)
throws IOException {
final Process p = Runtime.getRuntime().exec(commandline);
try {
p.waitFor(maxSeconds, TimeUnit.SECONDS);
} catch (final InterruptedException e) {
p.destroyForcibly();
ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
Thread.currentThread().interrupt(); // Keep the thread interrupted state
return false;
}
if(p.isAlive()) {
ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
p.destroyForcibly();
return false;
}
if (p.exitValue() == 0 && destination.exists()) {
return true;
}
final List<String> messages = OS.readStreams(p);
ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
for (final String message : messages) {
ConcurrentLog.warn("Html2Image", ">> " + message);
}
return false;
}
/**
* Run a wkhtmltopdf commandline in a separate process.
*
* @param proxy the eventual proxy address to use. Can be null.
* @param destination the destination PDF file that should be written. Must not
* be null.
* @param commandline the wkhtmltopdf command line to execute. Must not be null.
* @param maxSeconds the maximum time in seconds to wait for the process
* termination. Beyond this limit the process is killed.
* @return true when the destination file was successfully written
* @throws IOException when an unexpected error occurred
*/
private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException {
@SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(commandline);
try {
p.waitFor(maxSeconds, TimeUnit.SECONDS);
} catch (final InterruptedException e) {
p.destroyForcibly();
ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);
Thread.currentThread().interrupt(); // Keep the thread interrupted state
return false;
}
if(p.isAlive()) {
ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);
p.destroyForcibly();
return false;
}
if (p.exitValue() == 0 && destination.exists()) {
return true;
}
final List<String> messages = OS.readStreams(p);
ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);
for (final String message : messages) {
ConcurrentLog.warn("Html2Image", ">> " + message);
}
return false;
}
/**
* Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75
* using internal pdf library or external command line tool on linux or mac
@ -332,34 +334,34 @@ public class Html2Image {
* @return true when the ouput image file was successfully written.
*/
public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) {
/* Deduce the ouput image format from the file extension */
String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
if(imageFormat.isEmpty()) {
/* Use JPEG as a default fallback */
imageFormat = "jpg";
}
String convertCmd = null;
final File convert = convertExecutable();
if(convert != null) {
convertCmd = convert.getAbsolutePath();
} else if(convertAvailableInPath()) {
convertCmd = CONVERT_COMMAND;
} else {
ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
}
/* Deduce the ouput image format from the file extension */
String imageFormat = MultiProtocolURL.getFileExtension(image.getName());
if(imageFormat.isEmpty()) {
/* Use JPEG as a default fallback */
imageFormat = "jpg";
}
String convertCmd = null;
final File convert = convertExecutable();
if(convert != null) {
convertCmd = convert.getAbsolutePath();
} else if(convertAvailableInPath()) {
convertCmd = CONVERT_COMMAND;
} else {
ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");
}
// convert pdf to jpg using internal pdfbox capability
if (convertCmd == null) {
try (final PDDocument pdoc = PDDocument.load(pdf);) {
BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);
return ImageIO.write(bi, imageFormat, image);
} catch (final IOException ex) {
ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
+ (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
return false;
ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"
+ (ex.getMessage() != null ? " : " + ex.getMessage() : ""));
return false;
}
}
@ -367,18 +369,18 @@ public class Html2Image {
try {
// i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg
// note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf
String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();
List<String> message = OS.execSynchronous(command);
if (image.exists()) return true;
ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command);
for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
// another try for mac: use Image Events using AppleScript in osacript commands...
// the following command overwrites a pdf with an png, so we must make a copy first
if (!OS.isMacArchitecture) return false;
File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");
org.apache.commons.io.FileUtils.copyFile(pdf, pngFile);
String[] commandx = {"osascript",
final String[] commandx = {"osascript",
"-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"",
"-e", "tell application \"Image Events\"",
"-e", "set Img to open file ImgFile",
@ -386,15 +388,15 @@ public class Html2Image {
"-e", "end tell"};
//ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx);
message = OS.execSynchronous(commandx);
for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);
// now we must read and convert this file to the target format with the target size 1024x1024
try {
File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
final File newPngFile = new File(pngFile.getAbsolutePath() + ".png");
pngFile.renameTo(newPngFile);
final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile));
if(img == null) {
/* Should not happen. If so, ImageParser.parse() should already have logged about the error */
return false;
/* Should not happen. If so, ImageParser.parse() should already have logged about the error */
return false;
}
final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
final MediaTracker mediaTracker = new MediaTracker(new Container());
@ -406,24 +408,24 @@ public class Html2Image {
ImageIO.write(bi, imageFormat, image);
newPngFile.delete();
return image.exists();
} catch (IOException e) {
} catch (final IOException e) {
ConcurrentLog.logException(e);
return false;
}
} catch (IOException e) {
} catch (final IOException e) {
e.printStackTrace();
return false;
}
}
/**
* render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported!
* @param url
* @param size
* @throws IOException
* @throws IOException
*/
public static void writeSwingImage(String url, Dimension size, File destination) throws IOException {
// set up a pane for rendering
final JEditorPane htmlPane = new JEditorPane();
htmlPane.setSize(size);
@ -434,7 +436,7 @@ public class Html2Image {
@Override
public Document createDefaultDocument() {
HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();
doc.setAsynchronousLoadPriority(-1);
return doc;
}
@ -444,7 +446,7 @@ public class Html2Image {
return new HTMLFactory() {
@Override
public View create(Element elem) {
View view = super.create(elem);
final View view = super.create(elem);
if (view instanceof ImageView) {
((ImageView) view).setLoadsSynchronously(true);
}
@ -460,23 +462,23 @@ public class Html2Image {
public void propertyChange(PropertyChangeEvent evt) {
}
});
// load the page
try {
htmlPane.setPage(url);
} catch (IOException e) {
} catch (final IOException e) {
e.printStackTrace();
}
// render the page
Dimension prefSize = htmlPane.getPreferredSize();
BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
Graphics graphics = img.getGraphics();
final Dimension prefSize = htmlPane.getPreferredSize();
final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);
final Graphics graphics = img.getGraphics();
htmlPane.setSize(prefSize);
htmlPane.paint(graphics);
ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination);
}
/**
* Test PDF or image snapshot generation for a given URL.
* @param args main arguments list:
@ -491,88 +493,88 @@ public class Html2Image {
* </li>
* </ol>
*/
public static void main(String[] args) {
final String usageMessage = "Usage : java " + Html2Image.class.getName()
+ " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";
int exitStatus = 0;
try {
if (args.length < 2) {
System.out.println("Missing required parameter(s).");
System.out.println(usageMessage);
exitStatus = 1;
return;
}
final String targetPath = args[1];
if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
if(Html2Image.wkhtmltopdfAvailable()) {
final File targetPdfFile;
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
} else if(targetPath.endsWith(".pdf")) {
targetPdfFile = new File(targetPath);
} else {
System.out.println("Unsupported output format");
System.out.println(usageMessage);
exitStatus = 1;
return;
}
if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,
"en-us,en;q=0.5", targetPdfFile, 30)) {
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
} else {
exitStatus = 1;
return;
}
}
} else {
exitStatus = 1;
return;
}
} else {
System.out.println("Unable to locate wkhtmltopdf executable on this system!");
exitStatus = 1;
return;
}
} else if ("swing".equals(args[2])) {
if(targetPath.endsWith(".pdf")) {
System.out.println("Pdf output format is not supported with swing method.");
exitStatus = 1;
return;
}
if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
System.out.println("Unsupported output format");
System.out.println(usageMessage);
exitStatus = 1;
return;
}
try {
Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
} catch (final IOException e) {
e.printStackTrace();
exitStatus = 1;
return;
}
} else {
System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
exitStatus = 1;
return;
}
} finally {
/* Shutdown running threads */
Domains.close();
try {
HTTPClient.closeConnectionManager();
} catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // restore interrupted state
}
ConcurrentLog.shutdown();
if(exitStatus != 0) {
System.exit(exitStatus);
}
}
}
public static void main(String[] args) {
final String usageMessage = "Usage : java " + Html2Image.class.getName()
+ " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";
int exitStatus = 0;
try {
if (args.length < 2) {
System.out.println("Missing required parameter(s).");
System.out.println(usageMessage);
exitStatus = 1;
return;
}
final String targetPath = args[1];
if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {
if(Html2Image.wkhtmltopdfAvailable()) {
final File targetPdfFile;
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");
} else if(targetPath.endsWith(".pdf")) {
targetPdfFile = new File(targetPath);
} else {
System.out.println("Unsupported output format");
System.out.println(usageMessage);
exitStatus = 1;
return;
}
if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,
"en-us,en;q=0.5", targetPdfFile, 30)) {
if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {
if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {
ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);
} else {
exitStatus = 1;
return;
}
}
} else {
exitStatus = 1;
return;
}
} else {
System.out.println("Unable to locate wkhtmltopdf executable on this system!");
exitStatus = 1;
return;
}
} else if ("swing".equals(args[2])) {
if(targetPath.endsWith(".pdf")) {
System.out.println("Pdf output format is not supported with swing method.");
exitStatus = 1;
return;
}
if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {
System.out.println("Unsupported output format");
System.out.println(usageMessage);
exitStatus = 1;
return;
}
try {
Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));
} catch (final IOException e) {
e.printStackTrace();
exitStatus = 1;
return;
}
} else {
System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");
exitStatus = 1;
return;
}
} finally {
/* Shutdown running threads */
Domains.close();
try {
HTTPClient.closeConnectionManager();
} catch (final InterruptedException e) {
Thread.currentThread().interrupt(); // restore interrupted state
}
ConcurrentLog.shutdown();
if(exitStatus != 0) {
System.exit(exitStatus);
}
}
}
}

19
source/net/yacy/cora/util/Memory.java
Unescape View File

@ -115,8 +115,9 @@ public class Memory {
* @return the "recent cpu usage" for the whole operating environment;
* a negative value if not available.
*/
@SuppressWarnings("deprecation")
public static double getSystemCpuLoad() {
com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
return operatingSystemMXBean.getSystemCpuLoad();
}
@ -136,13 +137,13 @@ public class Memory {
* a negative value if not available.
*/
public static double getProcessCpuLoad() {
com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
return operatingSystemMXBean.getProcessCpuLoad();
}
public static Map<String, Object> status() {
Runtime runtime = Runtime.getRuntime();
Map<String, Object> status = new LinkedHashMap<>();
final Runtime runtime = Runtime.getRuntime();
final Map<String, Object> status = new LinkedHashMap<>();
status.put("service", "Peer");
status.put("assigned_memory", runtime.maxMemory());
status.put("used_memory", runtime.totalMemory() - runtime.freeMemory());
@ -153,7 +154,7 @@ public class Memory {
status.put("load_system_load_average", Memory.getSystemLoadAverage());
status.put("load_system_cpu_load", Memory.getSystemCpuLoad());
status.put("load_process_cpu_load", Memory.getProcessCpuLoad());
YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();
final YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();
status.put("server_threads", server == null ? 0 : server.getServerThreads());
return status;
}
@ -163,7 +164,7 @@ public class Memory {
* @return the number of deadlocked threads
*/
public static long deadlocks() {
long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
if (deadlockIDs == null) return 0;
return deadlockIDs.length;
}
@ -172,10 +173,10 @@ public class Memory {
* write deadlocked threads as to the log as warning
*/
public static void logDeadlocks() {
long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();
if (deadlockIDs == null) return;
ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);
for (ThreadInfo ti : infos) {
final ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);
for (final ThreadInfo ti : infos) {
ConcurrentLog.warn("DEADLOCKREPORT", ti.toString());
}
}

29
source/net/yacy/data/list/XMLBlacklistImporter.java
Unescape View File

@ -33,7 +33,6 @@ import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;
/**
* This class provides methods to import blacklists from an XML file (see
@ -54,13 +53,15 @@ public class XMLBlacklistImporter extends DefaultHandler {
* @throws java.io.IOException if input can't be read
* @throws org.xml.sax.SAXException if XML can't be parsed
*/
@SuppressWarnings("deprecation")
public synchronized ListAccumulator parse(InputSource input) throws IOException, SAXException {
XMLReader reader = XMLReaderFactory.createXMLReader();
@SuppressWarnings("deprecation")
final XMLReader reader = org.xml.sax.helpers.XMLReaderFactory.createXMLReader();
reader.setContentHandler(this);
reader.parse(input);
return ba;
return this.ba;
}
/**
@ -104,7 +105,7 @@ public class XMLBlacklistImporter extends DefaultHandler {
*/
@Override
public void startDocument() {
ba = new ListAccumulator();
this.ba = new ListAccumulator();
}
/**
@ -127,24 +128,24 @@ public class XMLBlacklistImporter extends DefaultHandler {
public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) {
if (qName.equalsIgnoreCase("list")) {
currentListName = attributes.getValue("name");
ba.addList(currentListName);
this.currentListName = attributes.getValue("name");
this.ba.addList(this.currentListName);
int attributesLength = 0;
if ((attributesLength = attributes.getLength()) > 1) {
for (int i = 0; i < attributesLength; i++) {
if (!attributes.getQName(i).equals("name")) {
ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));
this.ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));
}
}
}
}
if (qName.equalsIgnoreCase("item")) {
lastText = new StringBuilder();
this.lastText = new StringBuilder();
}
}
/**
@ -162,7 +163,7 @@ public class XMLBlacklistImporter extends DefaultHandler {
@Override
public void endElement(final String uri, final String localName, final String qName) throws SAXException {
if (qName.equalsIgnoreCase("item")) {
ba.addEntryToCurrent(lastText.toString());
this.ba.addEntryToCurrent(this.lastText.toString());
}
}
@ -175,8 +176,8 @@ public class XMLBlacklistImporter extends DefaultHandler {
*/
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
if (lastText == null) lastText = new StringBuilder();
lastText.append(ch, start, length);
if (this.lastText == null) this.lastText = new StringBuilder();
this.lastText.append(ch, start, length);
}
}

46
source/net/yacy/gui/framework/Browser.java
Unescape View File

@ -37,7 +37,7 @@ import net.yacy.cora.util.ConcurrentLog;
public class Browser {
// constants for system identification
// constants for system identification
public static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.*
public static final int systemMacOSX = 1; // all Mac OS X
public static final int systemUnix = 2; // all Unix/Linux type systems
@ -100,7 +100,7 @@ public class Browser {
openBrowserClassic(url);
}
}
public static void openBrowserClassic(final String url) {
try {
if (systemOS == systemMacOSX) {
@ -113,12 +113,12 @@ public class Browser {
throw new RuntimeException("System unknown");
}
} catch (final Throwable e) {
ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());
ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());
}
}
private static void openBrowserMac(final String url) throws Exception {
Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});
final Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});
p.waitFor();
if (p.exitValue() != 0) {
throw new RuntimeException("Mac Exec Error: " + errorResponse(p));
@ -131,12 +131,13 @@ public class Browser {
* @throws Exception when an error occured
*/
private static void openDefaultUnixBrowser(final String url) throws Exception {
/* Use the freedesktop xdg-open to open url with the default browser.
* xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)
* It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it
* (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */
String cmd = "xdg-open " + url;
Process p = Runtime.getRuntime().exec(cmd);
/* Use the freedesktop xdg-open to open url with the default browser.
* xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)
* It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it
* (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */
final String cmd = "xdg-open " + url;
@SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(cmd);
p.waitFor();
if (p.exitValue() != 0) {
throw new RuntimeException("Unix Exec Error/xdg-open: " + errorResponse(p));
@ -152,7 +153,8 @@ public class Browser {
cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\"";
}
//cmd = "cmd.exe /c start javascript:document.location='" + url + "'";
Process p = Runtime.getRuntime().exec(cmd);
@SuppressWarnings("deprecation")
final Process p = Runtime.getRuntime().exec(cmd);
p.waitFor();
if (p.exitValue() != 0) {
throw new RuntimeException("EXEC ERROR: " + errorResponse(p));
@ -183,15 +185,15 @@ public class Browser {
* @param args
*/
public static void main(final String[] args) {
try {
if (args.length > 0 && "-u".equals(args[0])) {
openBrowser(args[1]);
} else {
System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");
}
} finally {
ConcurrentLog.shutdown();
}
System.out.println("The End!");
}
try {
if (args.length > 0 && "-u".equals(args[0])) {
openBrowser(args[1]);
} else {
System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");
}
} finally {
ConcurrentLog.shutdown();
}
System.out.println("The End!");
}
}

201
source/net/yacy/kelondro/util/FileUtils.java
Unescape View File

@ -56,15 +56,14 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.GZIPInputStream;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.storage.Files;
import net.yacy.cora.util.ConcurrentLog;
import org.apache.commons.lang.StringUtils;
import org.mozilla.intl.chardet.nsDetector;
import org.mozilla.intl.chardet.nsPSMDetector;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.storage.Files;
import net.yacy.cora.util.ConcurrentLog;
public final class FileUtils {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
@ -98,7 +97,7 @@ public final class FileUtils {
* @see #copy(File source, File dest)
*/
public static long copy(final InputStream source, final OutputStream dest, final long count)
throws IOException {
throws IOException {
assert count < 0 || count > 0 : "precondition violated: count == " + count + " (nothing to copy)";
if ( count == 0 ) {
// no bytes to copy
@ -106,15 +105,15 @@ public final class FileUtils {
}
final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];
int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
final int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
int c;
long total = 0;
long remaining;
if(count > 0) {
remaining = count;
remaining = count;
} else {
remaining = Long.MAX_VALUE;
remaining = Long.MAX_VALUE;
}
while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) {
dest.write(buffer, 0, c);
@ -158,7 +157,7 @@ public final class FileUtils {
* @throws NullPointerException when a parameter is null
*/
public static int copy(final InputStream source, final Writer dest, final Charset inputCharset)
throws IOException {
throws IOException {
final InputStreamReader reader = new InputStreamReader(source, inputCharset);
return copy(reader, dest);
}
@ -212,8 +211,8 @@ public final class FileUtils {
// an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here
//Log.logException(e);
throw new IOException(
e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),
e);
e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),
e);
}
return count;
}
@ -260,8 +259,8 @@ public final class FileUtils {
fos.close();
} catch (final Exception e ) {
ConcurrentLog.warn(
"FileUtils",
"cannot close FileOutputStream for " + dest + "! " + e.getMessage());
"FileUtils",
"cannot close FileOutputStream for " + dest + "! " + e.getMessage());
}
}
}
@ -282,17 +281,17 @@ public final class FileUtils {
* @see #copy(File source, File dest)
*/
public static void copyRange(final File source, final OutputStream dest, final int start)
throws IOException {
throws IOException {
InputStream fis = null;
try {
fis = new FileInputStream(source);
final long skipped = fis.skip(start);
if ( skipped != start ) {
throw new IllegalStateException("Unable to skip '"
+ start
+ "' bytes. Only '"
+ skipped
+ "' bytes skipped.");
+ start
+ "' bytes. Only '"
+ skipped
+ "' bytes skipped.");
}
copy(fis, dest, -1);
} finally {
@ -300,7 +299,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
}
@ -328,7 +327,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
}
@ -368,17 +367,17 @@ public final class FileUtils {
* @throws NullPointerException when source parameter is null
*/
public static byte[] read(final InputStream source) throws IOException {
byte[] content;
try {
content = read(source, -1);
} finally {
/* source input stream must be closed here in all cases */
try {
source.close();
} catch(IOException ignoredException) {
}
}
return content;
byte[] content;
try {
content = read(source, -1);
} finally {
/* source input stream must be closed here in all cases */
try {
source.close();
} catch(final IOException ignoredException) {
}
}
return content;
}
/**
@ -392,7 +391,7 @@ public final class FileUtils {
*/
public static byte[] read(final InputStream source, final int count) throws IOException {
if(count == 0) {
return new byte[0];
return new byte[0];
}
final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
copy(source, baos, count);
@ -414,7 +413,7 @@ public final class FileUtils {
try {
fis.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);
}
}
fis = null;
@ -470,17 +469,17 @@ public final class FileUtils {
return source;
}
/**
* Generate a set of strings matching each line of the given file. Lines are
* lower cased and any eventual surrounding space characters are removed. Empty
* lines and lines starting with the '#' character are ignored.
*
* @param file
* a file to load
* @return a set of strings eventually empty
*/
/**
* Generate a set of strings matching each line of the given file. Lines are
* lower cased and any eventual surrounding space characters are removed. Empty
* lines and lines starting with the '#' character are ignored.
*
* @param file
* a file to load
* @return a set of strings eventually empty
*/
public static HashSet<String> loadList(final File file) {
final HashSet<String> set = new HashSet<String>();
final HashSet<String> set = new HashSet<>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
@ -497,7 +496,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);
}
}
}
@ -516,10 +515,10 @@ public final class FileUtils {
}
public static ConcurrentHashMap<String, byte[]> loadMapB(final File f) {
ConcurrentHashMap<String, String> m = loadMap(f);
final ConcurrentHashMap<String, String> m = loadMap(f);
if (m == null) return null;
ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<String, byte[]>();
for (Map.Entry<String, String> e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));
final ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<>();
for (final Map.Entry<String, String> e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));
return mb;
}
@ -529,7 +528,7 @@ public final class FileUtils {
private final static String[] unescaped_strings_out = {"\\", "\n", "="};
public static void saveMap(final File file, final Map<String, String> props, final String comment) {
boolean err = false;
boolean err = false;
PrintWriter pw = null;
final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000));
try {
@ -550,10 +549,7 @@ public final class FileUtils {
pw.println(key + "=" + value);
}
pw.println("# EOF");
} catch (final FileNotFoundException e ) {
ConcurrentLog.warn("FileUtils", e.getMessage(), e);
err = true;
} catch (final UnsupportedEncodingException e ) {
} catch (final FileNotFoundException | UnsupportedEncodingException e ) {
ConcurrentLog.warn("FileUtils", e.getMessage(), e);
err = true;
} finally {
@ -568,10 +564,10 @@ public final class FileUtils {
// ignore
}
}
public static void saveMapB(final File file, final Map<String, byte[]> props, final String comment) {
HashMap<String, String> m = new HashMap<String, String>();
for (Map.Entry<String, byte[]> e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));
final HashMap<String, String> m = new HashMap<>();
for (final Map.Entry<String, byte[]> e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));
saveMap(file, m, comment);
}
@ -582,7 +578,7 @@ public final class FileUtils {
public static ConcurrentHashMap<String, String> table(final Iterator<String> li) {
String line;
final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<String, String>();
final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<>();
while ( li.hasNext() ) {
int pos = 0;
line = li.next().trim();
@ -594,8 +590,8 @@ public final class FileUtils {
pos = line.indexOf('=', pos + 1);
} while ( pos > 0 && line.charAt(pos - 1) == '\\' );
if ( pos > 0 ) try {
String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);
String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);
final String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);
final String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);
//System.out.println("key = " + key + ", value = " + value);
props.put(key, value);
} catch (final IndexOutOfBoundsException e) {
@ -606,7 +602,7 @@ public final class FileUtils {
}
public static Map<String, String> table(final byte[] a) {
if (a == null) return new ConcurrentHashMap<String, String>();
if (a == null) return new ConcurrentHashMap<>();
//System.out.println("***TABLE: a.size = " + a.length);
return table(strings(a));
}
@ -627,7 +623,7 @@ public final class FileUtils {
*/
public static ArrayList<String> getListArray(final File listFile) {
String line;
final ArrayList<String> list = new ArrayList<String>();
final ArrayList<String> list = new ArrayList<>();
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile), StandardCharsets.UTF_8));
@ -642,7 +638,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
}
}
}
@ -711,7 +707,7 @@ public final class FileUtils {
try {
br.close();
} catch (final Exception e ) {
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);
}
}
}
@ -760,7 +756,7 @@ public final class FileUtils {
* @return array of file names
*/
public static List<String> getDirListing(final File dir, final String filter) {
final List<String> ret = new LinkedList<String>();
final List<String> ret = new LinkedList<>();
File[] fileList;
if ( dir != null ) {
if ( !dir.exists() ) {
@ -781,39 +777,39 @@ public final class FileUtils {
public static ArrayList<File> getDirsRecursive(final File dir, final String notdir) {
return getDirsRecursive(dir, notdir, true);
}
/**
* @param sourceDir source directory. Must be not null.
* @param notdir name of dir to exlcude. Can be null
* @param fileNameFilter filter to apply on file names. Can be null.
* @return list of all files passing fileFilter under sourceDir including sub directories
* @return list of all files passing fileFilter under sourceDir including sub directories
*/
public static List<File> getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) {
List<File> dirList = getDirsRecursive(sourceDir,
notdir);
dirList.add(sourceDir);
List<File> files = new ArrayList<>();
for (final File dir : dirList) {
Collections.addAll(files, dir.listFiles(fileNameFilter));
}
return files;
final List<File> dirList = getDirsRecursive(sourceDir,
notdir);
dirList.add(sourceDir);
final List<File> files = new ArrayList<>();
for (final File dir : dirList) {
Collections.addAll(files, dir.listFiles(fileNameFilter));
}
return files;
}
/**
* Returns a List of all dirs and subdirs as File Objects Warning: untested
*/
private static ArrayList<File> getDirsRecursive(
final File dir,
final String notdir,
final boolean excludeDotfiles) {
final File dir,
final String notdir,
final boolean excludeDotfiles) {
final File[] dirList = dir.listFiles();
final ArrayList<File> resultList = new ArrayList<File>();
final ArrayList<File> resultList = new ArrayList<>();
ArrayList<File> recursive;
Iterator<File> iter;
for ( int i = 0; i < dirList.length; i++ ) {
if ( dirList[i].isDirectory()
&& (!excludeDotfiles || !dirList[i].getName().startsWith("."))
&& !dirList[i].getName().equals(notdir) ) {
&& (!excludeDotfiles || !dirList[i].getName().startsWith("."))
&& !dirList[i].getName().equals(notdir) ) {
resultList.add(dirList[i]);
recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles);
iter = recursive.iterator();
@ -927,12 +923,12 @@ public final class FileUtils {
// create the temp file
final File tempFile =
File.createTempFile(
parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),
(!fileExt.isEmpty()) ? "." + fileExt : fileExt);
File.createTempFile(
parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),
(!fileExt.isEmpty()) ? "." + fileExt : fileExt);
return tempFile;
}
/**
* delete files and directories if a directory is not empty, delete also everything inside because
* deletion sometimes fails on windows, there is also a windows exec included
@ -973,7 +969,7 @@ public final class FileUtils {
break;
}
}
*/
*/
if ( path.exists() ) {
path.deleteOnExit();
String p = "";
@ -986,6 +982,7 @@ public final class FileUtils {
// deleting files on windows sometimes does not work with java
try {
final String command = "cmd /C del /F /Q \"" + p + "\"";
@SuppressWarnings("deprecation")
final Process r = Runtime.getRuntime().exec(command);
if ( r == null ) {
ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);
@ -1002,7 +999,7 @@ public final class FileUtils {
}
}
}
/**
* Checks if a certain file is in a given directory.
* @param file the file to check
@ -1010,9 +1007,9 @@ public final class FileUtils {
* @return true if file is contained in directory
*/
public static boolean isInDirectory(final File file, final File directory) {
boolean inDirectory;
try {
inDirectory = (
directory != null
@ -1024,10 +1021,10 @@ public final class FileUtils {
} catch (final IOException e) {
inDirectory = false;
}
return inDirectory;
}
/**
* Auto-detect the charset of content in a stream.
* Used code from http://jchardet.sourceforge.net/.
@ -1040,28 +1037,28 @@ public final class FileUtils {
public static List<String> detectCharset(final InputStream inStream) throws IOException {
// auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.html
List<String> result;
nsDetector det = new nsDetector(nsPSMDetector.ALL);
byte[] buf = new byte[1024] ;
final nsDetector det = new nsDetector(nsPSMDetector.ALL);
final byte[] buf = new byte[1024] ;
int len;
boolean done = false ;
boolean isAscii = true ;
while ((len = inStream.read(buf,0,buf.length)) != -1) {
if (isAscii) {
isAscii = det.isAscii(buf,len);
isAscii = det.isAscii(buf,len);
}
if (!isAscii && !done) {
done = det.DoIt(buf,len, false);
done = det.DoIt(buf,len, false);
}
} det.DataEnd();
result = new ArrayList<>();
if (isAscii) {
result.add(StandardCharsets.US_ASCII.name());
} else {
for (String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"
for (final String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"
}
return result;
}
/**
* Because the checking of very large files for their charset may take some time, we do this concurrently in this method
* This method does not return anything but it logs an info line if the charset is a good choice
@ -1071,19 +1068,19 @@ public final class FileUtils {
* @param concurrent if this shall run concurrently
*/
public static void checkCharset(final File file, final String givenCharset, final boolean concurrent) {
Thread t = new Thread("FileUtils.checkCharset") {
final Thread t = new Thread("FileUtils.checkCharset") {
@Override
public void run() {
try (final FileInputStream fileStream = new FileInputStream(file);
final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources
List<String> charsets = FileUtils.detectCharset(imp);
try (final FileInputStream fileStream = new FileInputStream(file);
final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources
final List<String> charsets = FileUtils.detectCharset(imp);
if (charsets.contains(givenCharset)) {
ConcurrentLog.info("checkCharset", "appropriate charset '" + givenCharset + "' for import of " + file + ", is part one detected " + charsets);
} else {
ConcurrentLog.warn("checkCharset", "possibly wrong charset '" + givenCharset + "' for import of " + file + ", use one of " + charsets);
}
} catch (IOException e) {}
} catch (final IOException e) {}
}
};
if (concurrent) t.start(); else t.run();

272
source/net/yacy/kelondro/util/OS.java
Unescape View File

@ -41,7 +41,7 @@ import net.yacy.server.serverCore;
public final class OS {
// constants for system identification
// constants for system identification
public enum System {
MacOSC, // 'classic' Mac OS 7.6.1/8.*/9.*
MacOSX, // all Mac OS X
@ -50,125 +50,129 @@ public final class OS {
Unknown; // any other system
}
// constants for file type identification (Mac only)
public static final String blankTypeString = "____";
// system-identification statics
private static final System systemOS;
public static final boolean isMacArchitecture;
private static final boolean isUnixFS;
public static final boolean canExecUnix;
public static final boolean isWindows;
public static final boolean isWin32;
// calculated system constants
public static int maxPathLength = 65535;
// Macintosh-specific statics
public static final Map<String, String> macFSTypeCache = new HashMap<String, String>();
public static final Map<String, String> macFSCreatorCache = new HashMap<String, String>();
// static initialization
static {
// check operation system type
final Properties sysprop = java.lang.System.getProperties();
final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;
else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;
else if (sysname.startsWith("windows")) systemOS = System.Windows;
else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;
else systemOS = System.Unknown;
isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));
isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));
canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));
isWindows = (systemOS == System.Windows);
isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));
// set up maximum path length according to system
if (isWindows) maxPathLength = 255; else maxPathLength = 65535;
}
public static String infoString() {
String s = "System=";
if (systemOS == System.Unknown) s += "unknown";
else if (systemOS == System.MacOSC) s += "Mac OS Classic";
else if (systemOS == System.MacOSX) s += "Mac OS X";
else if (systemOS == System.Unix) s += "Unix/Linux";
else if (systemOS == System.Windows) s += "Windows";
else s += "unknown";
if (isMacArchitecture) s += ", Mac System Architecture";
if (isUnixFS) s += ", has Unix-like File System";
if (canExecUnix) s += ", can execute Unix-Shell Commands";
return s;
}
/** generates a 2-character string containing information about the OS-type*/
public static String infoKey() {
String s = "";
if (systemOS == System.Unknown) s += "o";
else if (systemOS == System.MacOSC) s += "c";
else if (systemOS == System.MacOSX) s += "x";
else if (systemOS == System.Unix) s += "u";
else if (systemOS == System.Windows) s += "w";
else s += "o";
if (isMacArchitecture) s += "m";
if (isUnixFS) s += "f";
if (canExecUnix) s += "e";
return s;
}
public static void deployScript(final File scriptFile, final String theScript) throws IOException {
FileUtils.copy(UTF8.getBytes(theScript), scriptFile);
if(!isWindows){ // set executable
try {
Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
throw new IOException(e.getMessage());
}
}
}
// constants for file type identification (Mac only)
public static final String blankTypeString = "____";
// system-identification statics
private static final System systemOS;
public static final boolean isMacArchitecture;
private static final boolean isUnixFS;
public static final boolean canExecUnix;
public static final boolean isWindows;
public static final boolean isWin32;
// calculated system constants
public static int maxPathLength = 65535;
// Macintosh-specific statics
public static final Map<String, String> macFSTypeCache = new HashMap<>();
public static final Map<String, String> macFSCreatorCache = new HashMap<>();
// static initialization
static {
// check operation system type
final Properties sysprop = java.lang.System.getProperties();
final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;
else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;
else if (sysname.startsWith("windows")) systemOS = System.Windows;
else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;
else systemOS = System.Unknown;
isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));
isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));
canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));
isWindows = (systemOS == System.Windows);
isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));
// set up maximum path length according to system
if (isWindows) maxPathLength = 255; else maxPathLength = 65535;
}
public static String infoString() {
String s = "System=";
if (systemOS == System.Unknown) s += "unknown";
else if (systemOS == System.MacOSC) s += "Mac OS Classic";
else if (systemOS == System.MacOSX) s += "Mac OS X";
else if (systemOS == System.Unix) s += "Unix/Linux";
else if (systemOS == System.Windows) s += "Windows";
else s += "unknown";
if (isMacArchitecture) s += ", Mac System Architecture";
if (isUnixFS) s += ", has Unix-like File System";
if (canExecUnix) s += ", can execute Unix-Shell Commands";
return s;
}
/** generates a 2-character string containing information about the OS-type*/
public static String infoKey() {
String s = "";
if (systemOS == System.Unknown) s += "o";
else if (systemOS == System.MacOSC) s += "c";
else if (systemOS == System.MacOSX) s += "x";
else if (systemOS == System.Unix) s += "u";
else if (systemOS == System.Windows) s += "w";
else s += "o";
if (isMacArchitecture) s += "m";
if (isUnixFS) s += "f";
if (canExecUnix) s += "e";
return s;
}
/**
* use a hack to get the current process PID
* @return the PID of the current java process or -1 if the PID cannot be obtained
*/
public static int getPID() {
@SuppressWarnings("deprecation")
public static void deployScript(final File scriptFile, final String theScript) throws IOException {
FileUtils.copy(UTF8.getBytes(theScript), scriptFile);
if(!isWindows){ // set executable
try {
Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);
throw new IOException(e.getMessage());
}
}
}
/**
* use a hack to get the current process PID
* @return the PID of the current java process or -1 if the PID cannot be obtained
*/
public static int getPID() {
final String pids = ManagementFactory.getRuntimeMXBean().getName();
final int p = pids.indexOf('@');
return p >= 0 ? NumberTools.parseIntDecSubstring(pids, 0, p) : -1;
}
public static void execAsynchronous(final File scriptFile) throws IOException {
// runs a script as separate thread
String starterFileExtension = null;
String script = null;
if(isWindows){
starterFileExtension = ".starter.bat";
// use /K to debug, /C for release
script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";
} else { // unix/linux
starterFileExtension = ".starter.sh";
script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;
}
final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);
deployScript(starterFile, script);
try {
Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
throw new IOException(e.getMessage());
}
FileUtils.deletedelete(starterFile);
}
}
@SuppressWarnings("deprecation")
public static void execAsynchronous(final File scriptFile) throws IOException {
// runs a script as separate thread
String starterFileExtension = null;
String script = null;
if(isWindows){
starterFileExtension = ".starter.bat";
// use /K to debug, /C for release
script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";
} else { // unix/linux
starterFileExtension = ".starter.sh";
script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;
}
final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);
deployScript(starterFile, script);
try {
Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (final InterruptedException e) {
throw new IOException(e.getMessage());
}
FileUtils.deletedelete(starterFile);
}
@SuppressWarnings("deprecation")
public static List<String> execSynchronous(final String command) throws IOException {
// runs a unix/linux command and returns output as Vector of Strings
// this method blocks until the command is executed
final Process p = Runtime.getRuntime().exec(command);
return readStreams(p);
}
@SuppressWarnings("deprecation")
public static List<String> execSynchronous(final String[] command) throws IOException {
// runs a unix/linux command and returns output as Vector of Strings
// this method blocks until the command is executed
@ -184,40 +188,40 @@ public final class OS {
*/
public static List<String> readStreams(final Process p) throws IOException {
String line;
final List<String> output = new ArrayList<>();
if(p == null) {
return output;
}
try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());
final BufferedReader in = new BufferedReader(streamReader);) {
while ((line = in.readLine()) != null) {
output.add(line);
}
}
try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());
final BufferedReader in = new BufferedReader(streamReader);) {
while ((line = in.readLine()) != null) {
output.add(line);
}
}
final List<String> output = new ArrayList<>();
if(p == null) {
return output;
}
try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());
final BufferedReader in = new BufferedReader(streamReader);) {
while ((line = in.readLine()) != null) {
output.add(line);
}
}
try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());
final BufferedReader in = new BufferedReader(streamReader);) {
while ((line = in.readLine()) != null) {
output.add(line);
}
}
return output;
}
public static void main(final String[] args) {
public static void main(final String[] args) {
try {
List<String> v = execSynchronous("/usr/local/bin/wkhtmltoimage");
for (String r: v) java.lang.System.out.println(r);
} catch (IOException e) {
final List<String> v = execSynchronous("/usr/local/bin/wkhtmltoimage");
for (final String r: v) java.lang.System.out.println(r);
} catch (final IOException e) {
}
/*
if (args[0].equals("-m")) {
java.lang.System.out.println("Maximum possible memory: " + Integer.toString(getWin32MaxHeap()) + "m");
}
*/
}
*/
}
}

230
source/net/yacy/server/http/HTTPDProxyHandler.java
Unescape View File

@ -37,7 +37,7 @@
support gzip-ed encoding. We also do not support unrealistic
'expires' values that would force a cache to be flushed immediately
pragma non-cache attributes are supported
*/
*/
package net.yacy.server.http;
@ -67,6 +67,7 @@ import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.Logger;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@ -89,6 +90,7 @@ import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
@SuppressWarnings("deprecation")
public final class HTTPDProxyHandler {
@ -108,7 +110,7 @@ public final class HTTPDProxyHandler {
// creating a logger
private static final ConcurrentLog log = new ConcurrentLog("PROXY");
/**
/**
* Do logging configuration for special proxy access log file
*/
static {
@ -117,77 +119,77 @@ public final class HTTPDProxyHandler {
sb = Switchboard.getSwitchboard();
if (sb != null) {
// set timeout
timeout = sb.getConfigInt("proxy.clientTimeout", 60000);
// set timeout
timeout = sb.getConfigInt("proxy.clientTimeout", 60000);
// do logger initialization
try {
log.info("Configuring proxy access logging ...");
// getting the logging manager
final LogManager manager = LogManager.getLogManager();
final String className = HTTPDProxyHandler.class.getName();
// determining if proxy access logging is enabled
final String enabled = manager.getProperty(className + ".logging.enabled");
if ("true".equalsIgnoreCase(enabled)) {
// reading out some needed configuration properties
int limit = 1024*1024, count = 20;
String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");
if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";
// make pattern absolute
if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();
final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");
if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}
final String countStr = manager.getProperty(className + ".logging.FileHandler.count");
if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}
// creating the proxy access logger
final Logger proxyLogger = Logger.getLogger("PROXY.access");
proxyLogger.setUseParentHandlers(false);
proxyLogger.setLevel(Level.FINEST);
final FileHandler txtLog = new FileHandler(pattern, limit, count, true);
txtLog.setFormatter(new ProxyLogFormatter());
txtLog.setLevel(Level.FINEST);
proxyLogger.addHandler(txtLog);
log.info("Proxy access logging configuration done." +
"\n\tFilename: " + pattern +
"\n\tLimit: " + limitStr +
"\n\tCount: " + countStr);
} else {
log.info("Proxy access logging is deactivated.");
// do logger initialization
try {
log.info("Configuring proxy access logging ...");
// getting the logging manager
final LogManager manager = LogManager.getLogManager();
final String className = HTTPDProxyHandler.class.getName();
// determining if proxy access logging is enabled
final String enabled = manager.getProperty(className + ".logging.enabled");
if ("true".equalsIgnoreCase(enabled)) {
// reading out some needed configuration properties
int limit = 1024*1024, count = 20;
String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");
if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";
// make pattern absolute
if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();
final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");
if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}
final String countStr = manager.getProperty(className + ".logging.FileHandler.count");
if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}
// creating the proxy access logger
final Logger proxyLogger = Logger.getLogger("PROXY.access");
proxyLogger.setUseParentHandlers(false);
proxyLogger.setLevel(Level.FINEST);
final FileHandler txtLog = new FileHandler(pattern, limit, count, true);
txtLog.setFormatter(new ProxyLogFormatter());
txtLog.setLevel(Level.FINEST);
proxyLogger.addHandler(txtLog);
log.info("Proxy access logging configuration done." +
"\n\tFilename: " + pattern +
"\n\tLimit: " + limitStr +
"\n\tCount: " + countStr);
} else {
log.info("Proxy access logging is deactivated.");
}
} catch (final Exception e) {
log.severe("Unable to configure proxy access logging.",e);
}
} catch (final Exception e) {
log.severe("Unable to configure proxy access logging.",e);
}
// load the yellow-list
final String f = sb.getConfig("proxyYellowList", null);
if (f != null) {
yellowList = FileUtils.loadList(new File(f));
log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
} else {
yellowList = null;
}
// load the yellow-list
final String f = sb.getConfig("proxyYellowList", null);
if (f != null) {
yellowList = FileUtils.loadList(new File(f));
log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");
} else {
yellowList = null;
}
final String redirectorPath = sb.getConfig("externalRedirector", "");
if (redirectorPath.length() > 0 && !redirectorEnabled) {
try {
redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);
redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));
redirectorEnabled=true;
} catch (final IOException e) {
System.out.println("redirector not Found");
final String redirectorPath = sb.getConfig("externalRedirector", "");
if (redirectorPath.length() > 0 && !redirectorEnabled) {
try {
redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);
redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());
redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));
redirectorEnabled=true;
} catch (final IOException e) {
System.out.println("redirector not Found");
}
}
}
} else {
yellowList = null;
yellowList = null;
}
}
@ -342,25 +344,25 @@ public final class HTTPDProxyHandler {
// case 1 and case 3
if (cachedResponseHeader == null) {
if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web");
fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);
} else {
final Request request = new Request(
null,
final Request request = new Request(
null,
url,
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"",
cachedResponseHeader.lastModified(),
sb.crawler.defaultProxyProfile.handle(),
0,
sb.crawler.defaultProxyProfile.timezoneOffset());
"",
cachedResponseHeader.lastModified(),
sb.crawler.defaultProxyProfile.handle(),
0,
sb.crawler.defaultProxyProfile.timezoneOffset());
final Response response = new Response(
request,
request,
requestHeader,
cachedResponseHeader,
sb.crawler.defaultProxyProfile,
true,
null
);
);
final byte[] cacheContent = Cache.getContent(url.hash());
if (cacheContent != null && response.isFreshForProxy()) {
if (log.isFinest()) log.finest(reqID + " fulfill request from cache");
@ -433,9 +435,9 @@ public final class HTTPDProxyHandler {
// remove yacy-subdomain-path, when accessing /env
if ((yAddress != null)
&& (remotePath.startsWith("/env"))
&& (yAddress.indexOf('/') != -1)
) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
&& (remotePath.startsWith("/env"))
&& (yAddress.indexOf('/') != -1)
) yAddress = yAddress.substring(0, yAddress.indexOf('/'));
modifyProxyHeaders(requestHeader, httpVer);
@ -448,17 +450,17 @@ public final class HTTPDProxyHandler {
try (final HTTPClient client = new HTTPClient(agent, timeout)) {
client.setHeader(requestHeader.entrySet());
client.setRedirecting(false);
client.GET(getUrl, false);
client.GET(getUrl, false);
if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());
int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
// determine if it's an internal error of the httpc
if (responseHeader.isEmpty()) {
throw new Exception(client.getHttpResponse().getStatusLine().toString());
throw new Exception(client.getHttpResponse().getStatusLine().toString());
}
ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);
final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);
// the cache does either not exist or is (supposed to be) stale
long sizeBeforeDelete = -1;
@ -475,19 +477,19 @@ public final class HTTPDProxyHandler {
// reserver cache entry
final Request request = new Request(
null,
null,
url,
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"",
responseHeader.lastModified(),
sb.crawler.defaultProxyProfile.handle(),
0,
sb.crawler.defaultProxyProfile.timezoneOffset());
"",
responseHeader.lastModified(),
sb.crawler.defaultProxyProfile.handle(),
0,
sb.crawler.defaultProxyProfile.timezoneOffset());
// handle incoming cookies
handleIncomingCookies(responseHeader, host, ip);
// prepareResponseHeader(responseHeader, res.getHttpVer());
// prepareResponseHeader(responseHeader, res.getHttpVer());
prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());
// sending the respond header back to the client
@ -506,7 +508,7 @@ public final class HTTPDProxyHandler {
if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {
OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
final OutputStream outStream = chunkedOut != null ? chunkedOut : respond;
final Response response = new Response(
request,
requestHeader,
@ -514,7 +516,7 @@ public final class HTTPDProxyHandler {
sb.crawler.defaultProxyProfile,
true,
null
);
);
final String storeError = response.shallStoreCacheForProxy();
final boolean storeHTCache = response.profile().storeHTCache();
final String supportError = TextParser.supports(response.url(), response.getMimeType());
@ -530,14 +532,14 @@ public final class HTTPDProxyHandler {
* c) the content should be indexed
*/
((storeHTCache) || (supportError != null))
) {
) {
// we don't write actually into a file, only to RAM, and schedule writing the file.
// int l = res.getResponseHeader().size();
final int l = responseHeader.size();
// int l = res.getResponseHeader().size();
final int l = responseHeader.size();
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);
final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});
// FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
// FileUtils.copy(res.getDataAsStream(), toClientAndMemory);
client.writeTo(toClientAndMemory);
// cached bytes
byte[] cacheArray;
@ -627,7 +629,7 @@ public final class HTTPDProxyHandler {
final ResponseHeader cachedResponseHeader,
final byte[] cacheEntry,
OutputStream respond
) throws IOException {
) throws IOException {
final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);
final String clienthttpVer; // the http version of the client connection
@ -721,12 +723,12 @@ public final class HTTPDProxyHandler {
requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader));
}
// only gzip-encoding is supported, remove other encodings (e. g. deflate)
// only gzip-encoding is supported, remove other encodings (e. g. deflate)
if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) {
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
} else {
} else {
requestHeader.put(HeaderFramework.ACCEPT_ENCODING, "");
}
}
addXForwardedForHeader(conProp, requestHeader);
}
@ -782,7 +784,7 @@ public final class HTTPDProxyHandler {
if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {
forceConnectionClose(conProp);
} else {
chunkedOut = new ChunkedOutputStream(respond);
chunkedOut = new ChunkedOutputStream(respond);
}
responseHeader.remove(HeaderFramework.CONTENT_LENGTH);
}
@ -933,12 +935,12 @@ public final class HTTPDProxyHandler {
errorMessage = "IP address of the destination host could not be determined";
}
} else if ((exceptionMsg != null) &&
(
(exceptionMsg.indexOf("socket write error",0)>=0) ||
(exceptionMsg.indexOf("Read timed out",0) >= 0) ||
(exceptionMsg.indexOf("Broken pipe",0) >= 0) ||
(exceptionMsg.indexOf("server has closed connection",0) >= 0)
)) {
(
(exceptionMsg.indexOf("socket write error",0)>=0) ||
(exceptionMsg.indexOf("Read timed out",0) >= 0) ||
(exceptionMsg.indexOf("Broken pipe",0) >= 0) ||
(exceptionMsg.indexOf("server has closed connection",0) >= 0)
)) {
errorMessage = exceptionMsg;
ConcurrentLog.logException(e);
} else {
@ -983,7 +985,7 @@ public final class HTTPDProxyHandler {
final serverObjects detailedErrorMsgMap = new serverObjects();
// generic toplevel domains
final HashSet<String> topLevelDomains = new HashSet<String>(Arrays.asList(new String[]{
final HashSet<String> topLevelDomains = new HashSet<>(Arrays.asList(new String[]{
"aero", // Fluggesellschaften/Luftfahrt
"arpa", // Einrichtung des ARPANet
"biz", // Business
@ -1012,8 +1014,8 @@ public final class HTTPDProxyHandler {
}));
// getting some connection properties
DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
int orgHostPort = orgurl.getPort();
final DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
final int orgHostPort = orgurl.getPort();
String orgHostName = orgurl.getHost();
if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase(Locale.ROOT);
@ -1025,7 +1027,7 @@ public final class HTTPDProxyHandler {
detailedErrorMsgMap.put("hostName", orgHostName);
// guessing hostnames
final HashSet<String> testHostNames = new HashSet<String>();
final HashSet<String> testHostNames = new HashSet<>();
String testHostName = null;
if (!orgHostName.startsWith("www.")) {
testHostName = "www." + orgHostName;
@ -1042,7 +1044,7 @@ public final class HTTPDProxyHandler {
if (addr != null) testHostNames.add(testHostName);
}
int pos = orgHostName.lastIndexOf('.');
final int pos = orgHostName.lastIndexOf('.');
if (pos != -1) {
final Iterator<String> iter = topLevelDomains.iterator();
while (iter.hasNext()) {
@ -1142,7 +1144,7 @@ public final class HTTPDProxyHandler {
logMessage.append(' ');
// Method
HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
final HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);
final String requestMethod = origrequest.getMethod();
logMessage.append(requestMethod);
logMessage.append(' ');

75
source/net/yacy/visualization/DemoApplet.java
Unescape View File

@ -1,75 +0,0 @@
package net.yacy.visualization;
import java.applet.Applet;
import java.awt.Dimension;
import java.awt.Graphics;
@SuppressWarnings("removal")
public class DemoApplet extends Applet implements Runnable {
// can be run in eclipse with
// Run -> Run As -> Java Applet
// see http://www.javaworld.com/javaworld/jw-03-1996/jw-03-animation.html?page=3
private static final long serialVersionUID = -8230253094143014406L;
private int delay;
private Thread animator;
private RasterPlotter offGraphics;
@Override
public void init() {
final String str = getParameter("fps");
final int fps = (str != null) ? Integer.parseInt(str) : 10;
delay = (fps > 0) ? (1000 / fps) : 100;
}
@Override
public void start() {
animator = new Thread(this);
animator.start();
}
@Override
public void run() {
while (Thread.currentThread() == animator) {
final long time = System.currentTimeMillis();
repaint();
try {
Thread.sleep(delay - System.currentTimeMillis() + time);
} catch (final InterruptedException e) {
break;
}
}
}
@Override
public void stop() {
animator = null;
}
@Override
public void update(final Graphics g) {
final Dimension d = getSize();
offGraphics = new RasterPlotter(d.width, d.height, RasterPlotter.DrawMode.MODE_REPLACE, "FFFFFF");
paintFrame(offGraphics);
g.drawImage(offGraphics.getImage(), 0, 0, null);
}
@Override
public void paint(final Graphics g) {
if (offGraphics != null) {
g.drawImage(offGraphics.getImage(), 0, 0, null);
}
}
public void paintFrame(final RasterPlotter m) {
RasterPlotter.demoPaint(m);
final int y = (int) (System.currentTimeMillis() / 10 % 300);
m.setColor(RasterPlotter.GREY);
PrintTool.print(m, 0, y, 0, "Hello World", -1, 100);
}
}
Write Preview
Loading…
Cancel
Save