reduce teh effect of 'Bildersuche findet generierte HTML-Seiten als Bilder'

see http://bugs.yacy.net/view.php?id=9

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7705 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 0621a15f89
commit 9248a4eef4

@ -184,4 +184,149 @@ public class ViewImage {
return scaled;
}
/*
public static BufferedImage respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard)env;
// the url to the image can be either submitted with an url in clear text, or using a license key
// if the url is given as clear text, the user must be authorized as admin
// the license can be used also from non-authorized users
String urlString = post.get("url", "");
final String urlLicense = post.get("code", "");
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true); // handle access rights
DigestURI url = null;
if ((urlString.length() > 0) && (auth)) try {
url = new DigestURI(urlString);
} catch (final MalformedURLException e1) {
url = null;
}
if ((url == null) && (urlLicense.length() > 0)) {
url = sb.licensedURLs.releaseLicense(urlLicense);
urlString = (url == null) ? null : url.toNormalform(true, true);
}
if (urlString == null) return null;
int width = post.getInt("width", 0);
int height = post.getInt("height", 0);
int maxwidth = post.getInt("maxwidth", 0);
int maxheight = post.getInt("maxheight", 0);
// get the image as stream
if (MemoryControl.shortStatus()) iconcache.clear();
BufferedImage image = iconcache.get(urlString);
if (image == null) {
byte[] resourceb = null;
if (url != null) try {
resourceb = sb.loader.loadContent(sb.loader.request(url, false, true), CrawlProfile.CacheStrategy.IFEXIST);
} catch (IOException e) {
Log.logFine("ViewImage", "cannot load: " + e.getMessage());
}
byte[] imgb = null;
if (resourceb == null) {
if (urlString.endsWith(".ico")) {
// load default favicon dfltfvcn.ico
if (defaulticonb == null) try {
imgb = FileUtils.read(new File(sb.getAppPath(), defaulticon));
} catch (final IOException e) {
return null;
} else {
imgb = defaulticonb;
}
} else {
return null;
}
} else {
final InputStream imgStream = new ByteArrayInputStream(resourceb);
// read image data
try {
imgb = FileUtils.read(imgStream);
} catch (final IOException e) {
return null;
} finally {
try {
imgStream.close();
} catch (final Exception e) {}
}
}
// read image
image = ImageParser.parse(urlString, imgb);
if (image == null || (auth && (width == 0 || height == 0) && maxwidth == 0 && maxheight == 0)) return image;
// find original size
final int h = image.getHeight();
final int w = image.getWidth();
// in case of not-authorized access shrink the image to prevent
// copyright problems, so that images are not larger than thumbnails
if (auth) {
maxwidth = (maxwidth == 0) ? w : maxwidth;
maxheight = (maxheight == 0) ? h : maxheight;
} else if ((w > 16) || (h > 16)) {
maxwidth = Math.min(96, w);
maxheight = Math.min(96, h);
} else {
maxwidth = 16;
maxheight = 16;
}
// calculate width & height from maxwidth & maxheight
if (maxwidth < w || maxheight < h) {
// scale image
final double hs = (w <= maxwidth) ? 1.0 : ((double) maxwidth) / ((double) w);
final double vs = (h <= maxheight) ? 1.0 : ((double) maxheight) / ((double) h);
double scale = Math.min(hs, vs);
//if (!auth) scale = Math.min(scale, 0.6); // this is for copyright purpose
if (scale < 1.0) {
width = Math.max(1, (int) (w * scale));
height = Math.max(1, (int) (h * scale));
} else {
width = Math.max(1, w);
height = Math.max(1, h);
}
if (w != width && h != height) {
// compute scaled image
Image scaled = image.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
final MediaTracker mediaTracker = new MediaTracker(new Container());
mediaTracker.addImage(scaled, 0);
try {mediaTracker.waitForID(0);} catch (final InterruptedException e) {}
// make a BufferedImage out of that
BufferedImage i = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
try {
i.createGraphics().drawImage(scaled, 0, 0, width, height, null);
image = i;
// check outcome
//Raster raster = i.getData();
//int[] pixel = new int[3];
//pixel = raster.getPixel(0, 0, pixel);
//if (pixel[0] != 0 || pixel[1] != 0 || pixel[2] != 0) image = i;
} catch (Exception e) {
//java.lang.ClassCastException: [I cannot be cast to [B
}
}
} else {
// do not scale
width = w;
height = h;
}
if ((height == 16) && (width == 16) && (resourceb != null)) {
// this might be a favicon, store image to cache for faster re-load later on
iconcache.put(urlString, image);
}
}
return image;
}
*/
}

@ -606,6 +606,26 @@ public final class HTTPDFileHandler {
result.writeTo(out);
}
}
/*
if (img instanceof BufferedImage) {
final BufferedImage i = (BufferedImage) img;
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
// generate an byte array from the generated image
int width = i.getWidth(); if (width < 0) width = 96; // bad hack
int height = i.getHeight(); if (height < 0) height = 96; // bad hack
final ByteBuffer result = RasterPlotter.exportImage(i, targetExt);
// write the array to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
result.writeTo(out);
}
}
*/
if (img instanceof Image) {
final Image i = (Image) img;
// send an image to client

@ -348,19 +348,21 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
public Boolean EMPTY(final String arg, final Session session) throws IOException {
//System.out.println("EMPTY " + arg);
return (++this.emptyRequestCount > 10) ? serverCore.TERMINATE_CONNECTION : serverCore.RESUME_CONNECTION;
}
public Boolean UNKNOWN(final String requestLine, final Session session) throws IOException {
public Boolean UNKNOWN(final String arg, final Session session) throws IOException {
//System.out.println("UNKNOWN " + arg);
Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, requestLine, session);
Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session);
int pos;
String unknownCommand = null, args = null;
if ((pos = requestLine.indexOf(' ')) > 0) {
unknownCommand = requestLine.substring(0,pos);
args = requestLine.substring(pos+1);
if ((pos = arg.indexOf(' ')) > 0) {
unknownCommand = arg.substring(0,pos);
args = arg.substring(pos+1);
} else {
unknownCommand = requestLine;
unknownCommand = arg;
args = "";
}
@ -371,6 +373,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
public Boolean GET(final String arg, final Session session) {
//System.out.println("GET " + arg);
try {
// parsing the http request line
final Properties prop = parseRequestLine(HeaderFramework.METHOD_GET, arg, session);
@ -438,6 +441,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
public Boolean HEAD(final String arg, final Session session) {
//System.out.println("HEAD " + arg);
try {
final Properties prop = parseRequestLine(HeaderFramework.METHOD_HEAD, arg, session);
@ -482,6 +486,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
public Boolean POST(final String arg, final Session session) {
//System.out.println("POST " + arg);
try {
final Properties prop = parseRequestLine(HeaderFramework.METHOD_POST, arg, session);
@ -564,6 +569,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
public Boolean CONNECT(String arg, final Session session) throws IOException {
//System.out.println("CONNECT " + arg);
// establish a ssh-tunneled http connection
// this is to support https

@ -190,8 +190,8 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
url = new DigestURI(ientry.url());
String u = url.toString();
if (u.indexOf(".ico") >= 0 || u.indexOf("favicon") >= 0) continue;
if (ientry.height() > 0 && ientry.height() < 64) continue;
if (ientry.width() > 0 && ientry.width() < 64) continue;
if (ientry.height() > 0 && ientry.height() < 32) continue;
if (ientry.width() > 0 && ientry.width() < 32) continue;
desc = ientry.alt();
int appcount = queryhashes.size() * 2 -
removeAppearanceHashes(url.toNormalform(false, false), queryhashes).size() -

@ -33,6 +33,7 @@ import java.util.regex.Pattern;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.storage.ScoreMap;
import net.yacy.cora.storage.WeakPriorityBlockingQueue;
import net.yacy.cora.storage.WeakPriorityBlockingQueue.ReverseElement;
@ -47,6 +48,7 @@ import net.yacy.repository.LoaderDispatcher;
import de.anomic.crawler.CrawlProfile;
import de.anomic.data.WorkTables;
import de.anomic.http.client.Cache;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.graphics.ProfilingGraph;
@ -184,7 +186,13 @@ public class ResultFetcher {
// iterate over all images in the result
final List<MediaSnippet> imagemedia = result.mediaSnippets();
if (imagemedia != null) {
for (final MediaSnippet ms: imagemedia) {
feedloop: for (final MediaSnippet ms: imagemedia) {
// check cache to see if the mime type of the image url is correct
ResponseHeader header = Cache.getResponseHeader(ms.href.hash());
if (header != null) {
// this does not work for all urls since some of them may not be in the cache
if (header.mime().startsWith("text") || header.mime().startsWith("application")) continue feedloop;
}
images.put(new ReverseElement<MediaSnippet>(ms, ms.ranking)); // remove smallest in case of overflow
c++;
//System.out.println("*** image " + UTF8.String(ms.href.hash()) + " images.size = " + images.size() + "/" + images.size());

@ -1522,6 +1522,7 @@ public final class Switchboard extends serverSwitch {
if (!MemoryControl.request(8000000L, false)) {
for (final Segment indexSegment: this.indexSegments) indexSegment.urlMetadata().clearCache();
SearchEventCache.cleanupEvents(true);
this.trail.clear();
}
// set a random password if no password is configured

@ -29,7 +29,7 @@ import net.yacy.document.parser.images.bmpParser;
import net.yacy.document.parser.images.icoParser;
public class ImageParser {
public static final Image parse(final String filename, final byte[] source) {
final MediaTracker mediaTracker = new MediaTracker(new Container());
Image image;
@ -46,6 +46,21 @@ public class ImageParser {
} else {
// awt can handle jpg, png and gif formats, try it
image = Toolkit.getDefaultToolkit().createImage(source);
/*
try {
ImageIO.setUseCache(false); // do not write a cache to disc; keep in RAM
image = ImageIO.read(new ByteArrayInputStream(source));
} catch (IOException e) {
Image i = Toolkit.getDefaultToolkit().createImage(source);
mediaTracker.addImage(i, 0);
try {mediaTracker.waitForID(0);} catch (final InterruptedException ee) {}
int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
int height = i.getHeight(null); if (height < 0) height = 96; // bad hack
image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
image.createGraphics().drawImage(i, 0, 0, width, height, null);
}
*/
}
final int handle = image.hashCode();
@ -54,5 +69,5 @@ public class ImageParser {
return image;
}
}

@ -68,6 +68,7 @@ public class htmlParser extends AbstractParser implements Parser {
SUPPORTED_EXTENSIONS.add("aspx");
SUPPORTED_EXTENSIONS.add("tex");
SUPPORTED_EXTENSIONS.add("txt");
//SUPPORTED_EXTENSIONS.add("js");
SUPPORTED_EXTENSIONS.add("jsp");
SUPPORTED_EXTENSIONS.add("mf");
SUPPORTED_EXTENSIONS.add("pl");

Loading…
Cancel
Save