From 89e1848db650be6de3498b79b04c08c0f4fc9a4c Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 15 Jul 2007 01:34:01 +0000 Subject: [PATCH] fixed problem with favicons: target servers had been able to see search words from the referrer of the favicon fetch. This has been removed by using the getImage - servlet for favicon fetch. Since java does not support loading of bmp and ico-Images, such parsers had been added. The image parser had been coded from their original microsoft documentation. This influences also the image-search functionality: there can now be a preview of found bmp-images. Another benefit: favicons for search results are now cached with the HTCACHE. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3965 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ViewImage.java | 55 ++-- htroot/js/yacysearch.js | 2 +- htroot/yacysearch.html | 2 +- source/de/anomic/kelondro/kelondroStack.java | 12 +- .../de/anomic/plasma/plasmaSnippetCache.java | 1 - source/de/anomic/ymage/ymageBMPParser.java | 265 ++++++++++++++++++ source/de/anomic/ymage/ymageICOParser.java | 122 ++++++++ source/de/anomic/ymage/ymageImageParser.java | 38 +++ 8 files changed, 461 insertions(+), 36 deletions(-) create mode 100644 source/de/anomic/ymage/ymageBMPParser.java create mode 100644 source/de/anomic/ymage/ymageICOParser.java create mode 100644 source/de/anomic/ymage/ymageImageParser.java diff --git a/htroot/ViewImage.java b/htroot/ViewImage.java index 7e6d5b1d4..7b8478632 100644 --- a/htroot/ViewImage.java +++ b/htroot/ViewImage.java @@ -42,7 +42,7 @@ import java.awt.Container; import java.awt.Image; import java.awt.MediaTracker; -import java.awt.Toolkit; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; @@ -53,6 +53,7 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverFileUtils; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.ymage.ymageImageParser; public class ViewImage { @@ -75,32 +76,41 @@ public class ViewImage { // getting the image as stream Object[] resource = sb.snippetCache.getResource(url, true, timeout, false); - if (resource == null) return null; - InputStream imgStream = (InputStream) resource[0]; - if (imgStream == null) return null; - - // read image data byte[] imgb = null; - try { - imgb = serverFileUtils.read(imgStream); - } catch (IOException e) { - return null; - } finally { - try { imgStream.close(); } catch (Exception e) {/* ignore this */} + if (resource == null) { + if (urls.endsWith(".ico")) { + // load default favicon dfltfvcn.ico + try { + imgb = serverFileUtils.read(new File(sb.getRootPath(), "htroot/env/grafics/dfltfvcn.ico")); + } catch (IOException e) { + return null; + } + } else { + return null; + } + } else { + InputStream imgStream = (InputStream) resource[0]; + if (imgStream == null) return null; + + // read image data + try { + imgb = serverFileUtils.read(imgStream); + } catch (IOException e) { + return null; + } finally { + try { imgStream.close(); } catch (Exception e) {/* ignore this */} + } } - // create image - MediaTracker mediaTracker = new MediaTracker(new Container()); - Image original = Toolkit.getDefaultToolkit().createImage(imgb); - int handle = original.hashCode(); - mediaTracker.addImage(original, handle); - try {mediaTracker.waitForID(handle);} catch (InterruptedException e) {} + // read image + Image image = ymageImageParser.parse(urls.toString(), imgb); + boolean auth = ((String) header.get("CLIENTIP", "")).equals("localhost") || sb.verifyAuthentication(header, false); // handle access rights - if ((auth) && ((width == 0) || (height == 0)) && (maxwidth == 0) && (maxheight == 0)) return original; + if ((auth) && ((width == 0) || (height == 0)) && (maxwidth == 0) && (maxheight == 0)) return image; // find original size - int h = original.getHeight(null); - int w = original.getWidth(null); + int h = image.getHeight(null); + int w = image.getWidth(null); //System.out.println("DEBUG: get access to image " + url.toNormalform() + " is " + ((auth) ? "authorized" : "NOT authorized")); @@ -134,7 +144,8 @@ public class ViewImage { height = Math.max(height, 1); // scale image - Image scaled = original.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING); + Image scaled = image.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING); + MediaTracker mediaTracker = new MediaTracker(new Container()); mediaTracker.addImage(scaled, 0); try {mediaTracker.waitForID(0);} catch (InterruptedException e) {} diff --git a/htroot/js/yacysearch.js b/htroot/js/yacysearch.js index ccf1fe3fa..ad400d8a8 100644 --- a/htroot/js/yacysearch.js +++ b/htroot/js/yacysearch.js @@ -134,7 +134,7 @@ function handleTextState(req) { // set URL to favicon (if a link-tag was found in the document) if (response.getElementsByTagName("favicon")[0].firstChild != null) { var img = document.getElementById("f" + urlHash); - img.src = response.getElementsByTagName("favicon")[0].firstChild.data; + img.src = "ViewImage.png?width=16&height=16&url=" + response.getElementsByTagName("favicon")[0].firstChild.data; } // replace "" text by node diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html index bf4aad42a..1c73b59f0 100644 --- a/htroot/yacysearch.html +++ b/htroot/yacysearch.html @@ -150,7 +150,7 @@ document.getElementById("Enter").value = "search again - catch up more links"; #(/recommend)# #(/authorized)# -

#[description]#

+

#[description]#

#(snippet)#loading snippet ...::#[text]##(/snippet)#

#[urlname]#

#[date]# | YBR-#[ybr]# | Info | Pictures

diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index cd2f91595..8cf05ccd5 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -260,17 +260,7 @@ public final class kelondroStack extends kelondroRecords { if (h == null) return null; return getNode(h, true); } - /* - private Node queueNode(int dist, int side, int dir) throws IOException { - // with dist == 0 this is the same function as with - // getNode(getHandle(side), null, 0) - if (dist >= size()) return null; // that would exceed the stack - Handle h = getHandle(side); - if (h == null) return null; - while ((dist-- > 0) && (h != null)) h = getNode(h, false).getOHHandle(dir); // track through elements - if (h == null) return null; else return getNode(h, true); - } -*/ + public int imp(File file, String separator) throws IOException { // imports a value-separated file, returns number of records that have been read RandomAccessFile f = null; diff --git a/source/de/anomic/plasma/plasmaSnippetCache.java b/source/de/anomic/plasma/plasmaSnippetCache.java index 6ecb260d7..d6cbe85f8 100644 --- a/source/de/anomic/plasma/plasmaSnippetCache.java +++ b/source/de/anomic/plasma/plasmaSnippetCache.java @@ -97,7 +97,6 @@ public class plasmaSnippetCache { * */ private HashMap faviconCache; - private plasmaHTCache cacheManager; private plasmaParser parser; private serverLog log; diff --git a/source/de/anomic/ymage/ymageBMPParser.java b/source/de/anomic/ymage/ymageBMPParser.java new file mode 100644 index 000000000..aa30ac5d4 --- /dev/null +++ b/source/de/anomic/ymage/ymageBMPParser.java @@ -0,0 +1,265 @@ +// ymageBMPParser.java +// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 15.07.2007 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.ymage; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import javax.imageio.ImageIO; + +public class ymageBMPParser { + + // this is a implementation of http://de.wikipedia.org/wiki/Windows_Bitmap + + // file offsets + private static int FILEHEADER_offset = 0; + private static int INFOHEADER_offset = 14; + public static int INFOHEADER_size = 40; + + // compression tags + private static int BI_RGB = 0; + //private static int BI_RLE8 = 1; + //private static int BI_RLE4 = 2; + //private static int BI_BITFIELDS = 3; + + private IMAGEMAP imagemap; + //boolean debugmode = false; + + public static final boolean isBMP(byte[] source) { + // check the file magic + return (source[0] == 'B') && (source[1] == 'M'); + } + + public ymageBMPParser(byte[] source) { + + // read info-header + int bfOffBits = DWORD(source, FILEHEADER_offset + 10); + + INFOHEADER infoheader = new INFOHEADER(source, INFOHEADER_offset); + COLORTABLE colortable = new COLORTABLE(source, INFOHEADER_offset + INFOHEADER_size, infoheader); + + // check consistency with bfOffBits + assert bfOffBits == INFOHEADER_offset + 40 + colortable.colorbytes : "bfOffBits = " + bfOffBits + ", colorbytes = " + colortable.colorbytes; + assert infoheader.biSizeImage <= source.length - bfOffBits : "bfOffBits = " + bfOffBits + ", biSizeImage = " + infoheader.biSizeImage + ", source.length = " + source.length; + + imagemap = new IMAGEMAP(source, bfOffBits, infoheader.biWidth, infoheader.biHeight, infoheader.biCompression, infoheader.biBitCount, colortable); + } + + public BufferedImage getImage() { + return imagemap.image; + } + + public static final int DWORD(byte[] b, int offset) { + int ret = ((int) b[offset + 3] & 0xff); + ret = (ret << 8) | ((int) b[offset + 2] & 0xff); + ret = (ret << 8) | ((int) b[offset + 1] & 0xff); + ret = (ret << 8) | ((int) b[offset] & 0xff); + return ret; + } + + public static final int WORD(byte[] b, int offset) { + int ret = (((int) b[offset + 1] & 0xff) << 8) | ((int) b[offset] & 0xff); + return ret; + } + + public static final int BYTE(byte[] b, int offset) { + int ret = ((int) b[offset] & 0xff); + return ret; + } + + + public static class INFOHEADER { + + public int biWidth, biHeight, biBitCount, biCompression, biSizeImage, biClrUsed; + + public INFOHEADER(byte[] s, int offset) { + // read info-header + biWidth = DWORD(s, offset + 4); + biHeight = DWORD(s, offset + 8); + biBitCount = WORD(s, offset + 14); + biCompression = WORD(s, offset + 16); + biSizeImage = DWORD(s, offset + 20); + biClrUsed = DWORD(s, offset + 32); + } + } + + public static class COLORTABLE { + + public int colorbytes; + public int[] colorindex; + + public COLORTABLE(byte[] s, int offset, INFOHEADER infoheader) { + // read colortable + colorbytes = 0; // for consistency check + if (infoheader.biClrUsed == 0) { + if ((infoheader.biBitCount == 1) || (infoheader.biBitCount == 4) || (infoheader.biBitCount == 8)) { + colorindex = new int[1 << infoheader.biBitCount]; + colorbytes = 4 * colorindex.length; + int color; + for (int i = 0; i < colorindex.length; i++) { + // translate BGR into RGB color Scheme + color = 0xffffff & DWORD(s, offset + 4 * i); + colorindex[i] = color; + } + } else { + colorindex = null; + } + } else { + colorindex = new int[infoheader.biClrUsed]; + colorbytes = 4 * colorindex.length; + int color; + for (int i = 0; i < colorindex.length; i++) { + // translate BGR into RGB color Scheme + color = 0xffffff & DWORD(s, offset + 4 * i); + colorindex[i] = color; + //if (debugmode) System.out.println("Color " + i + " = " + Integer.toHexString(colorindex[i])); + } + } + } + } + + public static class IMAGEMAP { + + public BufferedImage image; + + public IMAGEMAP(byte[] s, int offset, int width, int height, int compression, int bitcount, COLORTABLE colortable) { + // parse picture content + if ((width != 0) && (height != 0)) { + image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + + if (compression == BI_RGB) { + if (bitcount == 1) parseBMP1(s, offset, width, height, colortable); + else if (bitcount == 4) parseBMP4(s, offset, width, height, colortable); + else if (bitcount == 8) parseBMP8(s, offset, width, height, colortable); + else if (bitcount == 24) parseBMP24(s, offset, width, height); + else if (bitcount == 32) parseBMP32(s, offset, width, height); + else System.out.println("unsupported BMP format: biCompression = " + compression + ", biBitCount = " + bitcount); + } else { + System.out.println("unsupported BMP format: biCompression = " + compression + ", biBitCount = " + bitcount); + } + } + } + + + private void parseBMP1(byte[] s, int offset, int width, int height, COLORTABLE colortable) { + int n = 0; + int b; + for (int rows = 0; rows < height; rows++) { + for (int columns = 0; columns < width; columns = columns + 2) { + b = ((int) s[offset + n] & 0xff); + n++; + image.setRGB(columns, (height - rows - 1), colortable.colorindex[(b & 0xc0) >> 6]); + image.setRGB(columns + 1, (height - rows - 1), colortable.colorindex[(b & 0x30) >> 4]); + image.setRGB(columns + 2, (height - rows - 1), colortable.colorindex[(b & 0xc) >> 2]); + image.setRGB(columns + 3, (height - rows - 1), colortable.colorindex[b & 0x3]); + } + n += fill4(n); + } + } + + private void parseBMP4(byte[] s, int offset, int width, int height, COLORTABLE colortable) { + int n = 0; + int b; + for (int rows = 0; rows < height; rows++) { + for (int columns = 0; columns < width; columns = columns + 2) { + b = ((int) s[offset + n] & 0xff); + n++; + image.setRGB(columns, (height - rows - 1), colortable.colorindex[(b & 0xf0) >> 4]); + image.setRGB(columns + 1, (height - rows - 1), colortable.colorindex[b & 0xf]); + } + n += fill4(n); + } + } + + private void parseBMP8(byte[] s, int offset, int width, int height, COLORTABLE colortable) { + int n = 0; + for (int rows = 0; rows < height; rows++) { + for (int columns = 0; columns < width; columns++) { + image.setRGB(columns, (height - rows - 1), colortable.colorindex[((int) s[offset + n] & 0xff)]); + n++; + } + n += fill4(n); + } + } + + private void parseBMP24(byte[] s, int offset, int width, int height) { + int n = 0; + for (int rows = 0; rows < height; rows++) { + for (int columns = 0; columns < width; columns++) { + image.setRGB(columns, (height - rows - 1), 0xffffff & DWORD(s, offset + n)); + n += 3; + } + n += fill4(n); + } + } + + private void parseBMP32(byte[] s, int offset, int width, int height) { + int n = 0; + for (int rows = 0; rows < height; rows++) { + for (int columns = 0; columns < width; columns++) { + image.setRGB(columns, (height - rows - 1), 0xffffff & DWORD(s, offset + n)); + n += 4; + } + } + } + + private final int fill4(int x) { + int r = x % 4; + if (r == 0) return 0; else return 4 - r; + } + } + + public static void main(String[] args) { + // read a bmp and write it as png + System.setProperty("java.awt.headless", "true"); + File in = new File(args[0]); + File out = new File(args[1]); + + byte[] file = new byte[(int) in.length()]; + FileInputStream fis = null; + try { + fis = new FileInputStream(in); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + try { + fis.read(file); + } catch (IOException e) { + e.printStackTrace(); + } + + ymageBMPParser parser = new ymageBMPParser(file); + + try { + ImageIO.write(parser.getImage(), "PNG", out); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/source/de/anomic/ymage/ymageICOParser.java b/source/de/anomic/ymage/ymageICOParser.java new file mode 100644 index 000000000..f69137c45 --- /dev/null +++ b/source/de/anomic/ymage/ymageICOParser.java @@ -0,0 +1,122 @@ +// ymageICOParser.java +// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 15.07.2007 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.ymage; + +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import javax.imageio.ImageIO; + +public class ymageICOParser { + + // this is a implementation of http://msdn2.microsoft.com/en-us/library/ms997538(d=printer).aspx + + public static int ICONDIRENTRY_size = 16; + + private int idCount; + ymageBMPParser.INFOHEADER[] infoheaders; + ymageBMPParser.IMAGEMAP[] imagemaps; + + public static final boolean isICO(byte[] source) { + // check the file magic + return (source[0] == 0) && (source[1] == 0) && (source[2] == 1) && (source[3] == 0); + } + + public ymageICOParser(byte[] source) { + // read info-header + idCount = ymageBMPParser.WORD(source, 4); + + // read the icon directory entry and the image entries + ICONDIRENTRY[] icondirentries = new ICONDIRENTRY[idCount]; + infoheaders = new ymageBMPParser.INFOHEADER[idCount]; + ymageBMPParser.COLORTABLE[] colortables = new ymageBMPParser.COLORTABLE[idCount]; + imagemaps = new ymageBMPParser.IMAGEMAP[idCount]; + for (int i = 0; i < idCount; i++) { + icondirentries[i] = new ICONDIRENTRY(source, 6 + i * ICONDIRENTRY_size); + infoheaders[i] = new ymageBMPParser.INFOHEADER(source, icondirentries[i].dwImageOffset); + colortables[i] = new ymageBMPParser.COLORTABLE(source, icondirentries[i].dwImageOffset + ymageBMPParser.INFOHEADER_size, infoheaders[i]); + imagemaps[i] = new ymageBMPParser.IMAGEMAP(source, icondirentries[i].dwImageOffset + ymageBMPParser.INFOHEADER_size + colortables[i].colorbytes, icondirentries[i].bWidth, icondirentries[i].bHeight, infoheaders[i].biCompression, infoheaders[i].biBitCount, colortables[i]); + } + } + + + public class ICONDIRENTRY { + + public int bWidth, bHeight, bColorCount, bReserved, wPlanes, wBitCount, dwBytesInRes, dwImageOffset; + + public ICONDIRENTRY(byte[] s, int offset) { + // read info-header + bWidth = ymageBMPParser.BYTE(s, offset + 0); + bHeight = ymageBMPParser.BYTE(s, offset + 1); + bColorCount = ymageBMPParser.BYTE(s, offset + 2); + bReserved = ymageBMPParser.BYTE(s, offset + 3); + wPlanes = ymageBMPParser.WORD(s, offset + 4); + wBitCount = ymageBMPParser.WORD(s, offset + 6); + dwBytesInRes = ymageBMPParser.DWORD(s, offset + 8); + dwImageOffset = ymageBMPParser.DWORD(s, offset + 12); + } + } + + public int images() { + // return number of images in icon + return idCount; + } + + public BufferedImage getImage(int index) { + return imagemaps[index].image; + } + + public static void main(String[] args) { + // read a ICO and write it as png + System.setProperty("java.awt.headless", "true"); + File in = new File(args[0]); + File out = new File(args[1]); + + byte[] file = new byte[(int) in.length()]; + FileInputStream fis = null; + try { + fis = new FileInputStream(in); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + try { + fis.read(file); + } catch (IOException e) { + e.printStackTrace(); + } + + ymageICOParser parser = new ymageICOParser(file); + + try { + ImageIO.write(parser.getImage(0), "PNG", out); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/source/de/anomic/ymage/ymageImageParser.java b/source/de/anomic/ymage/ymageImageParser.java new file mode 100644 index 000000000..5439cf825 --- /dev/null +++ b/source/de/anomic/ymage/ymageImageParser.java @@ -0,0 +1,38 @@ +package de.anomic.ymage; + +import java.awt.Container; +import java.awt.Image; +import java.awt.MediaTracker; +import java.awt.Toolkit; + + +public class ymageImageParser { + + public static final Image parse(String filename, byte[] source) { + MediaTracker mediaTracker = new MediaTracker(new Container()); + Image image; + if (((filename.endsWith(".ico")) || (filename.endsWith(".bmp"))) && (ymageBMPParser.isBMP(source))) { + // parse image with BMP parser + ymageBMPParser bmpparser; + bmpparser = new ymageBMPParser(source); + image = bmpparser.getImage(); + if (image == null) return null; + } else if ((filename.endsWith(".ico")) && (ymageICOParser.isICO(source))) { + // parse image with ICO parser + ymageICOParser icoparser; + icoparser = new ymageICOParser(source); + image = icoparser.getImage(0); + if (image == null) return null; + } else { + // awt can handle jpg, bmp and gif formats, try it + image = Toolkit.getDefaultToolkit().createImage(source); + } + + int handle = image.hashCode(); + mediaTracker.addImage(image, handle); + try {mediaTracker.waitForID(handle);} catch (InterruptedException e) {} + + return image; + } + +}