Merge pull request #39 from luccioman/master
Favicon retrieval and image preview enhancements. More details on mantis 629 (http://mantis.tokeek.de/view.php?id=629)pull/91/head
luccioman
9 years ago
committed by
GitHub
commit
d16e57b41e
@ -0,0 +1,195 @@
|
||||
|
||||
// ViewFavicon.java
|
||||
// -----------------------
|
||||
// part of YaCy
|
||||
// Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.stream.ImageInputStream;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.document.id.MultiProtocolURL;
|
||||
import net.yacy.cora.protocol.Domains;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.storage.ConcurrentARC;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.kelondro.util.FileUtils;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
import net.yacy.kelondro.workflow.WorkflowProcessor;
|
||||
import net.yacy.peers.graphics.EncodedImage;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
import net.yacy.visualization.ImageViewer;
|
||||
|
||||
/**
|
||||
* Extends ViewImage behavior : add a specific favicon cache and use of a
|
||||
* default image on loading error.
|
||||
*
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class ViewFavicon {
|
||||
|
||||
/** Single instance of ImageViewer */
|
||||
private static final ImageViewer VIEWER = new ImageViewer();
|
||||
|
||||
/** Icons cache encoded as png */
|
||||
private static Map<String, byte[]> pngIconCache = new ConcurrentARC<String, byte[]>(1000,
|
||||
Math.max(10, Math.min(32, WorkflowProcessor.availableCPU * 2)));
|
||||
|
||||
/** Default icon local file */
|
||||
private static final String defaulticon = "htroot/env/grafics/dfltfvcn.ico";
|
||||
|
||||
/**
|
||||
* Default icon encoded as png : we use a bvte array as it is thread-safe
|
||||
* instead of a ByteBuffer in EncodedImage
|
||||
*/
|
||||
private static byte[] defaultPNGEncodedIcon = null;
|
||||
|
||||
/**
|
||||
* Try parsing image from post "url" parameter (authenticated users) or from
|
||||
* "code" parameter (non authenticated users). When image could be parsed,
|
||||
* try encoding to target format specified by header "EXT". When any error
|
||||
* occurs, return default icon.
|
||||
*
|
||||
* @param header
|
||||
* request header
|
||||
* @param post
|
||||
* post parameters
|
||||
* @param env
|
||||
* Switchboard instance
|
||||
* @return an {@link EncodedImage} instance encoded in format specified in
|
||||
* post, or an InputStream pointing to original image data.
|
||||
*/
|
||||
public static Object respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
String ext = header.get("EXT", null);
|
||||
boolean isPNGTarget = "png".equalsIgnoreCase(ext);
|
||||
|
||||
ImageInputStream imageInStream = null;
|
||||
InputStream inStream = null;
|
||||
byte[] resultBytes = null;
|
||||
try {
|
||||
/* Clear icon cache when running out of memory */
|
||||
if (MemoryControl.shortStatus()) {
|
||||
pngIconCache.clear();
|
||||
}
|
||||
|
||||
final boolean auth = Domains.isLocalhost(header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""))
|
||||
|| sb.verifyAuthentication(header); // handle access rights
|
||||
|
||||
DigestURL url = VIEWER.parseURL(post, auth);
|
||||
|
||||
final String normalizedURL = url.toNormalform(false);
|
||||
|
||||
if (isPNGTarget) {
|
||||
resultBytes = pngIconCache.get(normalizedURL);
|
||||
}
|
||||
/* Icon is not already in cache */
|
||||
if (resultBytes == null) {
|
||||
String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());
|
||||
if (ext != null && ext.equalsIgnoreCase(urlExt) && ImageViewer.isBrowserRendered(urlExt)) {
|
||||
return VIEWER.openInputStream(post, sb.loader, auth, url);
|
||||
}
|
||||
/*
|
||||
* When opening a file, the most efficient is to open
|
||||
* ImageInputStream directly on file
|
||||
*/
|
||||
if (url.isFile()) {
|
||||
imageInStream = ImageIO.createImageInputStream(url.getFSFile());
|
||||
} else {
|
||||
inStream = VIEWER.openInputStream(post, sb.loader, auth, url);
|
||||
imageInStream = ImageIO.createImageInputStream(inStream);
|
||||
}
|
||||
// read image
|
||||
EncodedImage encodedIcon = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);
|
||||
|
||||
if (encodedIcon != null && !encodedIcon.getImage().isEmpty()) {
|
||||
resultBytes = encodedIcon.getImage().getBytes();
|
||||
if (isPNGTarget && encodedIcon.getImage().length() <= 10240) {
|
||||
/* Only store in cache icon images below 10KB, png encoded */
|
||||
pngIconCache.put(normalizedURL, resultBytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
ConcurrentLog.fine("ViewFavicon", "Error loading favicon, default one wille be used : " + e);
|
||||
} finally {
|
||||
/*
|
||||
* imageInStream.close() method doesn't close source input stream
|
||||
*/
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
if (resultBytes == null) {
|
||||
/*
|
||||
* I any error occured when loading icon, return default one
|
||||
*/
|
||||
if (ext == null || isPNGTarget) {
|
||||
/* Load default icon only once */
|
||||
if (defaultPNGEncodedIcon == null) {
|
||||
defaultPNGEncodedIcon = loadDefaultIcon(post, sb, ext);
|
||||
}
|
||||
resultBytes = defaultPNGEncodedIcon;
|
||||
} else {
|
||||
resultBytes = loadDefaultIcon(post, sb, ext);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return new ByteArrayInputStream(resultBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load default icon and encode it to ext format
|
||||
*
|
||||
* @param post
|
||||
* post parameters
|
||||
* @param sb
|
||||
* Switchboard instance
|
||||
* @param ext
|
||||
* target image format
|
||||
* @return icon encoded bytes, empty if and exception occured when loading
|
||||
* or rendering
|
||||
*/
|
||||
private static byte[] loadDefaultIcon(final serverObjects post, final Switchboard sb, String ext) {
|
||||
byte[] resultBytes;
|
||||
byte[] defaultBytes = new byte[0];
|
||||
try {
|
||||
defaultBytes = FileUtils.read(new File(sb.getAppPath(), defaulticon));
|
||||
} catch (final IOException initicon) {
|
||||
defaultBytes = new byte[0];
|
||||
} finally {
|
||||
resultBytes = new EncodedImage(defaultBytes, ext, post.getBoolean("isStatic")).getImage().getBytes();
|
||||
}
|
||||
return resultBytes;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
// InvalidURLLicenceException.java
|
||||
// Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate$
|
||||
// $LastChangedRevision$
|
||||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.data;
|
||||
|
||||
/**
|
||||
* Exception indicating a URLLicense is not valid.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class InvalidURLLicenceException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = 388769934848447613L;
|
||||
|
||||
/**
|
||||
* Default constructor : use generic message
|
||||
*/
|
||||
public InvalidURLLicenceException() {
|
||||
super("Url license code is not valid or empty");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message detail message
|
||||
*/
|
||||
public InvalidURLLicenceException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,197 @@
|
||||
/**
|
||||
* IconEntry
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.document.parser.html;
|
||||
|
||||
import java.awt.Dimension;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
|
||||
/**
|
||||
* Represents an icon in a document.
|
||||
*
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class IconEntry {
|
||||
|
||||
/** Patern to parse a HTML link sizes token attribute (ie. "16x16") */
|
||||
public static final Pattern SIZE_PATTERN = Pattern.compile("([1-9][0-9]*)[xX]([1-9][0-9]*)");
|
||||
|
||||
/** Icon URL */
|
||||
private final DigestURL url;
|
||||
/**
|
||||
* Icon links relations (one url may be used as multiple icon relations in
|
||||
* the same document)
|
||||
*/
|
||||
private final Set<String> rel;
|
||||
/** Icon sizes */
|
||||
private final Set<Dimension> sizes;
|
||||
|
||||
/**
|
||||
* Constructs instance from parameters.
|
||||
*
|
||||
* @param url
|
||||
* must not be null.
|
||||
* @param rel
|
||||
* must not be null and contain at least one item.
|
||||
* @param sizes
|
||||
* optional.
|
||||
*/
|
||||
public IconEntry(final DigestURL url, Set<String> rel, Set<Dimension> sizes) {
|
||||
if (url == null) {
|
||||
throw new IllegalArgumentException("url must not be null.");
|
||||
}
|
||||
if (rel == null || rel.isEmpty()) {
|
||||
throw new IllegalArgumentException("rel must be specified");
|
||||
}
|
||||
this.url = url;
|
||||
this.rel = rel;
|
||||
if (sizes != null) {
|
||||
this.sizes = sizes;
|
||||
} else {
|
||||
this.sizes = new HashSet<>();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true when rel property contains a standard IANA registered icon
|
||||
* link relation
|
||||
*/
|
||||
public boolean isStandardIcon() {
|
||||
boolean standard = false;
|
||||
for (String relation : this.rel) {
|
||||
if (IconLinkRelations.isStandardIconRel(relation)) {
|
||||
standard = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return standard;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param size1
|
||||
* @param size2
|
||||
* @return distance between two sizes, or Double.MAX_VALUE when one size is null
|
||||
*/
|
||||
public static double getDistance(Dimension size1, Dimension size2) {
|
||||
double result = Double.MAX_VALUE;
|
||||
if(size1 != null && size2 != null) {
|
||||
result = (Math.abs(size1.width - size2.width) + Math.abs(size1.height - size2.height)) / 2.0;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param preferredSize
|
||||
* @return the size among sizes property which is the closest to
|
||||
* preferredSize, or null when sizes is empty or preferredSize is null.
|
||||
*/
|
||||
public Dimension getClosestSize(Dimension preferredSize) {
|
||||
Dimension closest = null;
|
||||
if (preferredSize != null) {
|
||||
double closestDistance = Double.MAX_VALUE;
|
||||
for (Dimension size : this.sizes) {
|
||||
double currentDistance = getDistance(size, preferredSize);
|
||||
if (closest == null) {
|
||||
closest = size;
|
||||
closestDistance = currentDistance;
|
||||
} else {
|
||||
if (currentDistance < closestDistance) {
|
||||
closest = size;
|
||||
closestDistance = currentDistance;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return closest;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder res = new StringBuilder();
|
||||
res.append("<link");
|
||||
res.append(" href=\"").append(this.url.toNormalform(false)).append("\"");
|
||||
res.append(" rel=\"");
|
||||
res.append(relToString());
|
||||
res.append("\"");
|
||||
if (!this.sizes.isEmpty()) {
|
||||
res.append(" sizes=\"");
|
||||
res.append(sizesToString());
|
||||
res.append("\"");
|
||||
}
|
||||
res.append(">");
|
||||
return res.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return icon URL
|
||||
*/
|
||||
public DigestURL getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return icons link relations
|
||||
*/
|
||||
public Set<String> getRel() {
|
||||
return rel;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return icon eventual sizes
|
||||
*/
|
||||
public Set<Dimension> getSizes() {
|
||||
return sizes;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a string representation of sizes property, in the form of a valid
|
||||
* HTML link tag sizes attribute (e.g. "16x16 64x64")
|
||||
*/
|
||||
public String sizesToString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (Dimension size : this.sizes) {
|
||||
if (builder.length() > 0) {
|
||||
builder.append(" ");
|
||||
}
|
||||
builder.append(size.width).append("x").append(size.height);
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a string representation of rel property, int the form of a valid
|
||||
* HTML link tag rel attribute (e.g. "icon apple-touch-icon")
|
||||
*/
|
||||
public String relToString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (String relation : this.rel) {
|
||||
if (builder.length() > 0) {
|
||||
builder.append(" ");
|
||||
}
|
||||
builder.append(relation);
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
/**
|
||||
* IconLinkRelations
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.document.parser.html;
|
||||
|
||||
/**
|
||||
* Enumeration of HTML link relationships (rel attribute) designating icon.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public enum IconLinkRelations {
|
||||
/** Standard IANA registered icon link relation (see https://www.iana.org/assignments/link-relations/link-relations.xhtml) */
|
||||
ICON("icon", "Standard favicon"),
|
||||
/** Icon for IOS app shortcut */
|
||||
APPLE_TOUCH_ICON("apple-touch-icon", "IOS app shortcut icon"),
|
||||
/** Icon for IOS app shortcut (deprecated but still used by major websites in 2015) */
|
||||
APPLE_TOUCH_ICON_PRECOMPOSED("apple-touch-icon-precomposed", "Deprecated IOS app shortcut icon"),
|
||||
/** icon for Safari pinned tab */
|
||||
MASK_ICON("mask-icon", "Safari browser pinned tab icon"),
|
||||
/** Icon for Fluid web app */
|
||||
FLUID_ICON("fluid-icon", "Fluid app icon");
|
||||
|
||||
/** HTML rel attribute value */
|
||||
private String relValue;
|
||||
|
||||
/** Human readable description */
|
||||
private String description;
|
||||
|
||||
private IconLinkRelations(String relValue, String description) {
|
||||
this.relValue = relValue;
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return HTML rel attribute value
|
||||
*/
|
||||
public String getRelValue() {
|
||||
return relValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Human readable description of icon rel attribute
|
||||
*/
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param relToken HTML rel attribute token
|
||||
* @return true when relToken is an icon relationship (standard or non-standard)
|
||||
*/
|
||||
public static boolean isIconRel(String relToken) {
|
||||
boolean res = false;
|
||||
for(IconLinkRelations iconRel : IconLinkRelations.values()) {
|
||||
if(iconRel.getRelValue().equalsIgnoreCase(relToken)) {
|
||||
res = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param relToken HTML rel attribute token
|
||||
* @return true when relToken is Standard IANA registered icon link relation
|
||||
*/
|
||||
public static boolean isStandardIconRel(String relToken) {
|
||||
return ICON.getRelValue().equalsIgnoreCase(relToken);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
// TemplateMissingParameterException.java
|
||||
// Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.http.servlets;
|
||||
|
||||
/**
|
||||
* Use this to indicates a required parameter is missing for a template. Allows finer grained exception handling.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class TemplateMissingParameterException extends IllegalArgumentException {
|
||||
|
||||
private static final long serialVersionUID = -3443324572847193267L;
|
||||
|
||||
/**
|
||||
* Default constructor : use generic message.
|
||||
*/
|
||||
public TemplateMissingParameterException() {
|
||||
super("Missing required parameters");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message detail message
|
||||
*/
|
||||
public TemplateMissingParameterException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,490 @@
|
||||
/**
|
||||
* ImageViewer
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.visualization;
|
||||
|
||||
import java.awt.Container;
|
||||
import java.awt.Dimension;
|
||||
import java.awt.Graphics2D;
|
||||
import java.awt.Image;
|
||||
import java.awt.MediaTracker;
|
||||
import java.awt.Rectangle;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.awt.image.Raster;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import javax.imageio.ImageReader;
|
||||
import javax.imageio.stream.ImageInputStream;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.document.id.MultiProtocolURL;
|
||||
import net.yacy.cora.federate.yacy.CacheStrategy;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.data.InvalidURLLicenceException;
|
||||
import net.yacy.data.URLLicense;
|
||||
import net.yacy.http.servlets.TemplateMissingParameterException;
|
||||
import net.yacy.peers.graphics.EncodedImage;
|
||||
import net.yacy.repository.Blacklist.BlacklistType;
|
||||
import net.yacy.repository.LoaderDispatcher;
|
||||
import net.yacy.server.serverObjects;
|
||||
|
||||
/**
|
||||
* Provides methods for image or favicon viewing in YaCy servlets.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class ImageViewer {
|
||||
|
||||
/**
|
||||
* Try to get image URL from parameters.
|
||||
* @param post post parameters. Must not be null.
|
||||
* @param auth true when current user is authenticated
|
||||
* @return DigestURL instance
|
||||
* @throws MalformedURLException when url is malformed
|
||||
* @throws TemplateMissingParameterException when urlString or urlLicense is missing (the one needed depends on auth)
|
||||
*/
|
||||
public DigestURL parseURL(final serverObjects post, final boolean auth)
|
||||
throws MalformedURLException {
|
||||
final String urlString = post.get("url", "");
|
||||
final String urlLicense = post.get("code", "");
|
||||
DigestURL url;
|
||||
if(auth) {
|
||||
/* Authenticated user : rely on url parameter*/
|
||||
if (urlString.length() > 0) {
|
||||
url = new DigestURL(urlString);
|
||||
} else {
|
||||
throw new TemplateMissingParameterException("missing required url parameter");
|
||||
}
|
||||
} else {
|
||||
/* Non authenticated user : rely on urlLicense parameter */
|
||||
if((urlLicense.length() > 0)) {
|
||||
String licensedURL = URLLicense.releaseLicense(urlLicense);
|
||||
if (licensedURL != null) {
|
||||
url = new DigestURL(licensedURL);
|
||||
} else { // license is gone (e.g. released/remove in prev calls)
|
||||
ConcurrentLog.fine("ImageViewer", "image urlLicense not found key=" + urlLicense);
|
||||
/* Caller is responsible for handling this with appropriate HTTP status code */
|
||||
throw new InvalidURLLicenceException();
|
||||
}
|
||||
} else {
|
||||
throw new TemplateMissingParameterException("missing required code parameter");
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open input stream on image url using provided loader. All parameters must
|
||||
* not be null.
|
||||
*
|
||||
* @param post
|
||||
* post parameters.
|
||||
* @param loader.
|
||||
* Resources loader.
|
||||
* @param auth
|
||||
* true when user has credentials to load full images.
|
||||
* @param url
|
||||
* image url.
|
||||
* @return an open input stream instance (don't forget to close it).
|
||||
* @throws IOException
|
||||
* when a read/write error occured.
|
||||
*/
|
||||
public InputStream openInputStream(final serverObjects post, final LoaderDispatcher loader,
|
||||
final boolean auth, DigestURL url) throws IOException {
|
||||
InputStream inStream = null;
|
||||
if (url != null) {
|
||||
try {
|
||||
String agentName = post.get("agentName", auth ? ClientIdentification.yacyIntranetCrawlerAgentName
|
||||
: ClientIdentification.yacyInternetCrawlerAgentName);
|
||||
ClientIdentification.Agent agent = ClientIdentification.getAgent(agentName);
|
||||
inStream = loader.openInputStream(loader.request(url, false, true), CacheStrategy.IFEXIST,
|
||||
BlacklistType.SEARCH, agent);
|
||||
} catch (final IOException e) {
|
||||
/** No need to log full stack trace (in most cases resource is not available because of a network error) */
|
||||
ConcurrentLog.fine("ImageViewer", "cannot load image. URL : " + url.toNormalform(true));
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
if (inStream == null) {
|
||||
throw new IOException("Input stream could no be open");
|
||||
}
|
||||
return inStream;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param formatName
|
||||
* informal file format name. For example : "png".
|
||||
* @return true when image format will be rendered by browser and not by a YaCy service
|
||||
*/
|
||||
public static boolean isBrowserRendered(String formatName) {
|
||||
/*
|
||||
* gif images are not loaded because of an animated gif bug within jvm
|
||||
* which sends java into an endless loop with high CPU
|
||||
*/
|
||||
/*
|
||||
* svg images not supported by jdk, but by most browser, deliver just
|
||||
* content (without crop/scale)
|
||||
*/
|
||||
return ("gif".equalsIgnoreCase(formatName) || "svg".equalsIgnoreCase(formatName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Process source image to try to produce an EncodedImage instance
|
||||
* eventually scaled and clipped depending on post parameters. When
|
||||
* processed, imageInStream is closed.
|
||||
*
|
||||
* @param post
|
||||
* request post parameters. Must not be null.
|
||||
* @param auth
|
||||
* true when access rigths are OK.
|
||||
* @param url
|
||||
* image source URL. Must not be null.
|
||||
* @param ext
|
||||
* target image file format. May be null.
|
||||
* @param imageInStream
|
||||
* open stream on image content. Must not be null.
|
||||
* @return an EncodedImage instance.
|
||||
* @throws IOException
|
||||
* when image could not be parsed or encoded to specified format.
|
||||
*/
|
||||
public EncodedImage parseAndScale(serverObjects post, boolean auth, DigestURL url, String ext,
|
||||
ImageInputStream imageInStream) throws IOException {
|
||||
EncodedImage encodedImage;
|
||||
|
||||
// BufferedImage image = ImageIO.read(imageInStream);
|
||||
Iterator<ImageReader> readers = ImageIO.getImageReaders(imageInStream);
|
||||
if (!readers.hasNext()) {
|
||||
try {
|
||||
/* When no reader can be found, we have to close the stream */
|
||||
imageInStream.close();
|
||||
} catch (IOException ignoredException) {
|
||||
}
|
||||
String urlString = url.toNormalform(false);
|
||||
String errorMessage = "Image format (" + MultiProtocolURL.getFileExtension(urlString) + ") is not supported.";
|
||||
ConcurrentLog.fine("ImageViewer", errorMessage + "Image URL : " + urlString);
|
||||
/*
|
||||
* Throw an exception, wich will end in a HTTP 500 response, better
|
||||
* handled by browsers than an empty image
|
||||
*/
|
||||
throw new IOException(errorMessage);
|
||||
}
|
||||
ImageReader reader = readers.next();
|
||||
reader.setInput(imageInStream, true, true);
|
||||
|
||||
int maxwidth = post.getInt("maxwidth", 0);
|
||||
int maxheight = post.getInt("maxheight", 0);
|
||||
final boolean quadratic = post.containsKey("quadratic");
|
||||
boolean isStatic = post.getBoolean("isStatic");
|
||||
BufferedImage image = null;
|
||||
boolean returnRaw = true;
|
||||
if (!auth || maxwidth != 0 || maxheight != 0) {
|
||||
|
||||
// find original size
|
||||
final int originWidth = reader.getWidth(0);
|
||||
final int originHeigth = reader.getHeight(0);
|
||||
|
||||
// in case of not-authorized access shrink the image to
|
||||
// prevent
|
||||
// copyright problems, so that images are not larger than
|
||||
// thumbnails
|
||||
Dimension maxDimensions = calculateMaxDimensions(auth, originWidth, originHeigth, maxwidth, maxheight);
|
||||
|
||||
// if a quadratic flag is set, we cut the image out to be in
|
||||
// quadratic shape
|
||||
int w = originWidth;
|
||||
int h = originHeigth;
|
||||
if (quadratic && originWidth != originHeigth) {
|
||||
Rectangle square = getMaxSquare(originHeigth, originWidth);
|
||||
h = square.height;
|
||||
w = square.width;
|
||||
}
|
||||
|
||||
Dimension finalDimensions = calculateDimensions(w, h, maxDimensions);
|
||||
|
||||
if (originWidth != finalDimensions.width || originHeigth != finalDimensions.height) {
|
||||
returnRaw = false;
|
||||
image = readImage(reader);
|
||||
if (quadratic && originWidth != originHeigth) {
|
||||
image = makeSquare(image);
|
||||
}
|
||||
image = scale(finalDimensions.width, finalDimensions.height, image);
|
||||
}
|
||||
}
|
||||
/* Image do not need to be scaled or cropped */
|
||||
if (returnRaw) {
|
||||
if (!reader.getFormatName().equalsIgnoreCase(ext) || imageInStream.getFlushedPosition() != 0) {
|
||||
/*
|
||||
* image parsing and reencoding is only needed when source image
|
||||
* and target formats differ, or when first bytes have been discarded
|
||||
*/
|
||||
returnRaw = false;
|
||||
image = readImage(reader);
|
||||
}
|
||||
}
|
||||
if (returnRaw) {
|
||||
byte[] imageData = readRawImage(imageInStream);
|
||||
encodedImage = new EncodedImage(imageData, ext, isStatic);
|
||||
} else {
|
||||
/*
|
||||
* An error can still occur when transcoding from buffered image to
|
||||
* target ext : in that case EncodedImage.getImage() is empty.
|
||||
*/
|
||||
encodedImage = new EncodedImage(image, ext, isStatic);
|
||||
if (encodedImage.getImage().length() == 0) {
|
||||
String errorMessage = "Image could not be encoded to format : " + ext;
|
||||
ConcurrentLog.fine("ImageViewer", errorMessage + ". Image URL : " + url.toNormalform(false));
|
||||
throw new IOException(errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
return encodedImage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read image using specified reader and close ImageInputStream source.
|
||||
* Input must have bean set before using
|
||||
* {@link ImageReader#setInput(Object)}
|
||||
*
|
||||
* @param reader
|
||||
* image reader. Must not be null.
|
||||
* @return buffered image
|
||||
* @throws IOException
|
||||
* when an error occured
|
||||
*/
|
||||
private BufferedImage readImage(ImageReader reader) throws IOException {
|
||||
BufferedImage image;
|
||||
try {
|
||||
image = reader.read(0);
|
||||
} finally {
|
||||
reader.dispose();
|
||||
Object input = reader.getInput();
|
||||
if (input instanceof ImageInputStream) {
|
||||
try {
|
||||
((ImageInputStream) input).close();
|
||||
} catch (IOException ignoredException) {
|
||||
}
|
||||
}
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read image data without parsing.
|
||||
*
|
||||
* @param inStream
|
||||
* image source. Must not be null. First bytes must not have been marked discarded ({@link ImageInputStream#getFlushedPosition()} must be zero)
|
||||
* @return image data as bytes
|
||||
* @throws IOException
|
||||
* when a read/write error occured.
|
||||
*/
|
||||
private byte[] readRawImage(ImageInputStream inStream) throws IOException {
|
||||
byte[] buffer = new byte[4096];
|
||||
int l = 0;
|
||||
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
|
||||
inStream.seek(0);
|
||||
try {
|
||||
while ((l = inStream.read(buffer)) >= 0) {
|
||||
outStream.write(buffer, 0, l);
|
||||
}
|
||||
return outStream.toByteArray();
|
||||
} finally {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate image dimensions from image original dimensions, max
|
||||
* dimensions, and target dimensions.
|
||||
*
|
||||
* @return dimensions to render image
|
||||
*/
|
||||
protected Dimension calculateDimensions(final int originWidth, final int originHeight, final Dimension max) {
|
||||
int resultWidth;
|
||||
int resultHeight;
|
||||
if (max.width < originWidth || max.height < originHeight) {
|
||||
// scale image
|
||||
final double hs = (originWidth <= max.width) ? 1.0 : ((double) max.width) / ((double) originWidth);
|
||||
final double vs = (originHeight <= max.height) ? 1.0 : ((double) max.height) / ((double) originHeight);
|
||||
final double scale = Math.min(hs, vs);
|
||||
// if (!auth) scale = Math.min(scale, 0.6); // this is for copyright
|
||||
// purpose
|
||||
if (scale < 1.0) {
|
||||
resultWidth = Math.max(1, (int) (originWidth * scale));
|
||||
resultHeight = Math.max(1, (int) (originHeight * scale));
|
||||
} else {
|
||||
resultWidth = Math.max(1, originWidth);
|
||||
resultHeight = Math.max(1, originHeight);
|
||||
}
|
||||
|
||||
} else {
|
||||
// do not scale
|
||||
resultWidth = originWidth;
|
||||
resultHeight = originHeight;
|
||||
}
|
||||
return new Dimension(resultWidth, resultHeight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate image maximum dimentions from original and specified maximum
|
||||
* dimensions
|
||||
*
|
||||
* @param auth
|
||||
* true when acces rigths are OK.
|
||||
* @return maximum dimensions to render image
|
||||
*/
|
||||
protected Dimension calculateMaxDimensions(final boolean auth, final int originWidth, final int originHeight,
|
||||
final int maxWidth, final int maxHeight) {
|
||||
int resultWidth;
|
||||
int resultHeight;
|
||||
// in case of not-authorized access shrink the image to prevent
|
||||
// copyright problems, so that images are not larger than thumbnails
|
||||
if (auth) {
|
||||
resultWidth = (maxWidth == 0) ? originWidth : maxWidth;
|
||||
resultHeight = (maxHeight == 0) ? originHeight : maxHeight;
|
||||
} else if ((originWidth > 16) || (originHeight > 16)) {
|
||||
resultWidth = Math.min(96, originWidth);
|
||||
resultHeight = Math.min(96, originHeight);
|
||||
} else {
|
||||
resultWidth = 16;
|
||||
resultHeight = 16;
|
||||
}
|
||||
return new Dimension(resultWidth, resultHeight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale image to specified dimensions
|
||||
*
|
||||
* @param width
|
||||
* target width
|
||||
* @param height
|
||||
* target height
|
||||
* @param image
|
||||
* image to scale. Must not be null.
|
||||
* @return a scaled image
|
||||
*/
|
||||
public BufferedImage scale(final int width, final int height, final BufferedImage image) {
|
||||
// compute scaled image
|
||||
Image scaled = image.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
|
||||
final MediaTracker mediaTracker = new MediaTracker(new Container());
|
||||
mediaTracker.addImage(scaled, 0);
|
||||
try {
|
||||
mediaTracker.waitForID(0);
|
||||
} catch (final InterruptedException e) {
|
||||
}
|
||||
|
||||
// make a BufferedImage out of that
|
||||
BufferedImage result = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
|
||||
try {
|
||||
result.createGraphics().drawImage(scaled, 0, 0, width, height, null);
|
||||
// check outcome
|
||||
final Raster raster = result.getData();
|
||||
int[] pixel = new int[raster.getSampleModel().getNumBands()];
|
||||
pixel = raster.getPixel(0, 0, pixel);
|
||||
} catch (final Exception e) {
|
||||
/*
|
||||
* Exception may be caused by source image color model : try now to
|
||||
* convert to RGB before scaling
|
||||
*/
|
||||
try {
|
||||
BufferedImage converted = EncodedImage.convertToRGB(image);
|
||||
scaled = converted.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);
|
||||
mediaTracker.addImage(scaled, 1);
|
||||
try {
|
||||
mediaTracker.waitForID(1);
|
||||
} catch (final InterruptedException e2) {
|
||||
}
|
||||
result = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
|
||||
result.createGraphics().drawImage(scaled, 0, 0, width, height, null);
|
||||
|
||||
// check outcome
|
||||
final Raster raster = result.getData();
|
||||
int[] pixel = new int[result.getSampleModel().getNumBands()];
|
||||
pixel = raster.getPixel(0, 0, pixel);
|
||||
} catch (Exception e2) {
|
||||
result = image;
|
||||
}
|
||||
|
||||
ConcurrentLog.fine("ImageViewer", "Image could not be scaled");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param h
|
||||
* image height
|
||||
* @param w
|
||||
* image width
|
||||
* @return max square area fitting inside dimensions
|
||||
*/
|
||||
public Rectangle getMaxSquare(final int h, final int w) {
|
||||
Rectangle square;
|
||||
if (w > h) {
|
||||
final int offset = (w - h) / 2;
|
||||
square = new Rectangle(offset, 0, h, h);
|
||||
} else {
|
||||
final int offset = (h - w) / 2;
|
||||
square = new Rectangle(0, offset, w, w);
|
||||
}
|
||||
return square;
|
||||
}
|
||||
|
||||
/**
|
||||
* Crop image to make a square
|
||||
*
|
||||
* @param image
|
||||
* image to crop
|
||||
* @return
|
||||
*/
|
||||
public BufferedImage makeSquare(BufferedImage image) {
|
||||
final int w = image.getWidth();
|
||||
final int h = image.getHeight();
|
||||
if (w > h) {
|
||||
final BufferedImage dst = new BufferedImage(h, h, BufferedImage.TYPE_INT_ARGB);
|
||||
Graphics2D g = dst.createGraphics();
|
||||
final int offset = (w - h) / 2;
|
||||
try {
|
||||
g.drawImage(image, 0, 0, h - 1, h - 1, offset, 0, h + offset, h - 1, null);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
image = dst;
|
||||
} else {
|
||||
final BufferedImage dst = new BufferedImage(w, w, BufferedImage.TYPE_INT_ARGB);
|
||||
Graphics2D g = dst.createGraphics();
|
||||
final int offset = (h - w) / 2;
|
||||
try {
|
||||
g.drawImage(image, 0, 0, w - 1, w - 1, 0, offset, w - 1, w + offset, null);
|
||||
} finally {
|
||||
g.dispose();
|
||||
}
|
||||
image = dst;
|
||||
}
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,125 @@
|
||||
/**
|
||||
* ContentScraperTest
|
||||
* part of YaCy
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.document.parser.html;
|
||||
|
||||
import java.awt.Dimension;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Unit tests for ContentScrapper class.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class ContentScraperTest {
|
||||
|
||||
@Test
|
||||
public final void testParseSizes() {
|
||||
/* Normal case */
|
||||
Set<Dimension> sizes = ContentScraper.parseSizes("96x128");
|
||||
Assert.assertEquals(1, sizes.size());
|
||||
Assert.assertTrue(sizes.contains(new Dimension(96, 128)));
|
||||
|
||||
/* "any" keyword */
|
||||
sizes = ContentScraper.parseSizes("any");
|
||||
Assert.assertEquals(0, sizes.size());
|
||||
|
||||
/* Multiple valid sizes, lower and upper case separator */
|
||||
sizes = ContentScraper.parseSizes("96x128 16X16 1X2 1024x768");
|
||||
Assert.assertEquals(4, sizes.size());
|
||||
Assert.assertTrue(sizes.contains(new Dimension(96, 128)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(16, 16)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(1, 2)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(1024, 768)));
|
||||
|
||||
/* Duplicate entries */
|
||||
sizes = ContentScraper.parseSizes("96x128 96X128 1X2 96x128");
|
||||
Assert.assertEquals(2, sizes.size());
|
||||
Assert.assertTrue(sizes.contains(new Dimension(96, 128)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(1, 2)));
|
||||
|
||||
/* Mutiple inner and trailing spaces */
|
||||
sizes = ContentScraper.parseSizes(" 96x128 16X16 ");
|
||||
Assert.assertEquals(2, sizes.size());
|
||||
Assert.assertTrue(sizes.contains(new Dimension(96, 128)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(16, 16)));
|
||||
|
||||
/* Empty string */
|
||||
sizes = ContentScraper.parseSizes("");
|
||||
Assert.assertEquals(0, sizes.size());
|
||||
|
||||
/* null string */
|
||||
sizes = ContentScraper.parseSizes(null);
|
||||
Assert.assertEquals(0, sizes.size());
|
||||
|
||||
/* Invalid sizes */
|
||||
sizes = ContentScraper.parseSizes("096x0128 -16x-16 0x0 x768 78x axb 1242");
|
||||
Assert.assertEquals(0, sizes.size());
|
||||
|
||||
/* Mix of valid and invalid sizes */
|
||||
sizes = ContentScraper.parseSizes("96x128 16X16 axb 123 78x32");
|
||||
Assert.assertEquals(3, sizes.size());
|
||||
Assert.assertTrue(sizes.contains(new Dimension(96, 128)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(16, 16)));
|
||||
Assert.assertTrue(sizes.contains(new Dimension(78, 32)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public final void testParseSpaceSeparatedTokens() {
|
||||
/* Normal case */
|
||||
Set<String> tokens = ContentScraper.parseSpaceSeparatedTokens("abc de");
|
||||
Assert.assertEquals(2, tokens.size());
|
||||
Assert.assertTrue(tokens.contains("abc"));
|
||||
Assert.assertTrue(tokens.contains("de"));
|
||||
|
||||
/* One item only */
|
||||
tokens = ContentScraper.parseSpaceSeparatedTokens("abc");
|
||||
Assert.assertEquals(1, tokens.size());
|
||||
Assert.assertTrue(tokens.contains("abc"));
|
||||
|
||||
/* Mutiple inner and trailing spaces */
|
||||
tokens = ContentScraper.parseSpaceSeparatedTokens(" abc d efff fgj ");
|
||||
Assert.assertEquals(4, tokens.size());
|
||||
Assert.assertTrue(tokens.contains("abc"));
|
||||
Assert.assertTrue(tokens.contains("d"));
|
||||
Assert.assertTrue(tokens.contains("efff"));
|
||||
Assert.assertTrue(tokens.contains("fgj"));
|
||||
|
||||
/* Duplicate entries */
|
||||
tokens = ContentScraper.parseSpaceSeparatedTokens("abc bb abc abc ABC");
|
||||
Assert.assertEquals(3, tokens.size());
|
||||
Assert.assertTrue(tokens.contains("abc"));
|
||||
/* ignoring case is not the purpose of this function */
|
||||
Assert.assertTrue(tokens.contains("ABC"));
|
||||
Assert.assertTrue(tokens.contains("bb"));
|
||||
|
||||
/* Empty string */
|
||||
tokens = ContentScraper.parseSpaceSeparatedTokens("");
|
||||
Assert.assertEquals(0, tokens.size());
|
||||
|
||||
/* Null string */
|
||||
tokens = ContentScraper.parseSpaceSeparatedTokens(null);
|
||||
Assert.assertEquals(0, tokens.size());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,192 @@
|
||||
/**
|
||||
* IconEntryTest
|
||||
* part of YaCy
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.document.parser.html;
|
||||
|
||||
import java.awt.Dimension;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
|
||||
/**
|
||||
* Unit tests for IconEntry class.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class IconEntryTest {
|
||||
|
||||
@Test
|
||||
public final void testGetDistance() {
|
||||
/* Normal case : one size has both width and height greater */
|
||||
Dimension size1 = new Dimension(5, 8);
|
||||
Dimension size2 = new Dimension(7, 12);
|
||||
Assert.assertEquals(3.0, IconEntry.getDistance(size1, size2), 0.0);
|
||||
/* Check inverted parameters should produces same result */
|
||||
Assert.assertEquals(3.0, IconEntry.getDistance(size2, size1), 0.0);
|
||||
/* Equal sizes */
|
||||
size2 = new Dimension(5, 8);
|
||||
Assert.assertEquals(0.0, IconEntry.getDistance(size1, size2), 0.0);
|
||||
/* Equal sizes */
|
||||
size2 = new Dimension(5, 8);
|
||||
Assert.assertEquals(0.0, IconEntry.getDistance(size1, size2), 0.0);
|
||||
/* Only one dimension differs */
|
||||
size2 = new Dimension(5, 12);
|
||||
Assert.assertEquals(2.0, IconEntry.getDistance(size1, size2), 0.0);
|
||||
size2 = new Dimension(10, 8);
|
||||
Assert.assertEquals(2.5, IconEntry.getDistance(size1, size2), 0.0);
|
||||
/* width lower, height upper */
|
||||
size2 = new Dimension(3, 12);
|
||||
Assert.assertEquals(3.0, IconEntry.getDistance(size1, size2), 0.0);
|
||||
/* negative values */
|
||||
size1 = new Dimension(-5, -8);
|
||||
size2 = new Dimension(-7, -12);
|
||||
Assert.assertEquals(3.0, IconEntry.getDistance(size1, size2), 0.0);
|
||||
/* one null */
|
||||
size1 = null;
|
||||
size2 = new Dimension(-7, -12);
|
||||
Assert.assertEquals(Double.MAX_VALUE, IconEntry.getDistance(size1, size2), 0.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public final void testGetClosestSize() throws MalformedURLException {
|
||||
/* Preferred size in sizes set */
|
||||
Set<String> rels = new HashSet<>();
|
||||
rels.add(IconLinkRelations.ICON.getRelValue());
|
||||
|
||||
Set<Dimension> sizes = new HashSet<>();
|
||||
sizes.add(new Dimension(128,128));
|
||||
sizes.add(new Dimension(256,512));
|
||||
sizes.add(new Dimension(16,16));
|
||||
|
||||
Dimension preferredSize = new Dimension(16, 16);
|
||||
IconEntry icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
Dimension result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertEquals(preferredSize, result);
|
||||
|
||||
/* Preferred size lower than all sizes in set */
|
||||
preferredSize = new Dimension(12, 12);
|
||||
result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertEquals(new Dimension(16,16), result);
|
||||
|
||||
/* Preferred size over than all sizes in set */
|
||||
preferredSize = new Dimension(1992, 1024);
|
||||
result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertEquals(new Dimension(256, 512), result);
|
||||
|
||||
/* Preferred size between sizes in set */
|
||||
preferredSize = new Dimension(17, 18);
|
||||
result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertEquals(new Dimension(16, 16), result);
|
||||
|
||||
/* Sizes set contains only one item */
|
||||
sizes = new HashSet<>();
|
||||
sizes.add(new Dimension(128,128));
|
||||
icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
preferredSize = new Dimension(1992, 1024);
|
||||
result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertEquals(new Dimension(128, 128), result);
|
||||
|
||||
/* Empty sizes set */
|
||||
sizes = new HashSet<>();
|
||||
icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
preferredSize = new Dimension(16, 16);
|
||||
result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertNull(result);
|
||||
|
||||
/* Null preferred size */
|
||||
sizes = new HashSet<>();
|
||||
sizes.add(new Dimension(128,128));
|
||||
sizes.add(new Dimension(256,512));
|
||||
sizes.add(new Dimension(16,16));
|
||||
icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
preferredSize = null;
|
||||
result = icon.getClosestSize(preferredSize);
|
||||
Assert.assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public final void testSizesToString() throws MalformedURLException {
|
||||
/* Multiple values in sizes set */
|
||||
Set<String> rels = new HashSet<>();
|
||||
rels.add(IconLinkRelations.ICON.getRelValue());
|
||||
|
||||
Set<Dimension> sizes = new HashSet<>();
|
||||
sizes.add(new Dimension(128,128));
|
||||
sizes.add(new Dimension(256,512));
|
||||
sizes.add(new Dimension(16,16));
|
||||
|
||||
IconEntry icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
String sizesStr = icon.sizesToString();
|
||||
/* The set is not ordered, only check result contains what we expect */
|
||||
Assert.assertTrue(sizesStr.contains("128x128"));
|
||||
Assert.assertTrue(sizesStr.contains("256x512"));
|
||||
Assert.assertTrue(sizesStr.contains("16x16"));
|
||||
Assert.assertTrue(sizesStr.contains(" "));
|
||||
|
||||
/* One value in sizes set */
|
||||
sizes = new HashSet<>();
|
||||
sizes.add(new Dimension(128,128));
|
||||
|
||||
icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
sizesStr = icon.sizesToString();
|
||||
Assert.assertEquals("128x128", sizesStr);
|
||||
|
||||
/* Empty sizes set */
|
||||
sizes = new HashSet<>();
|
||||
|
||||
icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
sizesStr = icon.sizesToString();
|
||||
Assert.assertTrue(sizesStr.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public final void testRelToString() throws MalformedURLException {
|
||||
/* Multiple values in rel set */
|
||||
Set<String> rels = new HashSet<>();
|
||||
rels.add(IconLinkRelations.ICON.getRelValue());
|
||||
rels.add(IconLinkRelations.APPLE_TOUCH_ICON.getRelValue());
|
||||
rels.add(IconLinkRelations.MASK_ICON.getRelValue());
|
||||
|
||||
Set<Dimension> sizes = new HashSet<>();
|
||||
sizes.add(new Dimension(128,128));
|
||||
|
||||
IconEntry icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
String relStr = icon.relToString();
|
||||
/* The set is not ordered, only check result contains what we expect */
|
||||
Assert.assertTrue(relStr.contains(IconLinkRelations.ICON.getRelValue()));
|
||||
Assert.assertTrue(relStr.contains(IconLinkRelations.APPLE_TOUCH_ICON.getRelValue()));
|
||||
Assert.assertTrue(relStr.contains(IconLinkRelations.MASK_ICON.getRelValue()));
|
||||
Assert.assertTrue(relStr.contains(" "));
|
||||
|
||||
/* One value in rel set */
|
||||
rels = new HashSet<>();
|
||||
rels.add(IconLinkRelations.ICON.getRelValue());
|
||||
|
||||
icon = new IconEntry(new DigestURL("http://yacy.net"), rels, sizes);
|
||||
relStr = icon.relToString();
|
||||
Assert.assertEquals(IconLinkRelations.ICON.getRelValue(), relStr);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,213 @@
|
||||
/**
|
||||
* URIMetadataNodeTest
|
||||
* part of YaCy
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.kelondro.data.meta;
|
||||
|
||||
import java.awt.Dimension;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.document.parser.html.IconEntry;
|
||||
import net.yacy.search.schema.CollectionConfiguration;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
/**
|
||||
* Unit tests for URIMetadataNode class.
|
||||
*
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class URIMetadataNodeTest {
|
||||
|
||||
/**
|
||||
* Three standard icons with different sizes, one non-standard with a larger
|
||||
* size
|
||||
*/
|
||||
@Test
|
||||
public final void testGetIcons4Items() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode
|
||||
.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/icon16.png", "somehost.org/static/images/icon32.png",
|
||||
"somehost.org/static/images/icon64.png",
|
||||
"somehost.org/static/images/iconApple128.png" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http", "https", "https", "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(),
|
||||
new String[] { "icon", "icon", "icon", "apple-touch-icon" });
|
||||
metadataNode.setField(CollectionSchema.icons_sizes_sxt.getSolrFieldName(),
|
||||
new String[] { "16x24", "32x32", "58x64", "128x128" });
|
||||
|
||||
Collection<IconEntry> icons = metadataNode.getIcons();
|
||||
int nb = 0;
|
||||
/* Check results consistency */
|
||||
for (IconEntry icon : icons) {
|
||||
if ("http://somehost.org/static/images/icon16.png".equals(icon.getUrl().toNormalform(false))) {
|
||||
Assert.assertEquals(1, icon.getSizes().size());
|
||||
Dimension size = icon.getSizes().iterator().next();
|
||||
Assert.assertEquals(16, size.width);
|
||||
Assert.assertEquals(24, size.height);
|
||||
Assert.assertEquals(1, icon.getRel().size());
|
||||
Assert.assertEquals("icon", icon.getRel().iterator().next());
|
||||
nb++;
|
||||
} else if ("https://somehost.org/static/images/icon32.png".equals(icon.getUrl().toNormalform(false))) {
|
||||
Assert.assertEquals(1, icon.getSizes().size());
|
||||
Dimension size = icon.getSizes().iterator().next();
|
||||
Assert.assertEquals(32, size.width);
|
||||
Assert.assertEquals(32, size.height);
|
||||
Assert.assertEquals(1, icon.getRel().size());
|
||||
Assert.assertEquals("icon", icon.getRel().iterator().next());
|
||||
nb++;
|
||||
} else if ("https://somehost.org/static/images/icon64.png".equals(icon.getUrl().toNormalform(false))) {
|
||||
Assert.assertEquals(1, icon.getSizes().size());
|
||||
Dimension size = icon.getSizes().iterator().next();
|
||||
Assert.assertEquals(58, size.width);
|
||||
Assert.assertEquals(64, size.height);
|
||||
Assert.assertEquals(1, icon.getRel().size());
|
||||
Assert.assertEquals("icon", icon.getRel().iterator().next());
|
||||
nb++;
|
||||
} else if ("http://somehost.org/static/images/iconApple128.png".equals(icon.getUrl().toNormalform(false))) {
|
||||
Assert.assertEquals(1, icon.getSizes().size());
|
||||
Dimension size = icon.getSizes().iterator().next();
|
||||
Assert.assertEquals(128, size.width);
|
||||
Assert.assertEquals(128, size.height);
|
||||
Assert.assertEquals(1, icon.getRel().size());
|
||||
Assert.assertEquals("apple-touch-icon", icon.getRel().iterator().next());
|
||||
nb++;
|
||||
}
|
||||
}
|
||||
Assert.assertEquals(4, nb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Only icons_urlstub_sxt field valued
|
||||
*/
|
||||
@Test
|
||||
public final void testGetIconsOnlyIconsUrlstubSxt() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode
|
||||
.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/icon16.png", "somehost.org/static/images/icon32.png",
|
||||
"somehost.org/static/images/icon64.png",
|
||||
"somehost.org/static/images/iconApple124.png" });
|
||||
|
||||
Collection<IconEntry> icons = metadataNode.getIcons();
|
||||
Assert.assertEquals(4, icons.size());
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Only one standard icon
|
||||
*/
|
||||
@Test
|
||||
public final void testGetIcons1Item() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/icon16.png" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(), new String[] { "icon" });
|
||||
metadataNode.setField(CollectionSchema.icons_sizes_sxt.getSolrFieldName(), new String[] { "16x16" });
|
||||
|
||||
Collection<IconEntry> icons = metadataNode.getIcons();
|
||||
Assert.assertEquals(1, icons.size());
|
||||
IconEntry icon = icons.iterator().next();
|
||||
Assert.assertEquals(1, icon.getSizes().size());
|
||||
Dimension size = icon.getSizes().iterator().next();
|
||||
Assert.assertEquals(16.0, size.getWidth(), 0.0);
|
||||
Assert.assertEquals(16.0, size.getHeight(), 0.0);
|
||||
}
|
||||
|
||||
/**
|
||||
* No Icon
|
||||
*/
|
||||
@Test
|
||||
public final void testGetIconsNoIcon() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
|
||||
Collection<IconEntry> icons = metadataNode.getIcons();
|
||||
Assert.assertEquals(0, icons.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check encoding/decoding consistency
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testEncodeDecode() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode
|
||||
.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/icon16.png", "somehost.org/static/images/icon32.png",
|
||||
"somehost.org/static/images/icon64.png",
|
||||
"somehost.org/static/images/iconApple128.png" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http", "https", "https", "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(),
|
||||
new String[] { "icon", "icon", "icon", "apple-touch-icon" });
|
||||
metadataNode.setField(CollectionSchema.icons_sizes_sxt.getSolrFieldName(),
|
||||
new String[] { "16x24", "32x32", "58x64", "128x128" });
|
||||
|
||||
String encoded = metadataNode.toString();
|
||||
URIMetadataNode decoded = URIMetadataNode.importEntry(encoded, "dht");
|
||||
Collection<IconEntry> icons = decoded.getIcons();
|
||||
|
||||
/*
|
||||
* Only icon which is the closest to 16x16 pixels is encoded, and sizes
|
||||
* and rel attribute are not encoded
|
||||
*/
|
||||
Assert.assertEquals(1, icons.size());
|
||||
IconEntry icon = icons.iterator().next();
|
||||
|
||||
Assert.assertEquals(0, icon.getSizes().size());
|
||||
|
||||
Assert.assertEquals("http://somehost.org/static/images/icon16.png", icon.getUrl().toNormalform(false));
|
||||
|
||||
Assert.assertEquals(1, icon.getRel().size());
|
||||
Assert.assertEquals("icon", icon.getRel().iterator().next());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check encoding/decoding consistency when document has no indexed icon
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testEncodeDecodeNoIcon() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
|
||||
String encoded = metadataNode.toString();
|
||||
URIMetadataNode decoded = URIMetadataNode.importEntry(encoded, "dht");
|
||||
Collection<IconEntry> icons = decoded.getIcons();
|
||||
|
||||
Assert.assertEquals(0, icons.size());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,192 @@
|
||||
|
||||
/**
|
||||
* yacysearchitemTest
|
||||
* Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import java.awt.Dimension;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.kelondro.data.meta.URIMetadataNode;
|
||||
import net.yacy.search.schema.CollectionConfiguration;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
/**
|
||||
* Unit tests for yacysearchitem class.
|
||||
*
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class yacysearchitemTest {
|
||||
|
||||
/**
|
||||
* Three standard icons with different sizes, one non-standard with a larger
|
||||
* size
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testGetFaviconURL() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode
|
||||
.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "someHost.org/static/images/icon16.png", "somehost.org/static/images/icon32.png",
|
||||
"somehost.org/static/images/icon64.png",
|
||||
"somehost.org/static/images/iconApple124.png" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http", "http", "http", "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(),
|
||||
new String[] { "icon", "icon", "icon", "apple-touch-icon" });
|
||||
metadataNode.setField(CollectionSchema.icons_sizes_sxt.getSolrFieldName(),
|
||||
new String[] { "16x16", "32x32", "64x64", "128x128" });
|
||||
|
||||
/* Search for a size present in icons collection */
|
||||
DigestURL faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(32, 32));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/icon32.png", faviconURL.toNormalform(false));
|
||||
|
||||
/* Search for a size not in icons collection */
|
||||
faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(40, 40));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/icon32.png", faviconURL.toNormalform(false));
|
||||
|
||||
/*
|
||||
* Search for a size equals to non-standard : standard icon is stil
|
||||
* preffered
|
||||
*/
|
||||
faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(128, 128));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/icon64.png", faviconURL.toNormalform(false));
|
||||
}
|
||||
|
||||
/**
|
||||
* Only non-standard icons
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testGetFaviconURLNonStandard() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode
|
||||
.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/mask32.png",
|
||||
"somehost.org/static/images/fluid.64.png",
|
||||
"somehost.org/static/images/iconApple124.png" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http", "http", "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(),
|
||||
new String[] { "mask-icon", "fluid-icon", "apple-touch-icon" });
|
||||
metadataNode.setField(CollectionSchema.icons_sizes_sxt.getSolrFieldName(),
|
||||
new String[] { "32x32", "64x64", "128x128" });
|
||||
|
||||
/* Non standard icon is returned as fallback */
|
||||
DigestURL faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(32, 32));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/mask32.png", faviconURL.toNormalform(false));
|
||||
}
|
||||
|
||||
/**
|
||||
* One standard icon with multiple sizes
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testGetFaviconURLMultiSizes() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/favicon.ico" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(), new String[] { "icon" });
|
||||
metadataNode.setField(CollectionSchema.icons_sizes_sxt.getSolrFieldName(),
|
||||
new String[] { "16x16 32x32 64x64", });
|
||||
|
||||
/* Search for a size in sizes set */
|
||||
DigestURL faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(32, 32));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/favicon.ico", faviconURL.toNormalform(false));
|
||||
|
||||
/* Search for a size not in sizes set */
|
||||
faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(40, 40));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/favicon.ico", faviconURL.toNormalform(false));
|
||||
}
|
||||
|
||||
/**
|
||||
* One standard icon with no size
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testGetFaviconURLNoSize() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/favicon.ico" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(), new String[] { "icon" });
|
||||
|
||||
DigestURL faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(32, 32));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/favicon.ico", faviconURL.toNormalform(false));
|
||||
}
|
||||
|
||||
/**
|
||||
* One non-standard icon with no size
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testGetFaviconURLNonStandardNoSize() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://somehost.org"));
|
||||
metadataNode.setField(CollectionSchema.icons_urlstub_sxt.getSolrFieldName(),
|
||||
new String[] { "somehost.org/static/images/favicon.png" });
|
||||
List<String> protocols = CollectionConfiguration
|
||||
.protocolList2indexedList(Arrays.asList(new String[] { "http" }));
|
||||
metadataNode.setField(CollectionSchema.icons_protocol_sxt.getSolrFieldName(), protocols);
|
||||
metadataNode.setField(CollectionSchema.icons_rel_sxt.getSolrFieldName(), new String[] { "appel-touch-icon" });
|
||||
|
||||
DigestURL faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(32, 32));
|
||||
Assert.assertNotNull(faviconURL);
|
||||
Assert.assertEquals("http://somehost.org/static/images/favicon.png", faviconURL.toNormalform(false));
|
||||
}
|
||||
|
||||
/**
|
||||
* No icon in document
|
||||
*
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
@Test
|
||||
public final void testGetFaviconURLNoIcon() throws MalformedURLException {
|
||||
URIMetadataNode metadataNode = new URIMetadataNode(new DigestURL("http://someHost.org"));
|
||||
|
||||
/* Default fallback favicon URL should be generated */
|
||||
DigestURL faviconURL = yacysearchitem.getFaviconURL(metadataNode, new Dimension(32, 32));
|
||||
Assert.assertEquals("http://somehost.org/favicon.ico", faviconURL.toNormalform(false));
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,96 @@
|
||||
// URLLicense.java
|
||||
// Copyright 2016 by luccioman; https://github.com/luccioman
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.data;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
import net.yacy.cora.document.id.DigestURL;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
|
||||
/**
|
||||
* Test URLLicence reliability when used by concurrent threads
|
||||
*
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
public class URLLicenseConcurrentTest {
|
||||
|
||||
/**
|
||||
* Thread emulating a client who tries to fetch some url content.
|
||||
* @author luc
|
||||
*
|
||||
*/
|
||||
private static class ClientThread extends Thread {
|
||||
|
||||
private String testURL = "http://yacy.net";
|
||||
|
||||
private int steps = 100000;
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
System.out.println(this.getName() + " started...");
|
||||
DigestURL url = null;
|
||||
try {
|
||||
url = new DigestURL(this.testURL);
|
||||
} catch (MalformedURLException e1) {
|
||||
e1.printStackTrace();
|
||||
}
|
||||
String normalizedURL = url.toNormalform(true);
|
||||
for (int step = 0; step < this.steps; step++) {
|
||||
String license = URLLicense.aquireLicense(url);
|
||||
// You can eventually call here Thread.sleep()
|
||||
String retrievedURL = URLLicense.releaseLicense(license);
|
||||
if (!normalizedURL.equals(retrievedURL)) {
|
||||
System.err.println("Licence lost! license : " + license + ", step : " + step + ", Thread : " + this.getName());
|
||||
}
|
||||
}
|
||||
System.out.println(this.getName() + " finished!");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs clients concurrently : until the end, no error message should be displayed in console.
|
||||
* @param args
|
||||
*/
|
||||
public static void main(String args[]) {
|
||||
long beginTime = System.nanoTime();
|
||||
try {
|
||||
ClientThread[] threads = new ClientThread[10];
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
threads[i] = new URLLicenseConcurrentTest.ClientThread();
|
||||
threads[i].setName("ClientThread" + i);
|
||||
threads[i].start();
|
||||
}
|
||||
for (int i = 0; i < threads.length; i++) {
|
||||
try {
|
||||
threads[i].join();
|
||||
} catch (InterruptedException e) {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
long time = System.nanoTime() - beginTime;
|
||||
System.out.println("Test run in " + time / 1000000 + "ms");
|
||||
ConcurrentLog.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue