- 
+ 
#[title]#
#(heuristic)#::
diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java
index b6c2c41dd..7d1d1295a 100644
--- a/htroot/yacysearchitem.java
+++ b/htroot/yacysearchitem.java
@@ -194,8 +194,7 @@ public class yacysearchitem {
boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");
String resultFileName = resultURL.getFileName();
prop.putHTML("content_target", target);
- //if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL, 1024 * 1024 * 10, null, ClientIdentification.yacyIntranetCrawlerAgent);
- prop.putHTML("content_faviconCode", URLLicense.aquireLicense(faviconURL)); // acquire license for favicon url loading
+ prop.putHTML("content_faviconUrl", processFaviconURL(authenticated, faviconURL));
prop.put("content_urlhash", urlhash);
prop.put("content_ranking", Float.toString(result.score()));
Date[] events = result.events();
@@ -310,7 +309,7 @@ public class yacysearchitem {
if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) {
// image search; shows thumbnails
- processImage(sb, prop, item, theSearch, target_special_pattern, timeout);
+ processImage(sb, prop, item, theSearch, target_special_pattern, timeout, authenticated);
theSearch.query.transmitcount = item + 1;
return prop;
}
@@ -342,6 +341,28 @@ public class yacysearchitem {
return prop;
}
+
+ /**
+ * @param authenticated
+ * true when current user is authenticated
+ * @param faviconURL
+ * url icon of web site
+ * @return url to propose in search result or empty string when faviconURL
+ * is null
+ */
+ private static String processFaviconURL(final boolean authenticated, DigestURL faviconURL) {
+ /* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
+ StringBuilder contentFaviconURL = new StringBuilder();
+ if (faviconURL != null) {
+ contentFaviconURL.append("ViewImage.png?width=16&height=16&isStatic=true");
+ if (authenticated) {
+ contentFaviconURL.append("&url=").append(faviconURL.toNormalform(true));
+ } else {
+ contentFaviconURL.append("&code=").append(URLLicense.aquireLicense(faviconURL));
+ }
+ }
+ return contentFaviconURL.toString();
+ }
/**
@@ -352,9 +373,10 @@ public class yacysearchitem {
* @param theSearch search event
* @param target_special_pattern
* @param timeout result getting timeOut
+ * @param authenticated set to true when user authentication is ok
*/
private static void processImage(final Switchboard sb, final serverObjects prop, final int item,
- final SearchEvent theSearch, final String target_special_pattern, long timeout) {
+ final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean authenticated) {
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
try {
SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout);
@@ -366,9 +388,24 @@ public class yacysearchitem {
/* Image format ouput for ViewImage servlet : default is png, except with gif and svg images */
final String viewImageExt = !imageUrlExt.isEmpty() && ViewImage.isBrowserRendered(imageUrlExt) ? imageUrlExt : "png";
/* Thumb URL */
- prop.putHTML("content_item_hrefCache", "ViewImage." + viewImageExt + "?maxwidth=" + DEFAULT_IMG_WIDTH + "&maxheight=" + DEFAULT_IMG_HEIGHT + "&code="+license+"&isStatic=true&quadratic=&url=" + imageUrlstring);
+ StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=")
+ .append(DEFAULT_IMG_WIDTH).append("&maxheight=").append(DEFAULT_IMG_HEIGHT)
+ .append("&isStatic=true&quadratic");
+ /* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
+ if(authenticated) {
+ thumbURLBuilder.append("&url=").append(imageUrlstring);
+ } else {
+ thumbURLBuilder.append("&code=").append(URLLicense.aquireLicense(image.imageUrl));
+ }
+ String thumbURL = thumbURLBuilder.toString();
+ prop.putHTML("content_item_hrefCache", thumbURL);
/* Full size preview URL */
- prop.putHTML("content_item_hrefFullPreview", "ViewImage." + viewImageExt + "?code="+license+"&isStatic=true&url=" + imageUrlstring);
+ if(authenticated) {
+ prop.putHTML("content_item_hrefFullPreview", "ViewImage." + viewImageExt + "?isStatic=true&url=" + imageUrlstring);
+ } else {
+ /* Not authenticated : full preview URL must be the same as thumb URL */
+ prop.putHTML("content_item_hrefFullPreview", thumbURL);
+ }
prop.putHTML("content_item_href", imageUrlstring);
prop.putHTML("content_item_target", target);
prop.put("content_item_code", license);
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
index 3588a336c..621c6dcca 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
@@ -30,14 +30,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import net.yacy.cora.document.analysis.Classification;
-import net.yacy.cora.document.id.MultiProtocolURL;
-import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;
-import net.yacy.cora.protocol.HeaderFramework;
-import net.yacy.data.URLLicense;
-import net.yacy.search.schema.CollectionSchema;
-import net.yacy.server.serverObjects;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.util.NamedList;
@@ -50,6 +42,13 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
+import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;
+import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.search.schema.CollectionSchema;
+import net.yacy.server.serverObjects;
+
/**
* write the opensearch result in YaCys special way to include as much as in opensearch is included.
@@ -159,8 +158,6 @@ public class YJsonResponseWriter implements QueryResponseWriter {
String filename = url.getFileName();
solitaireTag(writer, "link", u);
solitaireTag(writer, "file", filename);
- // get image license
- if (Classification.isImageExtension(MultiProtocolURL.getFileExtension(filename))) URLLicense.aquireLicense(urlhash, url.toNormalform(true));
} catch (final MalformedURLException e) {}
continue;
}
@@ -221,7 +218,9 @@ public class YJsonResponseWriter implements QueryResponseWriter {
if (i < responseCount - 1) {
writer.write(",\n".toCharArray());
}
- } catch (final Throwable ee) {}
+ } catch (final Throwable ee) {
+ ConcurrentLog.fine("YJsonResponseWriter", "Document writing error : " + ee.getMessage());
+ }
}
writer.write("],\n".toCharArray());
diff --git a/source/net/yacy/data/URLLicense.java b/source/net/yacy/data/URLLicense.java
index 2f0f2cded..4e8dfe001 100644
--- a/source/net/yacy/data/URLLicense.java
+++ b/source/net/yacy/data/URLLicense.java
@@ -28,36 +28,53 @@ package net.yacy.data;
import java.util.Collections;
import java.util.Map;
+import java.util.UUID;
-import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.storage.SizeLimitedMap;
-
+/**
+ * This class defines a license-generation for URLs.
+ * It is used in case of preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet,
+ * but to prevent them to use this servlet as a proxy.
+ */
public class URLLicense {
- // this class defines a license-generation for URLs
- // it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet
+
private static final int maxQueue = 10000;
+
+ /** Map URLs by licence keys */
private static final Map permissions = Collections.synchronizedMap(new SizeLimitedMap(maxQueue));
-
+
+ /**
+ * Generates and stores a unique licence key for delayed url data fetching.
+ * @param url URL for whose data should be fectched later
+ * @return licence key generated or null when url is null
+ */
public static String aquireLicense(final DigestURL url) {
- if (url == null) return "";
- // generate license key
- String license = ASCII.String(url.hash());
- // store reference to url with license key
- permissions.put(license, url.toNormalform(true));
- // return the license key
- return license;
- }
+ if (url == null) return null;
+ /* Generate license key : it must absolutely be a unique key, not related to url parameter (thus url.hash can not be used).
+ * If the same key is generated for each call of this method with the same url parameter,
+ * problem may occur concurrent non authorized users try to fetch same url content.
+ * Example scenario (emulated in URLLicenseConcurrentTest) :
+ * 1 - userA aquireLicence for url
+ * 2 - userB aquireLicence for same url as A
+ * 3 - userA releaseLicense : he can now fetch url content
+ * 4 - userB releaseLicense : if the same license was generated, it has been already released and url content can not be fetched! */
+ String license = UUID.randomUUID().toString();
- public static String aquireLicense(final String license, final String url) {
- // store reference to url with license key
- permissions.put(license, url);
+ permissions.put(license, url.toNormalform(true));
+
// return the license key
return license;
}
+ /**
+ * Use it to retrieve source url and to ensures YaCy url containing this licence code can not be reused by non-authorized users.
+ * @param license unique code associated to source url
+ * @return source url or null licence is no more valid
+ * @throws NullPointerException when license is null
+ */
public static String releaseLicense(final String license) {
return permissions.remove(license);
}
diff --git a/test/net/yacy/data/URLLicenseConcurrentTest.java b/test/net/yacy/data/URLLicenseConcurrentTest.java
new file mode 100644
index 000000000..78fbac0b6
--- /dev/null
+++ b/test/net/yacy/data/URLLicenseConcurrentTest.java
@@ -0,0 +1,101 @@
+// URLLicense.java
+// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 03.07.2007 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate$
+// $LastChangedRevision$
+// $LastChangedBy$
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.data;
+
+import java.net.MalformedURLException;
+
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.util.ConcurrentLog;
+
+/**
+ * Test URLLicence reliability when used by concurrent threads
+ *
+ * @author luc
+ *
+ */
+public class URLLicenseConcurrentTest {
+
+ /**
+ * Thread emulating a client who tries to fetch some url content.
+ * @author luc
+ *
+ */
+ private static class ClientThread extends Thread {
+
+ private String testURL = "http://yacy.net";
+
+ private int steps = 100000;
+
+ @Override
+ public void run() {
+ System.out.println(this.getName() + " started...");
+ DigestURL url = null;
+ try {
+ url = new DigestURL(this.testURL);
+ } catch (MalformedURLException e1) {
+ e1.printStackTrace();
+ }
+ String normalizedURL = url.toNormalform(true);
+ for (int step = 0; step < this.steps; step++) {
+ String license = URLLicense.aquireLicense(url);
+ // You can eventually call here Thread.sleep()
+ String retrievedURL = URLLicense.releaseLicense(license);
+ if (!normalizedURL.equals(retrievedURL)) {
+ System.err.println("Licence lost! license : " + license + ", step : " + step + ", Thread : " + this.getName());
+ }
+ }
+ System.out.println(this.getName() + " finished!");
+ }
+ }
+
+ /**
+ * Runs clients concurrently : until the end, no error message should be displayed in console.
+ * @param args
+ */
+ public static void main(String args[]) {
+ long beginTime = System.nanoTime();
+ try {
+ ClientThread[] threads = new ClientThread[10];
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new URLLicenseConcurrentTest.ClientThread();
+ threads[i].setName("ClientThread" + i);
+ threads[i].start();
+ }
+ for (int i = 0; i < threads.length; i++) {
+ try {
+ threads[i].join();
+ } catch (InterruptedException e) {
+ }
+ }
+ } finally {
+ long time = System.nanoTime() - beginTime;
+ System.out.println("Test run in " + time / 1000000 + "ms");
+ ConcurrentLog.shutdown();
+ }
+ }
+
+}