Improved URLLicence reliability for use by conccurrent non authaurized

users.
Removed URLLicence generation when unnecessary (authorized users)
pull/39/head
luc 9 years ago
parent e3d53f0248
commit cfdbc2b487

@ -1,7 +1,7 @@
#(content)#:: #(content)#::
<div class="searchresults"> <div class="searchresults">
<h4 class="linktitle"> <h4 class="linktitle">
<img width="16" height="16" src="ViewImage.png?width=16&amp;height=16&amp;code=#[faviconCode]#&amp;isStatic=true" id="f#[urlhash]#" class="favicon" style="width:16px; height:16px;" alt="" /> <img width="16" height="16" src="#[faviconUrl]#" id="f#[urlhash]#" class="favicon" style="width:16px; height:16px;" alt="" />
<a href="#[link]#" target="#[target]#">#[title]#</a></h4> <a href="#[link]#" target="#[target]#">#[title]#</a></h4>
<div class="urlactions"> <div class="urlactions">
#(heuristic)#:: #(heuristic)#::

@ -194,8 +194,7 @@ public class yacysearchitem {
boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom"); boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");
String resultFileName = resultURL.getFileName(); String resultFileName = resultURL.getFileName();
prop.putHTML("content_target", target); prop.putHTML("content_target", target);
//if (faviconURL != null && fileType == FileType.HTML) sb.loader.loadIfNotExistBackground(faviconURL, 1024 * 1024 * 10, null, ClientIdentification.yacyIntranetCrawlerAgent); prop.putHTML("content_faviconUrl", processFaviconURL(authenticated, faviconURL));
prop.putHTML("content_faviconCode", URLLicense.aquireLicense(faviconURL)); // acquire license for favicon url loading
prop.put("content_urlhash", urlhash); prop.put("content_urlhash", urlhash);
prop.put("content_ranking", Float.toString(result.score())); prop.put("content_ranking", Float.toString(result.score()));
Date[] events = result.events(); Date[] events = result.events();
@ -310,7 +309,7 @@ public class yacysearchitem {
if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) { if (theSearch.query.contentdom == Classification.ContentDomain.IMAGE) {
// image search; shows thumbnails // image search; shows thumbnails
processImage(sb, prop, item, theSearch, target_special_pattern, timeout); processImage(sb, prop, item, theSearch, target_special_pattern, timeout, authenticated);
theSearch.query.transmitcount = item + 1; theSearch.query.transmitcount = item + 1;
return prop; return prop;
} }
@ -343,6 +342,28 @@ public class yacysearchitem {
return prop; return prop;
} }
/**
* @param authenticated
* true when current user is authenticated
* @param faviconURL
* url icon of web site
* @return url to propose in search result or empty string when faviconURL
* is null
*/
private static String processFaviconURL(final boolean authenticated, DigestURL faviconURL) {
/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
StringBuilder contentFaviconURL = new StringBuilder();
if (faviconURL != null) {
contentFaviconURL.append("ViewImage.png?width=16&height=16&isStatic=true");
if (authenticated) {
contentFaviconURL.append("&url=").append(faviconURL.toNormalform(true));
} else {
contentFaviconURL.append("&code=").append(URLLicense.aquireLicense(faviconURL));
}
}
return contentFaviconURL.toString();
}
/** /**
* Process search of image type and feed prop object. All parameters must not be null. * Process search of image type and feed prop object. All parameters must not be null.
@ -352,9 +373,10 @@ public class yacysearchitem {
* @param theSearch search event * @param theSearch search event
* @param target_special_pattern * @param target_special_pattern
* @param timeout result getting timeOut * @param timeout result getting timeOut
* @param authenticated set to true when user authentication is ok
*/ */
private static void processImage(final Switchboard sb, final serverObjects prop, final int item, private static void processImage(final Switchboard sb, final serverObjects prop, final int item,
final SearchEvent theSearch, final String target_special_pattern, long timeout) { final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean authenticated) {
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
try { try {
SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout); SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout);
@ -366,9 +388,24 @@ public class yacysearchitem {
/* Image format ouput for ViewImage servlet : default is png, except with gif and svg images */ /* Image format ouput for ViewImage servlet : default is png, except with gif and svg images */
final String viewImageExt = !imageUrlExt.isEmpty() && ViewImage.isBrowserRendered(imageUrlExt) ? imageUrlExt : "png"; final String viewImageExt = !imageUrlExt.isEmpty() && ViewImage.isBrowserRendered(imageUrlExt) ? imageUrlExt : "png";
/* Thumb URL */ /* Thumb URL */
prop.putHTML("content_item_hrefCache", "ViewImage." + viewImageExt + "?maxwidth=" + DEFAULT_IMG_WIDTH + "&maxheight=" + DEFAULT_IMG_HEIGHT + "&code="+license+"&isStatic=true&quadratic=&url=" + imageUrlstring); StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=")
.append(DEFAULT_IMG_WIDTH).append("&maxheight=").append(DEFAULT_IMG_HEIGHT)
.append("&isStatic=true&quadratic");
/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
if(authenticated) {
thumbURLBuilder.append("&url=").append(imageUrlstring);
} else {
thumbURLBuilder.append("&code=").append(URLLicense.aquireLicense(image.imageUrl));
}
String thumbURL = thumbURLBuilder.toString();
prop.putHTML("content_item_hrefCache", thumbURL);
/* Full size preview URL */ /* Full size preview URL */
prop.putHTML("content_item_hrefFullPreview", "ViewImage." + viewImageExt + "?code="+license+"&isStatic=true&url=" + imageUrlstring); if(authenticated) {
prop.putHTML("content_item_hrefFullPreview", "ViewImage." + viewImageExt + "?isStatic=true&url=" + imageUrlstring);
} else {
/* Not authenticated : full preview URL must be the same as thumb URL */
prop.putHTML("content_item_hrefFullPreview", thumbURL);
}
prop.putHTML("content_item_href", imageUrlstring); prop.putHTML("content_item_href", imageUrlstring);
prop.putHTML("content_item_target", target); prop.putHTML("content_item_target", target);
prop.put("content_item_code", license); prop.put("content_item_code", license);

@ -30,14 +30,6 @@ import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.data.URLLicense;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
@ -50,6 +42,13 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList; import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
/** /**
* write the opensearch result in YaCys special way to include as much as in opensearch is included. * write the opensearch result in YaCys special way to include as much as in opensearch is included.
@ -159,8 +158,6 @@ public class YJsonResponseWriter implements QueryResponseWriter {
String filename = url.getFileName(); String filename = url.getFileName();
solitaireTag(writer, "link", u); solitaireTag(writer, "link", u);
solitaireTag(writer, "file", filename); solitaireTag(writer, "file", filename);
// get image license
if (Classification.isImageExtension(MultiProtocolURL.getFileExtension(filename))) URLLicense.aquireLicense(urlhash, url.toNormalform(true));
} catch (final MalformedURLException e) {} } catch (final MalformedURLException e) {}
continue; continue;
} }
@ -221,7 +218,9 @@ public class YJsonResponseWriter implements QueryResponseWriter {
if (i < responseCount - 1) { if (i < responseCount - 1) {
writer.write(",\n".toCharArray()); writer.write(",\n".toCharArray());
} }
} catch (final Throwable ee) {} } catch (final Throwable ee) {
ConcurrentLog.fine("YJsonResponseWriter", "Document writing error : " + ee.getMessage());
}
} }
writer.write("],\n".toCharArray()); writer.write("],\n".toCharArray());

@ -28,36 +28,53 @@ package net.yacy.data;
import java.util.Collections; import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.UUID;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.storage.SizeLimitedMap; import net.yacy.cora.storage.SizeLimitedMap;
/**
* This class defines a license-generation for URLs.
* It is used in case of preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet,
* but to prevent them to use this servlet as a proxy.
*/
public class URLLicense { public class URLLicense {
// this class defines a license-generation for URLs
// it is used in case of snippet- and preview-Image-fetching to grant also non-authorized users the usage of a image-fetcher servlet
private static final int maxQueue = 10000; private static final int maxQueue = 10000;
/** Map URLs by licence keys */
private static final Map<String, String> permissions = Collections.synchronizedMap(new SizeLimitedMap<String, String>(maxQueue)); private static final Map<String, String> permissions = Collections.synchronizedMap(new SizeLimitedMap<String, String>(maxQueue));
/**
* Generates and stores a unique licence key for delayed url data fetching.
* @param url URL for whose data should be fectched later
* @return licence key generated or null when url is null
*/
public static String aquireLicense(final DigestURL url) { public static String aquireLicense(final DigestURL url) {
if (url == null) return ""; if (url == null) return null;
// generate license key /* Generate license key : it must absolutely be a unique key, not related to url parameter (thus url.hash can not be used).
String license = ASCII.String(url.hash()); * If the same key is generated for each call of this method with the same url parameter,
// store reference to url with license key * problem may occur concurrent non authorized users try to fetch same url content.
* Example scenario (emulated in URLLicenseConcurrentTest) :
* 1 - userA aquireLicence for url
* 2 - userB aquireLicence for same url as A
* 3 - userA releaseLicense : he can now fetch url content
* 4 - userB releaseLicense : if the same license was generated, it has been already released and url content can not be fetched! */
String license = UUID.randomUUID().toString();
permissions.put(license, url.toNormalform(true)); permissions.put(license, url.toNormalform(true));
// return the license key
return license;
}
public static String aquireLicense(final String license, final String url) {
// store reference to url with license key
permissions.put(license, url);
// return the license key // return the license key
return license; return license;
} }
/**
* Use it to retrieve source url and to ensures YaCy url containing this licence code can not be reused by non-authorized users.
* @param license unique code associated to source url
* @return source url or null licence is no more valid
* @throws NullPointerException when license is null
*/
public static String releaseLicense(final String license) { public static String releaseLicense(final String license) {
return permissions.remove(license); return permissions.remove(license);
} }

@ -0,0 +1,101 @@
// URLLicense.java
// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 03.07.2007 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.data;
import java.net.MalformedURLException;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.ConcurrentLog;
/**
* Test URLLicence reliability when used by concurrent threads
*
* @author luc
*
*/
public class URLLicenseConcurrentTest {
/**
* Thread emulating a client who tries to fetch some url content.
* @author luc
*
*/
private static class ClientThread extends Thread {
private String testURL = "http://yacy.net";
private int steps = 100000;
@Override
public void run() {
System.out.println(this.getName() + " started...");
DigestURL url = null;
try {
url = new DigestURL(this.testURL);
} catch (MalformedURLException e1) {
e1.printStackTrace();
}
String normalizedURL = url.toNormalform(true);
for (int step = 0; step < this.steps; step++) {
String license = URLLicense.aquireLicense(url);
// You can eventually call here Thread.sleep()
String retrievedURL = URLLicense.releaseLicense(license);
if (!normalizedURL.equals(retrievedURL)) {
System.err.println("Licence lost! license : " + license + ", step : " + step + ", Thread : " + this.getName());
}
}
System.out.println(this.getName() + " finished!");
}
}
/**
* Runs clients concurrently : until the end, no error message should be displayed in console.
* @param args
*/
public static void main(String args[]) {
long beginTime = System.nanoTime();
try {
ClientThread[] threads = new ClientThread[10];
for (int i = 0; i < threads.length; i++) {
threads[i] = new URLLicenseConcurrentTest.ClientThread();
threads[i].setName("ClientThread" + i);
threads[i].start();
}
for (int i = 0; i < threads.length; i++) {
try {
threads[i].join();
} catch (InterruptedException e) {
}
}
} finally {
long time = System.nanoTime() - beginTime;
System.out.println("Test run in " + time / 1000000 + "ms");
ConcurrentLog.shutdown();
}
}
}
Loading…
Cancel
Save