boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added json and rss output for image search

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6503 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 016fdfa310
commit 969123385b
14 changed files with 167 additions and 50 deletions
Show Stats Download Patch File Download Diff File

3
htroot/yacysearch.rss
Unescape View File

@ -3,7 +3,8 @@
<rss version="2.0"
xmlns:yacy="http://www.yacy.net/"
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:atom="http://www.w3.org/2005/Atom">
xmlns:media="http://search.yahoo.com/mrss/"
xmlns:atom="http://www.w3.org/2005/Atom">
<!-- YaCy Search Engine; http://yacy.net -->
<channel>
<title>#[promoteSearchPageGreeting]#</title>

6
htroot/yacysearchitem.html
Unescape View File

@ -20,8 +20,7 @@
<p class="urlinfo">#[date]# | #[sizename]# | YBR-#[ybr]# | <a href="ViewFile.html?urlHash=#[urlhash]#&amp;words=#[words]#&amp;display=#[display]#">Info</a> | <a href="yacysearch.html?cat=image&amp;url=#[link]#&amp;search=#[former]#&amp;display=#[display]#">Pictures</a></p>
</div>
::
#{items}#
<div class="thumbcontainer">
#(item)#::<div class="thumbcontainer">
<a href="#[hrefCache]#" class="thumblink" onclick="return hs.expand(this)">
<img src="/ViewImage.png?maxwidth=96&amp;maxheight=96&amp;code=#[code]#" alt="#[name]#" />
</a>
@ -29,8 +28,7 @@
<!--
<div class="TableCellDark"><a href="#[href]#">#[name]#</a><br \>#[attr]#</div>
-->
</div>
#{/items}#
</div>#(/item)#
::
#{items}#
<tr class="#(col)#TableCellLight::TableCellDark#(/col)#"><td>#[name]#</td><td><a href="#[href]#">#[hrefshort]#</a></tr>

24
htroot/yacysearchitem.java
Unescape View File

@ -163,16 +163,22 @@ public class yacysearchitem {
prop.put("content", theQuery.contentdom.getCode() + 1); // switch on specific content
final MediaSnippet ms = theSearch.result().oneImage(item);
if (ms == null) {
prop.put("content_items", "0");
prop.put("content_item", "0");
} else {
prop.putHTML("content_items_0_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false));
prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false));
prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href));
prop.putHTML("content_items_0_name", shorten(ms.name, namelength));
prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
prop.put("content_items_0_source", ms.source.toNormalform(true, false));
prop.put("content_items_0_sourcedom", ms.source.getHost());
prop.put("content_items", 1);
prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false));
prop.putHTML("content_item_href", ms.href.toNormalform(true, false));
prop.put("content_item_code", sb.licensedURLs.aquireLicense(ms.href));
prop.putHTML("content_item_name", shorten(ms.name, namelength));
prop.put("content_item_mime", ms.mime);
prop.put("content_item_fileSize", ms.fileSize);
prop.put("content_item_width", ms.width);
prop.put("content_item_height", ms.height);
prop.put("content_item_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
prop.put("content_item_urlhash", ms.source.hash());
prop.put("content_item_source", ms.source.toNormalform(true, false));
prop.put("content_item_sourcedom", ms.source.getHost());
prop.put("content_item_nl", (item == 0) ? 0 : 1);
prop.put("content_item", 1);
}
return prop;
}

11
htroot/yacysearchitem.json
Unescape View File

@ -11,15 +11,18 @@
"host": "#[host]#",
"path": "#[path]#",
"file": "#[file]#"
}::#{items}##(nl)#:: ,#(/nl)#
}::#(item)#::#(nl)#:: ,#(/nl)#
{
"title": "#[name]#",
"icon": "/ViewImage.png?maxwidth=96&amp;maxheight=96&amp;code=#[code]#",
"image": "#[href]#",
"cache": "#[hrefCache]#",
"link": "#[source]#",
"host": "#[sourcedom]#"
}#{/items}#::
"url": "#[source]#",
"urlhash": "#[urlhash]#",
"host": "#[sourcedom]#",
"width": "#[width]#",
"height": "#[height]#"
}#(/item)#::
::
::
::

39
htroot/yacysearchitem.xml
Unescape View File

@ -9,4 +9,41 @@
<yacy:path>#[path]#</yacy:path>
<yacy:file>#[file]#</yacy:file>
<guid isPermaLink="false">#[urlhash]#</guid>
</item>#(/content)#
</item>::#(item)#::<item>
<title>#[name]#</title>
<link>#[source]#</link>
<description></description>
<pubDate></pubDate>
<guid isPermaLink="false">#[urlhash]#</guid>
<yacy:host>#[sourcedom]#</yacy:host>
<media:group>
<media:content
url="#[href]#"
fileSize="#[fileSize]#"
type="#[mime]#"
medium="image"
isDefault="true"
expression="full"
height="#[width]#"
width="#[height]#" />
<media:content
url="#[hrefCache]#"
fileSize="#[fileSize]#"
type="#[mime]#"
medium="image"
isDefault="false"
expression="full"
height="#[width]#"
width="#[height]#" />
<media:content
url="/ViewImage.png?maxwidth=96&amp;maxheight=96&amp;code=#[code]#"
fileSize="#[fileSize]#"
type="#[mime]#"
medium="image"
isDefault="false"
expression="sample"
height="96"
width="96" />
</media:group>
</item>#(/item)#::
#(/content)#

48
source/de/anomic/data/MimeTable.java
Unescape View File

@ -0,0 +1,48 @@
package de.anomic.data;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.util.Properties;
import net.yacy.kelondro.data.meta.DigestURI;
public class MimeTable {
private static final Properties mimeTable = new Properties();
public static void init(File mimeFile) {
if (mimeTable.size() == 0) {
// load the mime table
BufferedInputStream mimeTableInputStream = null;
try {
mimeTableInputStream = new BufferedInputStream(new FileInputStream(mimeFile));
mimeTable.load(mimeTableInputStream);
} catch (final Exception e) {
e.printStackTrace();
} finally {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (final Exception e1) {}
}
}
}
public static int size() {
return mimeTable.size();
}
public static String ext2mime(String ext) {
return mimeTable.getProperty(ext, "application/" + ext);
}
public static String ext2mime(String ext, String dfltMime) {
return mimeTable.getProperty(ext, dfltMime);
}
public static String url2mime(DigestURI url, String dfltMime) {
return ext2mime(url.getFileExtension(), dfltMime);
}
public static String url2mime(DigestURI url) {
return ext2mime(url.getFileExtension());
}
}

23
source/de/anomic/http/server/HTTPDFileHandler.java
Unescape View File

@ -92,6 +92,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.visualization.RasterPlotter;
import de.anomic.data.MimeTable;
import de.anomic.http.server.servlets.crawlReceipt;
import de.anomic.http.server.servlets.transferURL;
import de.anomic.search.Switchboard;
@ -107,7 +108,6 @@ public final class HTTPDFileHandler {
private static final boolean safeServletsMode = false; // if true then all servlets are called synchronized
private static final Properties mimeTable = new Properties();
// create a class loader
private static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
private static serverSwitch switchboard = null;
@ -142,20 +142,11 @@ public final class HTTPDFileHandler {
if (switchboard == null) {
switchboard = theSwitchboard;
if (mimeTable.size() == 0) {
if (MimeTable.size() == 0) {
// load the mime table
final String mimeTablePath = theSwitchboard.getConfig("mimeTable","");
BufferedInputStream mimeTableInputStream = null;
try {
Log.logConfig("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);
mimeTableInputStream = new BufferedInputStream(new FileInputStream(new File(theSwitchboard.getRootPath(), mimeTablePath)));
mimeTable.load(mimeTableInputStream);
} catch (final Exception e) {
Log.logSevere("HTTPDFiles", "ERROR: path to configuration file or configuration invalid\n" + e);
System.exit(1);
} finally {
if (mimeTableInputStream != null) try { mimeTableInputStream.close(); } catch (final Exception e1) {}
}
Log.logConfig("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);
MimeTable.init(new File(theSwitchboard.getRootPath(), mimeTablePath));
}
// create default files array
@ -537,7 +528,7 @@ public final class HTTPDFileHandler {
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = mimeTable.getProperty(targetExt, "text/html");
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), targetExt);
// write the array to the client
@ -551,7 +542,7 @@ public final class HTTPDFileHandler {
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = mimeTable.getProperty(targetExt, "text/html");
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
// generate an byte array from the generated image
int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
@ -721,7 +712,7 @@ public final class HTTPDFileHandler {
// we have found a file that can be written to the client
// if this file uses templates, then we use the template
// re-write - method to create an result
String mimeType = mimeTable.getProperty(targetExt,"text/html");
String mimeType = MimeTable.ext2mime(targetExt, "text/html");
final boolean zipContent = requestHeader.acceptGzip() && HTTPDemon.shallTransportZipped("." + conProp.getProperty("EXT",""));
if (path.endsWith("html") ||
path.endsWith("htm") ||

39
source/de/anomic/search/MediaSnippet.java
Unescape View File

@ -30,6 +30,8 @@ import java.util.Iterator;
import java.util.Map;
import java.util.TreeSet;
import de.anomic.data.MimeTable;
import net.yacy.document.Document;
import net.yacy.document.parser.html.ImageEntry;
import net.yacy.kelondro.data.meta.DigestURI;
@ -41,15 +43,40 @@ import net.yacy.repository.LoaderDispatcher;
public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaSnippet> {
public ContentDomain type;
public DigestURI href, source;
public String name, attr;
public String name, attr, mime;
public int ranking;
public int width, height;
public long fileSize;
public MediaSnippet(final ContentDomain type, final DigestURI href, final String name, final String attr, final int ranking, final DigestURI source) {
public MediaSnippet(final ContentDomain type, final DigestURI href, final String mime, final String name, final long fileSize, final String attr, final int ranking, final DigestURI source) {
this.type = type;
this.href = href;
this.mime = mime;
this.fileSize = fileSize;
this.source = source; // the web page where the media resource appeared
this.name = name;
this.attr = attr;
this.width = -1;
this.height = -1;
int p = 0;
if ((p = attr.indexOf(" x ")) > 0) {
this.width = Integer.parseInt(attr.substring(0, p).trim());
this.height = Integer.parseInt(attr.substring(p + 3).trim());
}
this.ranking = ranking; // the smaller the better! small values should be shown first
if ((this.name == null) || (this.name.length() == 0)) this.name = "_";
if ((this.attr == null) || (this.attr.length() == 0)) this.attr = "_";
}
public MediaSnippet(final ContentDomain type, final DigestURI href, final String mime, final String name, final long fileSize, final int width, final int height, final int ranking, final DigestURI source) {
this.type = type;
this.href = href;
this.fileSize = fileSize;
this.source = source; // the web page where the media resource appeared
this.name = name;
this.attr = width + " x " + height;
this.width = width;
this.height = height;
this.ranking = ranking; // the smaller the better! small values should be shown first
if ((this.name == null) || (this.name.length() == 0)) this.name = "_";
if ((this.attr == null) || (this.attr.length() == 0)) this.attr = "_";
@ -115,10 +142,10 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
entry = i.next();
url = entry.getKey();
desc = entry.getValue();
int ranking = TextSnippet.removeAppearanceHashes(url.toNormalform(false, false), queryhashes).size() +
int ranking = TextSnippet.removeAppearanceHashes(url.toNormalform(false, false), queryhashes).size() +
TextSnippet.removeAppearanceHashes(desc, queryhashes).size();
if (ranking < 2 * queryhashes.size()) {
result.add(new MediaSnippet(mediatype, url, desc, null, ranking, document.dc_source()));
result.add(new MediaSnippet(mediatype, url, MimeTable.url2mime(url), desc, document.getTextLength(), null, ranking, document.dc_source()));
}
}
return result;
@ -146,8 +173,8 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
int appcount = queryhashes.size() * 2 -
TextSnippet.removeAppearanceHashes(url.toNormalform(false, false), queryhashes).size() -
TextSnippet.removeAppearanceHashes(desc, queryhashes).size();
final int ranking = Integer.MAX_VALUE - (ientry.height() + 1) * (ientry.width() + 1) * (appcount + 1);
result.add(new MediaSnippet(ContentDomain.IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking, document.dc_source()));
final int ranking = Integer.MAX_VALUE - (ientry.height() + 1) * (ientry.width() + 1) * (appcount + 1);
result.add(new MediaSnippet(ContentDomain.IMAGE, url, MimeTable.url2mime(url), desc, ientry.fileSize(), ientry.width(), ientry.height(), ranking, document.dc_source()));
}
return result;
}

2
source/net/yacy/document/Document.java
Unescape View File

@ -386,7 +386,7 @@ dc_rights
if (Classification.isMediaExtension(ext)) {
// this is not a normal anchor, its a media link
if (Classification.isImageExtension(ext)) {
ContentScraper.addImage(collectedImages, new ImageEntry(url, entry.getValue(), -1, -1));
ContentScraper.addImage(collectedImages, new ImageEntry(url, entry.getValue(), -1, -1, -1));
}
else if (Classification.isAudioExtension(ext)) audiolinks.put(url, entry.getValue());
else if (Classification.isVideoExtension(ext)) videolinks.put(url, entry.getValue());

6
source/net/yacy/document/parser/html/ContentScraper.java
Unescape View File

@ -148,7 +148,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
final float ratio = (float) Math.min(width, height) / Math.max(width, height);
if (ratio > 0.4) {
final DigestURI url = absolutePath(tagopts.getProperty("src", ""));
final ImageEntry ie = new ImageEntry(url, tagopts.getProperty("alt", ""), width, height);
final ImageEntry ie = new ImageEntry(url, tagopts.getProperty("alt", ""), width, height, -1);
addImage(images, ie);
}
// i think that real pictures have witdth & height tags - thq
@ -190,7 +190,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
final String linktitle = tagopts.getProperty("title", "");
if (type.equalsIgnoreCase("shortcut icon")) {
final ImageEntry ie = new ImageEntry(newLink, linktitle, -1,-1);
final ImageEntry ie = new ImageEntry(newLink, linktitle, -1, -1, -1);
images.put(ie.url().hash(), ie);
this.favicon = newLink;
} else if (!type.equalsIgnoreCase("stylesheet") && !type.equalsIgnoreCase("alternate stylesheet")) {
@ -225,7 +225,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
final String type = (p < 0) ? "" : f.substring(p + 1);
if (type.equals("png") || type.equals("gif") || type.equals("jpg") || type.equals("jpeg")) {
// special handling of such urls: put them to the image urls
final ImageEntry ie = new ImageEntry(url, super.stripAll(new String(text)).trim(), -1, -1);
final ImageEntry ie = new ImageEntry(url, super.stripAll(new String(text)).trim(), -1, -1, -1);
addImage(images, ie);
} else {
anchors.put(url, super.stripAll(new String(text)).trim());

10
source/net/yacy/document/parser/html/ImageEntry.java
Unescape View File

@ -33,12 +33,14 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
private final DigestURI url;
private final String alt;
private final int width, height;
public ImageEntry(final DigestURI url, final String alt, final int width, final int height) {
private final long fileSize;
public ImageEntry(final DigestURI url, final String alt, final int width, final int height, long fileSize) {
this.url = url;
this.alt = alt;
this.width = width;
this.height = height;
this.fileSize = fileSize;
}
public DigestURI url() {
@ -56,6 +58,10 @@ public class ImageEntry implements Comparable<ImageEntry>, Comparator<ImageEntry
public int height() {
return this.height;
}
public long fileSize() {
return this.fileSize;
}
@Override
public String toString() {

2
source/net/yacy/document/parser/images/bmpParser.java
Unescape View File

@ -131,7 +131,7 @@ public class bmpParser extends AbstractParser implements Idiom {
final HashMap<DigestURI, String> anchors = new HashMap<DigestURI, String>();
final HashMap<String, ImageEntry> images = new HashMap<String, ImageEntry>();
// add this image to the map of images
images.put(sb.toString(), new ImageEntry(location, "", width, height));
images.put(sb.toString(), new ImageEntry(location, "", width, height, -1));
return new Document(
location,

2
source/net/yacy/document/parser/images/genericImageParser.java
Unescape View File

@ -110,7 +110,7 @@ public class genericImageParser extends AbstractParser implements Idiom {
final HashMap<DigestURI, String> anchors = new HashMap<DigestURI, String>();
final HashMap<String, ImageEntry> images = new HashMap<String, ImageEntry>();
// add this image to the map of images
images.put(sb.toString(), new ImageEntry(location, "", width, height));
images.put(sb.toString(), new ImageEntry(location, "", width, height, -1));
return new Document(
location,

2
source/net/yacy/document/parser/rssParser.java
Unescape View File

@ -105,7 +105,7 @@ public class rssParser extends AbstractParser implements Idiom {
if (feed.getImage() != null) {
final DigestURI imgURL = new DigestURI(feed.getImage(), null);
images.put(imgURL.hash(), new ImageEntry(imgURL, feedTitle, -1, -1));
images.put(imgURL.hash(), new ImageEntry(imgURL, feedTitle, -1, -1, -1));
}
// loop through the feed items

Write Preview
Loading…
Cancel
Save