boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added Collage servlet from datengrab and modified it:

Browse Source 
* all images are queued
* private/public is respected
* inserted into switchboard
* added collageQueue class that stores all the queued images

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4683 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 959f448e5f
commit 82bf9ac1c8
5 changed files with 226 additions and 3 deletions
Show Stats Download Patch File Download Diff File

18
htroot/Collage.html
Unescape View File

@ -0,0 +1,18 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>YaCy '#[clientname]#': Collage</title>
#%env/templates/metas.template%#
<meta http-equiv="REFRESH" content="5" />
</head>
<body style="margin:0px;">
#%env/templates/header.template%#
#(imgurl)#
::
#{list}#
#[url]#
#{/list}#
#(/imgurl)#
#%env/templates/footer.template%#
</body>
</html>

110
htroot/Collage.java
Unescape View File

@ -0,0 +1,110 @@
// Collage.java
// -----------------------
// part of YaCy
// (C) by Detlef Reichl; detlef!reichl()gmx!org
// Pforzheim, Germany, 2008
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
// Any changes to this file according to the GPL as documented in the file
// gpl.txt aside this file in the shipment you received can be done to the
// lines that follows this copyright notice here, but changes must not be
// done inside the copyright notive above. A re-distribution must contain
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
import java.util.Random;
import de.anomic.data.collageQueue;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
public class Collage {
private static final int fifoMax = 20;
private static final int posXMax = 800;
private static final int posYMax = 500;
private static int fifoPos = -1;
private static int fifoSize = 0;
private static long zIndex = 0;
private static collageQueue.ImageOriginEntry origins[] = new collageQueue.ImageOriginEntry[fifoMax];
private static Integer imgWidth[] = new Integer[fifoMax];
private static Integer imgHeight[] = new Integer[fifoMax];
private static Integer imgPosX[] = new Integer[fifoMax];
private static Integer imgPosY[] = new Integer[fifoMax];
private static long imgZIndex[] = new long[fifoMax];
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
final serverObjects prop = new serverObjects();
final plasmaSwitchboard sb = (plasmaSwitchboard) env;
Random rand = new Random();
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
collageQueue.ImageOriginEntry nextOrigin = collageQueue.next(!authenticated);
if (nextOrigin != null) {
if (fifoSize == 0 || origins[fifoPos] != nextOrigin) {
fifoPos = fifoPos + 1 == fifoMax ? 0 : fifoPos + 1;
fifoSize = fifoSize + 1 > fifoMax ? fifoMax : fifoSize + 1;
origins[fifoPos] = nextOrigin;
float scale = rand.nextFloat() * 1.5f + 1;
imgWidth[fifoPos] = (int) (((float)nextOrigin.imageEntry.width()) / scale);
imgHeight[fifoPos] = (int) (((float)nextOrigin.imageEntry.height()) / scale);
imgPosX[fifoPos] = rand.nextInt(posXMax);
imgPosY[fifoPos] = rand.nextInt(posYMax);
imgZIndex[fifoPos] = zIndex;
zIndex += 1;
}
}
if (fifoSize > 0) {
prop.put("imgurl", "1");
for (int i = 0; i < fifoSize; i++)
prop.put("imgurl_list_" + i + "_url",
"<a href=\"" + origins[i].baseURL.toNormalform(true, false) + "\">"
+ "<img src=\"" + origins[i].imageEntry.url().toNormalform(true, false) + "\" style=\"width:" + imgWidth[i]
+ "px;height:" + imgHeight[i]
+ "px;position:absolute;top:" + imgPosY[i]
+ "px;left:" + imgPosX[i]
+ "px;z-index:" + imgZIndex[i] + "\" title=\"" + origins[i].baseURL.toNormalform(true, false) + "\">"
+ "</a><br>");
prop.put("imgurl_list", fifoSize);
} else {
prop.put("imgurl", "0");
}
return prop;
}
}

93
source/de/anomic/data/collageQueue.java
Unescape View File

@ -0,0 +1,93 @@
// collageQueue.java
// (C) 2008 by by Detlef Reichl; detlef!reichl()gmx!org and Michael Peter Christen; mc@yacy.net
// first published 13.04.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.data;
import java.util.HashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import de.anomic.htmlFilter.htmlFilterImageEntry;
import de.anomic.plasma.plasmaParserDocument;
import de.anomic.yacy.yacyURL;
public class collageQueue {
private static final ConcurrentLinkedQueue<ImageOriginEntry> privateImageQueue = new ConcurrentLinkedQueue<ImageOriginEntry>();
private static final ConcurrentLinkedQueue<ImageOriginEntry> publicImageQueue = new ConcurrentLinkedQueue<ImageOriginEntry>();
public static void registerImages(plasmaParserDocument document, boolean privateEntry) {
if (document == null) return;
if (document.dc_source() == null) return;
HashMap<String, htmlFilterImageEntry> images = document.getImages();
for (htmlFilterImageEntry image: images.values()) {
String name = image.url().getFile();
if (image.width() > 120 &&
image.height() > 100 &&
image.width() < 1200 &&
image.height() < 1000 &&
name.lastIndexOf(".gif") == -1) {
// && ((urlString.lastIndexOf(".jpg") != -1)) ||
// ((urlString.lastIndexOf(".png") != -1)){
float ratio;
if (image.width() > image.height()) {
ratio = (float) image.width() / (float) image.height();
} else {
ratio = (float) image.height() / (float) image.width();
}
if (ratio >= 1.0f && ratio <= 2.0f) {
if (privateEntry) {
privateImageQueue.add(new ImageOriginEntry(image, document.dc_source()));
} else {
publicImageQueue.add(new ImageOriginEntry(image, document.dc_source()));
}
}
}
}
}
public static ImageOriginEntry next(boolean privateEntryOnly) {
ImageOriginEntry e = null;
if (privateEntryOnly) {
e = privateImageQueue.poll();
} else {
e = publicImageQueue.poll();
if (e == null) e = privateImageQueue.poll();
}
return e;
}
public static class ImageOriginEntry {
public htmlFilterImageEntry imageEntry;
public yacyURL baseURL;
public ImageOriginEntry(htmlFilterImageEntry imageEntry, yacyURL baseURL) {
this.imageEntry = imageEntry;
this.baseURL = baseURL;
}
}
}

2
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -89,7 +89,6 @@ import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.logging.FileHandler;
import java.util.logging.Level;
@ -97,7 +96,6 @@ import java.util.logging.LogManager;
import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream;
import de.anomic.data.htmlTools;
import de.anomic.htmlFilter.htmlFilterContentTransformer;
import de.anomic.htmlFilter.htmlFilterTransformer;
import de.anomic.htmlFilter.htmlFilterWriter;

4
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -112,6 +112,7 @@ import de.anomic.data.URLLicense;
import de.anomic.data.blogBoard;
import de.anomic.data.blogBoardComments;
import de.anomic.data.bookmarksDB;
import de.anomic.data.collageQueue;
import de.anomic.data.listManager;
import de.anomic.data.messageBoard;
import de.anomic.data.userDB;
@ -2126,6 +2127,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo
}
in.queueEntry.close();
return null;
} else {
plasmaCrawlProfile.entry profile = profilesActiveCrawls.getEntry(in.queueEntry.profileHandle);
collageQueue.registerImages(document, (profile == null) ? true : profile.remoteIndexing());
}
return new indexingQueueEntry(in.queueEntry, document, null);
}

Write Preview
Loading…
Cancel
Save