- renamed also the plasmaCrawlResults to have a consistent naming for url and image queues - added a double-check for the images - added additional queues for the images: all worse-quality images go there, so the queue can be used also if no sizes are given; no image is lost - added a cleanup for the stacks so they cannot flood the memory git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4722 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
17 years ago
parent
d7e89c2aca
commit
1995faef8d
@ -1,101 +0,0 @@
|
||||
// collageQueue.java
|
||||
// (C) 2008 by by Detlef Reichl; detlef!reichl()gmx!org and Michael Peter Christen; mc@yacy.net
|
||||
// first published 13.04.2008 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.data;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
import de.anomic.htmlFilter.htmlFilterImageEntry;
|
||||
import de.anomic.plasma.plasmaParserDocument;
|
||||
import de.anomic.yacy.yacyURL;
|
||||
|
||||
public class collageQueue {
|
||||
|
||||
private static final ConcurrentLinkedQueue<ImageOriginEntry> privateImageQueue = new ConcurrentLinkedQueue<ImageOriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<ImageOriginEntry> publicImageQueue = new ConcurrentLinkedQueue<ImageOriginEntry>();
|
||||
|
||||
public static void registerImages(plasmaParserDocument document, boolean privateEntry) {
|
||||
if (document == null) return;
|
||||
if (document.dc_source() == null) return;
|
||||
|
||||
HashMap<String, htmlFilterImageEntry> images = document.getImages();
|
||||
for (htmlFilterImageEntry image: images.values()) {
|
||||
String name = image.url().getFile();
|
||||
|
||||
if (image.width() > 120 &&
|
||||
image.height() > 100 &&
|
||||
image.width() < 1200 &&
|
||||
image.height() < 1000 &&
|
||||
name.lastIndexOf(".gif") == -1) {
|
||||
// && ((urlString.lastIndexOf(".jpg") != -1)) ||
|
||||
// ((urlString.lastIndexOf(".png") != -1)){
|
||||
float ratio;
|
||||
if (image.width() > image.height()) {
|
||||
ratio = (float) image.width() / (float) image.height();
|
||||
} else {
|
||||
ratio = (float) image.height() / (float) image.width();
|
||||
}
|
||||
if (ratio >= 1.0f && ratio <= 2.0f) {
|
||||
if (privateEntry) {
|
||||
privateImageQueue.add(new ImageOriginEntry(image, document.dc_source()));
|
||||
} else {
|
||||
publicImageQueue.add(new ImageOriginEntry(image, document.dc_source()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static ImageOriginEntry next(boolean privateEntryOnly) {
|
||||
ImageOriginEntry e = null;
|
||||
if (privateEntryOnly) {
|
||||
e = privateImageQueue.poll();
|
||||
} else {
|
||||
e = publicImageQueue.poll();
|
||||
if (e == null) e = privateImageQueue.poll();
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
public static int privateQueueSize() {
|
||||
return privateImageQueue.size();
|
||||
}
|
||||
|
||||
public static int publicQueueSize() {
|
||||
return publicImageQueue.size();
|
||||
}
|
||||
|
||||
public static class ImageOriginEntry {
|
||||
public htmlFilterImageEntry imageEntry;
|
||||
public yacyURL baseURL;
|
||||
public ImageOriginEntry(htmlFilterImageEntry imageEntry, yacyURL baseURL) {
|
||||
this.imageEntry = imageEntry;
|
||||
this.baseURL = baseURL;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,152 @@
|
||||
// plasmaCrawlResultImages.java
|
||||
// (C) 2008 by by Detlef Reichl; detlef!reichl()gmx!org and Michael Peter Christen; mc@yacy.net
|
||||
// first published 13.04.2008 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||
// $LastChangedRevision: 1986 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.plasma;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
||||
import de.anomic.htmlFilter.htmlFilterImageEntry;
|
||||
import de.anomic.yacy.yacyURL;
|
||||
|
||||
public class plasmaCrawlResultImages {
|
||||
|
||||
// we maintain two different queues for private and public crawls and divide both into two halves:
|
||||
// such images that appear to be good quality for a image monitor bacause their size is known, and other images
|
||||
// that are not declared with sizes.
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> privateImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueHigh = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
private static final ConcurrentLinkedQueue<OriginEntry> publicImageQueueLow = new ConcurrentLinkedQueue<OriginEntry>();
|
||||
|
||||
// we also check all links for a double-check so we don't get the same image more than once in any queue
|
||||
// image links may appear double here even if the pages where the image links are embedded already are checked for double-occurrence:
|
||||
// the same images may be linked from different pages
|
||||
private static final ConcurrentHashMap<String, Long> doubleCheck = new ConcurrentHashMap<String, Long>(); // (url-hash, time) when the url appeared first
|
||||
|
||||
public static void registerImages(plasmaParserDocument document, boolean privateEntry) {
|
||||
if (document == null) return;
|
||||
if (document.dc_source() == null) return;
|
||||
|
||||
HashMap<String, htmlFilterImageEntry> images = document.getImages();
|
||||
for (htmlFilterImageEntry image: images.values()) {
|
||||
// do a double-check; attention: this can be time-consuming since this possibly needs a DNS-lookup
|
||||
if (doubleCheck.containsKey(image.url().hash())) continue;
|
||||
doubleCheck.put(image.url().hash(), System.currentTimeMillis());
|
||||
|
||||
String name = image.url().getFile();
|
||||
boolean good = false;
|
||||
if (image.width() > 120 &&
|
||||
image.height() > 100 &&
|
||||
image.width() < 1200 &&
|
||||
image.height() < 1000 &&
|
||||
name.lastIndexOf(".gif") == -1) {
|
||||
// && ((urlString.lastIndexOf(".jpg") != -1)) ||
|
||||
// ((urlString.lastIndexOf(".png") != -1)){
|
||||
|
||||
good = true;
|
||||
float ratio;
|
||||
if (image.width() > image.height()) {
|
||||
ratio = (float) image.width() / (float) image.height();
|
||||
} else {
|
||||
ratio = (float) image.height() / (float) image.width();
|
||||
}
|
||||
if (ratio < 1.0f || ratio > 2.0f) good = false;
|
||||
}
|
||||
if (good) {
|
||||
if (privateEntry) {
|
||||
privateImageQueueHigh.add(new OriginEntry(image, document.dc_source()));
|
||||
} else {
|
||||
publicImageQueueHigh.add(new OriginEntry(image, document.dc_source()));
|
||||
}
|
||||
} else {
|
||||
if (privateEntry) {
|
||||
privateImageQueueLow.add(new OriginEntry(image, document.dc_source()));
|
||||
} else {
|
||||
publicImageQueueLow.add(new OriginEntry(image, document.dc_source()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static OriginEntry next(boolean privateEntryOnly) {
|
||||
OriginEntry e = null;
|
||||
if (privateEntryOnly) {
|
||||
e = privateImageQueueHigh.poll();
|
||||
if (e == null) e = privateImageQueueLow.poll();
|
||||
} else {
|
||||
e = publicImageQueueHigh.poll();
|
||||
if (e == null) e = privateImageQueueHigh.poll();
|
||||
if (e == null) e = publicImageQueueLow.poll();
|
||||
if (e == null) e = privateImageQueueLow.poll();
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
public static int queueSize(boolean privateEntryOnly) {
|
||||
if (privateEntryOnly) {
|
||||
return privateImageQueueHigh.size() + privateImageQueueLow.size();
|
||||
} else {
|
||||
return privateImageQueueHigh.size() + privateImageQueueLow.size() +
|
||||
publicImageQueueHigh.size() + publicImageQueueLow.size();
|
||||
}
|
||||
}
|
||||
|
||||
public static int privateQueueHighSize() {
|
||||
return privateImageQueueHigh.size();
|
||||
}
|
||||
|
||||
public static int privateQueueLowSize() {
|
||||
return privateImageQueueLow.size();
|
||||
}
|
||||
|
||||
public static int publicQueueHighSize() {
|
||||
return publicImageQueueHigh.size();
|
||||
}
|
||||
|
||||
public static int publicQueueLowSize() {
|
||||
return publicImageQueueLow.size();
|
||||
}
|
||||
|
||||
public static void clearQueues() {
|
||||
privateImageQueueHigh.clear();
|
||||
privateImageQueueLow.clear();
|
||||
publicImageQueueHigh.clear();
|
||||
publicImageQueueLow.clear();
|
||||
doubleCheck.clear();
|
||||
}
|
||||
|
||||
public static class OriginEntry {
|
||||
public htmlFilterImageEntry imageEntry;
|
||||
public yacyURL baseURL;
|
||||
public OriginEntry(htmlFilterImageEntry imageEntry, yacyURL baseURL) {
|
||||
this.imageEntry = imageEntry;
|
||||
this.baseURL = baseURL;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue